diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -1379,8 +1379,8 @@ void t6_ktls_modload(void); void t6_ktls_modunload(void); int t6_ktls_try(struct ifnet *, struct socket *, struct ktls_session *); -int t6_ktls_parse_pkt(struct mbuf *, int *, int *); -int t6_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int, u_int); +int t6_ktls_parse_pkt(struct mbuf *); +int t6_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int); #endif /* t4_keyctx.c */ diff --git a/sys/dev/cxgbe/crypto/t6_kern_tls.c b/sys/dev/cxgbe/crypto/t6_kern_tls.c --- a/sys/dev/cxgbe/crypto/t6_kern_tls.c +++ b/sys/dev/cxgbe/crypto/t6_kern_tls.c @@ -72,6 +72,7 @@ struct vi_info *vi; /* virtual interface */ struct adapter *sc; struct l2t_entry *l2te; /* L2 table entry used by this connection */ + struct sge_txq *txq; int tid; /* Connection identifier */ int tx_key_addr; @@ -95,7 +96,6 @@ /* Fields only used during setup and teardown. */ struct inpcb *inp; /* backpointer to host stack's PCB */ - struct sge_txq *txq; struct sge_wrq *ctrlq; struct clip_entry *ce; /* CLIP table entry used by this tid */ @@ -785,12 +785,7 @@ /* This should always be the last TLS record in a chain. */ MPASS(m_tls->m_next == NULL); - - /* - * XXX: Set a bogus 'nsegs' value to avoid tripping an - * assertion in mbuf_nsegs() in t4_sge.c. - */ - *nsegsp = 1; + *nsegsp = 0; return (wr_len); } @@ -893,7 +888,7 @@ } int -t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p) +t6_ktls_parse_pkt(struct mbuf *m) { struct tlspcb *tlsp; struct ether_header *eh; @@ -901,6 +896,7 @@ struct ip6_hdr *ip6; struct tcphdr *tcp; struct mbuf *m_tls; + void *items[1]; int nsegs; u_int wr_len, tot_len; @@ -976,7 +972,6 @@ * Each of the remaining mbufs in the chain should reference a * TLS record. */ - *nsegsp = 0; for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { MPASS(m_tls->m_flags & M_EXTPG); @@ -993,8 +988,8 @@ * Store 'nsegs' for the first TLS record in the * header mbuf's metadata. */ - if (*nsegsp == 0) - *nsegsp = nsegs; + if (m_tls == m->m_next) + set_mbuf_nsegs(m, nsegs); } MPASS(tot_len != 0); @@ -1035,12 +1030,13 @@ tot_len += roundup2(wr_len, EQ_ESIZE); - *len16p = tot_len / 16; + set_mbuf_len16(m, tot_len / 16); #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %d len16 %d nsegs %d", __func__, - tlsp->tid, *len16p, *nsegsp); + tlsp->tid, mbuf_len16(m), mbuf_nsegs(m)); #endif - return (0); + items[0] = m; + return (mp_ring_enqueue(tlsp->txq->r, items, 1, 256)); } /* @@ -1316,10 +1312,9 @@ "SND_NXT_RAW and SND_UNA_RAW are in different words"); static int -ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, - void *dst, struct mbuf *m, struct tcphdr *tcp, struct mbuf *m_tls, - u_int nsegs, u_int available, tcp_seq tcp_seqno, uint32_t *tsopt, - u_int pidx, bool set_l2t_idx) +ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, void *dst, + struct mbuf *m, struct tcphdr *tcp, struct mbuf *m_tls, u_int available, + tcp_seq tcp_seqno, uint32_t *tsopt, u_int pidx, bool set_l2t_idx) { struct sge_eq *eq = &txq->eq; struct tx_sdesc *txsd; @@ -1335,7 +1330,7 @@ u_int auth_start, auth_stop, auth_insert; u_int cipher_start, cipher_stop, iv_offset; u_int imm_len, mss, ndesc, offset, plen, tlen, twr_len, wr_len; - u_int fields, tx_max_offset, tx_max; + u_int fields, nsegs, tx_max_offset, tx_max; bool first_wr, last_wr, using_scratch; ndesc = 0; @@ -1457,7 +1452,7 @@ out = (void *)(wr + 1); fields = 0; if (set_l2t_idx) { - KASSERT(nsegs != 0, + KASSERT(m->m_next == m_tls, ("trying to set L2T_IX for subsequent TLS WR")); #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: tid %d set L2T_IX to %d", __func__, @@ -1469,7 +1464,7 @@ fields++; } if (tsopt != NULL && tlsp->prev_tsecr != ntohl(tsopt[1])) { - KASSERT(nsegs != 0, + KASSERT(m->m_next == m_tls, ("trying to set T_RTSEQ_RECENT for subsequent TLS WR")); #ifdef VERBOSE_TRACES CTR2(KTR_CXGBE, "%s: tid %d wrote updated T_RTSEQ_RECENT", @@ -1485,7 +1480,7 @@ } if (first_wr || tlsp->prev_seq != tx_max) { - KASSERT(nsegs != 0, + KASSERT(m->m_next == m_tls, ("trying to set TX_MAX for subsequent TLS WR")); #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, @@ -1504,7 +1499,7 @@ * reset SND_UNA_RAW to 0 so that SND_UNA == TX_MAX. */ if (tlsp->prev_seq != tx_max || mtod(m_tls, vm_offset_t) != 0) { - KASSERT(nsegs != 0, + KASSERT(m->m_next == m_tls, ("trying to clear SND_UNA_RAW for subsequent TLS WR")); #ifdef VERBOSE_TRACES CTR2(KTR_CXGBE, "%s: tid %d clearing SND_UNA_RAW", __func__, @@ -1524,7 +1519,7 @@ tlsp->prev_seq = tcp_seqno + tlen; if (first_wr || tlsp->prev_ack != ntohl(tcp->th_ack)) { - KASSERT(nsegs != 0, + KASSERT(m->m_next == m_tls, ("trying to set RCV_NXT for subsequent TLS WR")); write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_NXT, V_TCB_RCV_NXT(M_TCB_RCV_NXT), @@ -1536,7 +1531,7 @@ } if (first_wr || tlsp->prev_win != ntohs(tcp->th_win)) { - KASSERT(nsegs != 0, + KASSERT(m->m_next == m_tls, ("trying to set RCV_WND for subsequent TLS WR")); write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_WND, V_TCB_RCV_WND(M_TCB_RCV_WND), @@ -1547,8 +1542,10 @@ tlsp->prev_win = ntohs(tcp->th_win); } - /* Recalculate 'nsegs' if cached value is not available. */ - if (nsegs == 0) + /* Use cached value for first record in chain. */ + if (m->m_next == m_tls) + nsegs = mbuf_nsegs(m); + else nsegs = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset, plen - (m_tls->m_epg_hdrlen + offset)); @@ -1948,7 +1945,7 @@ } int -t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs, +t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int available) { struct sge_eq *eq = &txq->eq; @@ -2043,18 +2040,11 @@ } ndesc = ktls_write_tls_wr(tlsp, txq, dst, m, tcp, m_tls, - nsegs, available - totdesc, tcp_seqno, tsopt, pidx, - set_l2t_idx); + available - totdesc, tcp_seqno, tsopt, pidx, set_l2t_idx); totdesc += ndesc; IDXINCR(pidx, ndesc, eq->sidx); dst = &eq->desc[pidx]; - /* - * The value of nsegs from the header mbuf's metadata - * is only valid for the first TLS record. - */ - nsegs = 0; - /* Only need to set the L2T index once. */ set_l2t_idx = false; } @@ -2126,13 +2116,13 @@ } int -t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p) +t6_ktls_parse_pkt(struct mbuf *m) { return (EINVAL); } int -t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs, +t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int available) { panic("can't happen"); diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -2691,16 +2691,12 @@ #endif #ifdef KERN_TLS if (mst != NULL && mst->sw->type == IF_SND_TAG_TYPE_TLS) { - int len16; - cflags |= MC_TLS; set_mbuf_cflags(m0, cflags); - rc = t6_ktls_parse_pkt(m0, &nsegs, &len16); + rc = t6_ktls_parse_pkt(m0); if (rc != 0) goto fail; - set_mbuf_nsegs(m0, nsegs); - set_mbuf_len16(m0, len16); - return (0); + return (EINPROGRESS); } #endif if (nsegs > max_nsegs_allowed(m0, vm_wr)) { @@ -3260,8 +3256,7 @@ #ifdef KERN_TLS } else if (mbuf_cflags(m0) & MC_TLS) { ETHER_BPF_MTAP(ifp, m0); - n = t6_ktls_write_wr(txq, wr, m0, mbuf_nsegs(m0), - avail); + n = t6_ktls_write_wr(txq, wr, m0, avail); #endif } else { ETHER_BPF_MTAP(ifp, m0);