diff --git a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c index b9f7c6355b6f..01759d929c0e 100644 --- a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c +++ b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c @@ -1,1313 +1,1353 @@ /*- * Copyright (c) 2012 The FreeBSD Foundation * Copyright (c) 2015 Chelsio Communications, Inc. * All rights reserved. * * This software was developed by Edward Tomasz Napierala under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * cxgbei implementation of iSCSI Common Layer kobj(9) interface. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include "opt_inet6.h" #ifdef TCP_OFFLOAD #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/common.h" #include "common/t4_tcb.h" #include "tom/t4_tom.h" #include "cxgbei.h" /* * Use the page pod tag for the TT hash. */ #define TT_HASH(icc, tt) (G_PPOD_TAG(tt) & (icc)->cmp_hash_mask) struct cxgbei_ddp_state { struct ppod_reservation prsv; struct cxgbei_cmp cmp; }; static MALLOC_DEFINE(M_CXGBEI, "cxgbei", "cxgbei(4)"); SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Chelsio iSCSI offload"); static int first_burst_length = 8192; SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, first_burst_length, CTLFLAG_RWTUN, &first_burst_length, 0, "First burst length"); static int max_burst_length = 2 * 1024 * 1024; SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, max_burst_length, CTLFLAG_RWTUN, &max_burst_length, 0, "Maximum burst length"); static int sendspace = 1048576; SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN, &sendspace, 0, "Default send socket buffer size"); static int recvspace = 1048576; SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN, &recvspace, 0, "Default receive socket buffer size"); static volatile u_int icl_cxgbei_ncons; #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) static icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu; static icl_conn_pdu_data_segment_length_t icl_cxgbei_conn_pdu_data_segment_length; static icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data; static icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data; static icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue; static icl_conn_pdu_queue_cb_t icl_cxgbei_conn_pdu_queue_cb; static icl_conn_handoff_t icl_cxgbei_conn_handoff; static icl_conn_free_t icl_cxgbei_conn_free; static icl_conn_close_t icl_cxgbei_conn_close; static icl_conn_task_setup_t icl_cxgbei_conn_task_setup; static icl_conn_task_done_t icl_cxgbei_conn_task_done; static icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup; static icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done; static kobj_method_t icl_cxgbei_methods[] = { KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu), KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free), KOBJMETHOD(icl_conn_pdu_data_segment_length, icl_cxgbei_conn_pdu_data_segment_length), KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data), KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data), KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue), KOBJMETHOD(icl_conn_pdu_queue_cb, icl_cxgbei_conn_pdu_queue_cb), KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff), KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free), KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close), KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup), KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done), KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup), KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done), { 0, 0 } }; DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn)); void icl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) { struct icl_cxgbei_pdu *icp = ip_to_icp(ip); KASSERT(icp->ref_cnt != 0, ("freeing deleted PDU")); MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); MPASS(ic == ip->ip_conn); m_freem(ip->ip_ahs_mbuf); m_freem(ip->ip_data_mbuf); m_freem(ip->ip_bhs_mbuf); KASSERT(ic != NULL || icp->ref_cnt == 1, ("orphaned PDU has oustanding references")); if (atomic_fetchadd_int(&icp->ref_cnt, -1) != 1) return; free(icp, M_CXGBEI); #ifdef DIAGNOSTIC if (__predict_true(ic != NULL)) refcount_release(&ic->ic_outstanding_pdus); #endif } static void icl_cxgbei_pdu_call_cb(struct icl_pdu *ip) { struct icl_cxgbei_pdu *icp = ip_to_icp(ip); MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); if (icp->cb != NULL) icp->cb(ip, icp->error); #ifdef DIAGNOSTIC if (__predict_true(ip->ip_conn != NULL)) refcount_release(&ip->ip_conn->ic_outstanding_pdus); #endif free(icp, M_CXGBEI); } static void icl_cxgbei_pdu_done(struct icl_pdu *ip, int error) { struct icl_cxgbei_pdu *icp = ip_to_icp(ip); if (error != 0) icp->error = error; m_freem(ip->ip_ahs_mbuf); ip->ip_ahs_mbuf = NULL; m_freem(ip->ip_data_mbuf); ip->ip_data_mbuf = NULL; m_freem(ip->ip_bhs_mbuf); ip->ip_bhs_mbuf = NULL; /* * All other references to this PDU should have been dropped * by the m_freem() of ip_data_mbuf. */ if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) icl_cxgbei_pdu_call_cb(ip); else __assert_unreachable(); } static void icl_cxgbei_mbuf_done(struct mbuf *mb) { struct icl_cxgbei_pdu *icp = (struct icl_cxgbei_pdu *)mb->m_ext.ext_arg1; /* * NB: mb_free_mext() might leave ref_cnt as 1 without * decrementing it if it hits the fast path in the ref_cnt * check. */ icl_cxgbei_pdu_call_cb(&icp->ip); } struct icl_pdu * icl_cxgbei_new_pdu(int flags) { struct icl_cxgbei_pdu *icp; struct icl_pdu *ip; struct mbuf *m; icp = malloc(sizeof(*icp), M_CXGBEI, flags | M_ZERO); if (__predict_false(icp == NULL)) return (NULL); icp->icp_signature = CXGBEI_PDU_SIGNATURE; icp->ref_cnt = 1; ip = &icp->ip; m = m_gethdr(flags, MT_DATA); if (__predict_false(m == NULL)) { free(icp, M_CXGBEI); return (NULL); } ip->ip_bhs_mbuf = m; ip->ip_bhs = mtod(m, struct iscsi_bhs *); memset(ip->ip_bhs, 0, sizeof(*ip->ip_bhs)); m->m_len = sizeof(struct iscsi_bhs); m->m_pkthdr.len = m->m_len; return (ip); } void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic) { ip->ip_conn = ic; #ifdef DIAGNOSTIC refcount_acquire(&ic->ic_outstanding_pdus); #endif } /* * Allocate icl_pdu with empty BHS to fill up by the caller. */ static struct icl_pdu * icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags) { struct icl_pdu *ip; ip = icl_cxgbei_new_pdu(flags); if (__predict_false(ip == NULL)) return (NULL); icl_cxgbei_new_pdu_set_conn(ip, ic); return (ip); } static size_t icl_pdu_data_segment_length(const struct icl_pdu *request) { uint32_t len = 0; len += request->ip_bhs->bhs_data_segment_len[0]; len <<= 8; len += request->ip_bhs->bhs_data_segment_len[1]; len <<= 8; len += request->ip_bhs->bhs_data_segment_len[2]; return (len); } size_t icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic, const struct icl_pdu *request) { return (icl_pdu_data_segment_length(request)); } static struct mbuf * finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp) { struct icl_pdu *ip = &icp->ip; uint8_t ulp_submode, padding; struct mbuf *m, *last; struct iscsi_bhs *bhs; /* * Fix up the data segment mbuf first. */ m = ip->ip_data_mbuf; ulp_submode = icc->ulp_submode; if (m) { last = m_last(m); /* * Round up the data segment to a 4B boundary. Pad with 0 if * necessary. There will definitely be room in the mbuf. */ padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len; if (padding) { bzero(mtod(last, uint8_t *) + last->m_len, padding); last->m_len += padding; } } else { MPASS(ip->ip_data_len == 0); ulp_submode &= ~ULP_CRC_DATA; padding = 0; } /* * Now the header mbuf that has the BHS. */ m = ip->ip_bhs_mbuf; MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs)); MPASS(m->m_len == sizeof(struct iscsi_bhs)); bhs = ip->ip_bhs; bhs->bhs_data_segment_len[2] = ip->ip_data_len; bhs->bhs_data_segment_len[1] = ip->ip_data_len >> 8; bhs->bhs_data_segment_len[0] = ip->ip_data_len >> 16; /* * Extract mbuf chain from PDU. */ m->m_pkthdr.len += ip->ip_data_len + padding; m->m_next = ip->ip_data_mbuf; set_mbuf_ulp_submode(m, ulp_submode); ip->ip_bhs_mbuf = NULL; ip->ip_data_mbuf = NULL; ip->ip_bhs = NULL; /* * Drop PDU reference on icp. Additional references might * still be held by zero-copy PDU buffers (ICL_NOCOPY). */ if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) icl_cxgbei_pdu_call_cb(ip); return (m); } int icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip, const void *addr, size_t len, int flags) { struct icl_cxgbei_pdu *icp = ip_to_icp(ip); struct mbuf *m, *m_tail; const char *src; MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); MPASS(ic == ip->ip_conn); KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len)); m_tail = ip->ip_data_mbuf; if (m_tail != NULL) for (; m_tail->m_next != NULL; m_tail = m_tail->m_next) ; if (flags & ICL_NOCOPY) { m = m_get(flags & ~ICL_NOCOPY, MT_DATA); if (m == NULL) { ICL_WARN("failed to allocate mbuf"); return (ENOMEM); } m->m_flags |= M_RDONLY; m_extaddref(m, __DECONST(char *, addr), len, &icp->ref_cnt, icl_cxgbei_mbuf_done, icp, NULL); m->m_len = len; if (ip->ip_data_mbuf == NULL) { ip->ip_data_mbuf = m; ip->ip_data_len = len; } else { m_tail->m_next = m; m_tail = m_tail->m_next; ip->ip_data_len += len; } return (0); } src = (const char *)addr; /* Allocate as jumbo mbufs of size MJUM16BYTES. */ while (len >= MJUM16BYTES) { m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES); if (__predict_false(m == NULL)) { if ((flags & M_WAITOK) != 0) { /* Fall back to non-jumbo mbufs. */ break; } return (ENOMEM); } memcpy(mtod(m, void *), src, MJUM16BYTES); m->m_len = MJUM16BYTES; if (ip->ip_data_mbuf == NULL) { ip->ip_data_mbuf = m_tail = m; ip->ip_data_len = MJUM16BYTES; } else { m_tail->m_next = m; m_tail = m_tail->m_next; ip->ip_data_len += MJUM16BYTES; } src += MJUM16BYTES; len -= MJUM16BYTES; } /* Allocate mbuf chain for the remaining data. */ if (len != 0) { m = m_getm2(NULL, len, flags, MT_DATA, 0); if (__predict_false(m == NULL)) return (ENOMEM); if (ip->ip_data_mbuf == NULL) { ip->ip_data_mbuf = m; ip->ip_data_len = len; } else { m_tail->m_next = m; ip->ip_data_len += len; } for (; m != NULL; m = m->m_next) { m->m_len = min(len, M_SIZE(m)); memcpy(mtod(m, void *), src, m->m_len); src += m->m_len; len -= m->m_len; } MPASS(len == 0); } MPASS(ip->ip_data_len <= ic->ic_max_send_data_segment_length); return (0); } void icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, size_t off, void *addr, size_t len) { struct icl_cxgbei_pdu *icp = ip_to_icp(ip); if (icp->icp_flags & ICPF_RX_DDP) return; /* data is DDP'ed, no need to copy */ m_copydata(ip->ip_data_mbuf, off, len, addr); } void icl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) { icl_cxgbei_conn_pdu_queue_cb(ic, ip, NULL); } void icl_cxgbei_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip, icl_pdu_cb cb) { struct epoch_tracker et; struct icl_cxgbei_conn *icc = ic_to_icc(ic); struct icl_cxgbei_pdu *icp = ip_to_icp(ip); struct socket *so = ic->ic_socket; struct toepcb *toep = icc->toep; struct inpcb *inp; struct mbuf *m; MPASS(ic == ip->ip_conn); MPASS(ip->ip_bhs_mbuf != NULL); /* The kernel doesn't generate PDUs with AHS. */ MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0); ICL_CONN_LOCK_ASSERT(ic); icp->cb = cb; /* NOTE: sowriteable without so_snd lock is a mostly harmless race. */ if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) { icl_cxgbei_pdu_done(ip, ENOTCONN); return; } m = finalize_pdu(icc, icp); M_ASSERTPKTHDR(m); MPASS((m->m_pkthdr.len & 3) == 0); /* * Do not get inp from toep->inp as the toepcb might have detached * already. */ inp = sotoinpcb(so); CURVNET_SET(toep->vnet); NET_EPOCH_ENTER(et); INP_WLOCK(inp); if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) || __predict_false((toep->flags & TPF_ATTACHED) == 0)) m_freem(m); else { mbufq_enqueue(&toep->ulp_pduq, m); t4_push_pdus(icc->sc, toep, 0); } INP_WUNLOCK(inp); NET_EPOCH_EXIT(et); CURVNET_RESTORE(); } static struct icl_conn * icl_cxgbei_new_conn(const char *name, struct mtx *lock) { struct icl_cxgbei_conn *icc; struct icl_conn *ic; refcount_acquire(&icl_cxgbei_ncons); icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE, M_WAITOK | M_ZERO); icc->icc_signature = CXGBEI_CONN_SIGNATURE; STAILQ_INIT(&icc->rcvd_pdus); icc->cmp_table = hashinit(64, M_CXGBEI, &icc->cmp_hash_mask); mtx_init(&icc->cmp_lock, "cxgbei_cmp", NULL, MTX_DEF); ic = &icc->ic; ic->ic_lock = lock; /* XXXNP: review. Most of these icl_conn fields aren't really used */ STAILQ_INIT(&ic->ic_to_send); cv_init(&ic->ic_send_cv, "icl_cxgbei_tx"); cv_init(&ic->ic_receive_cv, "icl_cxgbei_rx"); #ifdef DIAGNOSTIC refcount_init(&ic->ic_outstanding_pdus, 0); #endif ic->ic_name = name; ic->ic_offload = "cxgbei"; ic->ic_unmapped = false; CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); return (ic); } void icl_cxgbei_conn_free(struct icl_conn *ic) { struct icl_cxgbei_conn *icc = ic_to_icc(ic); MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); cv_destroy(&ic->ic_send_cv); cv_destroy(&ic->ic_receive_cv); mtx_destroy(&icc->cmp_lock); hashdestroy(icc->cmp_table, M_CXGBEI, icc->cmp_hash_mask); kobj_delete((struct kobj *)icc, M_CXGBE); refcount_release(&icl_cxgbei_ncons); } static int icl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so, int sspace, int rspace) { struct sockopt opt; int error, one = 1, ss, rs; ss = max(sendspace, sspace); rs = max(recvspace, rspace); error = soreserve(so, ss, rs); if (error != 0) { icl_cxgbei_conn_close(ic); return (error); } SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_flags |= SB_AUTOSIZE; SOCKBUF_UNLOCK(&so->so_snd); SOCKBUF_LOCK(&so->so_rcv); so->so_rcv.sb_flags |= SB_AUTOSIZE; SOCKBUF_UNLOCK(&so->so_rcv); /* * Disable Nagle. */ bzero(&opt, sizeof(opt)); opt.sopt_dir = SOPT_SET; opt.sopt_level = IPPROTO_TCP; opt.sopt_name = TCP_NODELAY; opt.sopt_val = &one; opt.sopt_valsize = sizeof(one); error = sosetopt(so, &opt); if (error != 0) { icl_cxgbei_conn_close(ic); return (error); } return (0); } /* * Request/response structure used to find out the adapter offloading a socket. */ struct find_ofld_adapter_rr { struct socket *so; struct adapter *sc; /* result */ }; static void find_offload_adapter(struct adapter *sc, void *arg) { struct find_ofld_adapter_rr *fa = arg; struct socket *so = fa->so; struct tom_data *td = sc->tom_softc; struct tcpcb *tp; struct inpcb *inp; /* Non-TCP were filtered out earlier. */ MPASS(so->so_proto->pr_protocol == IPPROTO_TCP); if (fa->sc != NULL) return; /* Found already. */ if (td == NULL) return; /* TOE not enabled on this adapter. */ inp = sotoinpcb(so); INP_WLOCK(inp); if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { tp = intotcpcb(inp); if (tp->t_flags & TF_TOE && tp->tod == &td->tod) fa->sc = sc; /* Found. */ } INP_WUNLOCK(inp); } /* XXXNP: move this to t4_tom. */ static void send_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen) { struct wrqe *wr; struct fw_flowc_wr *flowc; const u_int nparams = 1; u_int flowclen; struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq); if (wr == NULL) { /* XXX */ panic("%s: allocation failure.", __func__); } flowc = wrtod(wr); memset(flowc, 0, wr->wr_len); flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | V_FW_FLOWC_WR_NPARAMS(nparams)); flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | V_FW_WR_FLOWID(toep->tid)); flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; flowc->mnemval[0].val = htobe32(maxlen); txsd->tx_credits = howmany(flowclen, 16); txsd->plen = 0; KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, ("%s: not enough credits (%d)", __func__, toep->tx_credits)); toep->tx_credits -= txsd->tx_credits; if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) toep->txsd_pidx = 0; toep->txsd_avail--; t4_wrq_tx(sc, wr); } static void set_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, u_int ulp_submode) { uint64_t val; CTR3(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, submode=%#x", __func__, toep->tid, ulp_submode); val = V_TCB_ULP_TYPE(ULP_MODE_ISCSI) | V_TCB_ULP_RAW(ulp_submode); t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_ULP_TYPE, V_TCB_ULP_TYPE(M_TCB_ULP_TYPE) | V_TCB_ULP_RAW(M_TCB_ULP_RAW), val, 0, 0); val = V_TF_RX_FLOW_CONTROL_DISABLE(1ULL); t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, val, val, 0, 0); } /* * XXXNP: Who is responsible for cleaning up the socket if this returns with an * error? Review all error paths. * * XXXNP: What happens to the socket's fd reference if the operation is * successful, and how does that affect the socket's life cycle? */ int icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd) { struct icl_cxgbei_conn *icc = ic_to_icc(ic); struct cxgbei_data *ci; struct find_ofld_adapter_rr fa; struct file *fp; struct socket *so; struct inpcb *inp; struct tcpcb *tp; struct toepcb *toep; cap_rights_t rights; int error; MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); ICL_CONN_LOCK_ASSERT_NOT(ic); /* * Steal the socket from userland. */ error = fget(curthread, fd, cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); if (error != 0) return (error); if (fp->f_type != DTYPE_SOCKET) { fdrop(fp, curthread); return (EINVAL); } so = fp->f_data; if (so->so_type != SOCK_STREAM || so->so_proto->pr_protocol != IPPROTO_TCP) { fdrop(fp, curthread); return (EINVAL); } ICL_CONN_LOCK(ic); if (ic->ic_socket != NULL) { ICL_CONN_UNLOCK(ic); fdrop(fp, curthread); return (EBUSY); } ic->ic_disconnecting = false; ic->ic_socket = so; fp->f_ops = &badfileops; fp->f_data = NULL; fdrop(fp, curthread); ICL_CONN_UNLOCK(ic); /* Find the adapter offloading this socket. */ fa.sc = NULL; fa.so = so; t4_iterate(find_offload_adapter, &fa); if (fa.sc == NULL) return (EINVAL); icc->sc = fa.sc; ci = icc->sc->iscsi_ulp_softc; inp = sotoinpcb(so); INP_WLOCK(inp); tp = intotcpcb(inp); if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) error = EBUSY; else { /* * socket could not have been "unoffloaded" if here. */ MPASS(tp->t_flags & TF_TOE); MPASS(tp->tod != NULL); MPASS(tp->t_toe != NULL); toep = tp->t_toe; MPASS(toep->vi->adapter == icc->sc); icc->toep = toep; icc->cwt = cxgbei_select_worker_thread(icc); icc->ulp_submode = 0; if (ic->ic_header_crc32c) icc->ulp_submode |= ULP_CRC_HEADER; if (ic->ic_data_crc32c) icc->ulp_submode |= ULP_CRC_DATA; so->so_options |= SO_NO_DDP; toep->params.ulp_mode = ULP_MODE_ISCSI; toep->ulpcb = icc; send_iscsi_flowc_wr(icc->sc, toep, ci->max_tx_pdu_len); set_ulp_mode_iscsi(icc->sc, toep, icc->ulp_submode); error = 0; } INP_WUNLOCK(inp); if (error == 0) { error = icl_cxgbei_setsockopt(ic, so, ci->max_tx_pdu_len, ci->max_rx_pdu_len); } return (error); } void icl_cxgbei_conn_close(struct icl_conn *ic) { struct icl_cxgbei_conn *icc = ic_to_icc(ic); struct icl_pdu *ip; struct socket *so; struct sockbuf *sb; struct inpcb *inp; struct toepcb *toep = icc->toep; MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); ICL_CONN_LOCK_ASSERT_NOT(ic); ICL_CONN_LOCK(ic); so = ic->ic_socket; if (ic->ic_disconnecting || so == NULL) { CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p", __func__, icc, ic->ic_disconnecting, so); ICL_CONN_UNLOCK(ic); return; } ic->ic_disconnecting = true; /* These are unused in this driver right now. */ MPASS(STAILQ_EMPTY(&ic->ic_to_send)); MPASS(ic->ic_receive_pdu == NULL); #ifdef DIAGNOSTIC KASSERT(ic->ic_outstanding_pdus == 0, ("destroying session with %d outstanding PDUs", ic->ic_outstanding_pdus)); #endif ICL_CONN_UNLOCK(ic); CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1, icc); inp = sotoinpcb(so); sb = &so->so_rcv; INP_WLOCK(inp); if (toep != NULL) { /* NULL if connection was never offloaded. */ toep->ulpcb = NULL; mbufq_drain(&toep->ulp_pduq); SOCKBUF_LOCK(sb); if (icc->rx_flags & RXF_ACTIVE) { volatile u_int *p = &icc->rx_flags; SOCKBUF_UNLOCK(sb); INP_WUNLOCK(inp); while (*p & RXF_ACTIVE) pause("conclo", 1); INP_WLOCK(inp); SOCKBUF_LOCK(sb); } while (!STAILQ_EMPTY(&icc->rcvd_pdus)) { ip = STAILQ_FIRST(&icc->rcvd_pdus); STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next); icl_cxgbei_pdu_done(ip, ENOTCONN); } SOCKBUF_UNLOCK(sb); } INP_WUNLOCK(inp); ICL_CONN_LOCK(ic); ic->ic_socket = NULL; ICL_CONN_UNLOCK(ic); /* * XXXNP: we should send RST instead of FIN when PDUs held in various * queues were purged instead of delivered reliably but soabort isn't * really general purpose and wouldn't do the right thing here. */ soclose(so); } static void cxgbei_insert_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp, uint32_t tt) { #ifdef INVARIANTS struct cxgbei_cmp *cmp2; #endif cmp->tt = tt; mtx_lock(&icc->cmp_lock); #ifdef INVARIANTS LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, tt)], link) { KASSERT(cmp2->tt != tt, ("%s: duplicate cmp", __func__)); } #endif LIST_INSERT_HEAD(&icc->cmp_table[TT_HASH(icc, tt)], cmp, link); mtx_unlock(&icc->cmp_lock); } struct cxgbei_cmp * cxgbei_find_cmp(struct icl_cxgbei_conn *icc, uint32_t tt) { struct cxgbei_cmp *cmp; mtx_lock(&icc->cmp_lock); LIST_FOREACH(cmp, &icc->cmp_table[TT_HASH(icc, tt)], link) { if (cmp->tt == tt) break; } mtx_unlock(&icc->cmp_lock); return (cmp); } static void cxgbei_rm_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp) { #ifdef INVARIANTS struct cxgbei_cmp *cmp2; #endif mtx_lock(&icc->cmp_lock); #ifdef INVARIANTS LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, cmp->tt)], link) { if (cmp2 == cmp) goto found; } panic("%s: could not find cmp", __func__); found: #endif LIST_REMOVE(cmp, link); mtx_unlock(&icc->cmp_lock); } int icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, struct ccb_scsiio *csio, uint32_t *ittp, void **arg) { struct icl_cxgbei_conn *icc = ic_to_icc(ic); struct toepcb *toep = icc->toep; struct adapter *sc = icc->sc; struct cxgbei_data *ci = sc->iscsi_ulp_softc; struct ppod_region *pr = &ci->pr; struct cxgbei_ddp_state *ddp; struct ppod_reservation *prsv; + struct inpcb *inp; + struct mbufq mq; uint32_t itt; int rc = 0; /* This is for the offload driver's state. Must not be set already. */ MPASS(arg != NULL); MPASS(*arg == NULL); if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN || csio->dxfer_len < ci->ddp_threshold) { no_ddp: /* * No DDP for this I/O. Allocate an ITT (based on the one * passed in) that cannot be a valid hardware DDP tag in the * iSCSI region. */ itt = *ittp & M_PPOD_TAG; itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit; *ittp = htobe32(itt); MPASS(*arg == NULL); /* State is maintained for DDP only. */ if (rc != 0) counter_u64_add( toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1); return (0); } /* * Reserve resources for DDP, update the itt that should be used in the * PDU, and save DDP specific state for this I/O in *arg. */ ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO); if (ddp == NULL) { rc = ENOMEM; goto no_ddp; } prsv = &ddp->prsv; /* XXX add support for all CAM_DATA_ types */ MPASS((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR); rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr, csio->dxfer_len, prsv); if (rc != 0) { free(ddp, M_CXGBEI); goto no_ddp; } + mbufq_init(&mq, INT_MAX); rc = t4_write_page_pods_for_buf(sc, toep, prsv, - (vm_offset_t)csio->data_ptr, csio->dxfer_len); + (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq); if (__predict_false(rc != 0)) { + mbufq_drain(&mq); t4_free_page_pods(prsv); free(ddp, M_CXGBEI); goto no_ddp; } + /* + * Do not get inp from toep->inp as the toepcb might have + * detached already. + */ + inp = sotoinpcb(ic->ic_socket); + INP_WLOCK(inp); + if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) { + INP_WUNLOCK(inp); + mbufq_drain(&mq); + t4_free_page_pods(prsv); + free(ddp, M_CXGBEI); + return (ECONNRESET); + } + mbufq_concat(&toep->ulp_pduq, &mq); + INP_WUNLOCK(inp); + ddp->cmp.last_datasn = -1; cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); *ittp = htobe32(prsv->prsv_tag); *arg = prsv; counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1); return (0); } void icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg) { if (arg != NULL) { struct cxgbei_ddp_state *ddp = arg; cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp); t4_free_page_pods(&ddp->prsv); free(ddp, M_CXGBEI); } } static inline bool ddp_sgl_check(struct ctl_sg_entry *sg, int entries, int xferlen) { int total_len = 0; MPASS(entries > 0); if (((vm_offset_t)sg[--entries].addr & 3U) != 0) return (false); total_len += sg[entries].len; while (--entries >= 0) { if (((vm_offset_t)sg[entries].addr & PAGE_MASK) != 0 || (sg[entries].len % PAGE_SIZE) != 0) return (false); total_len += sg[entries].len; } MPASS(total_len == xferlen); return (true); } /* XXXNP: PDU should be passed in as parameter, like on the initiator. */ #define io_to_request_pdu(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr) #define io_to_ddp_state(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr) int icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, uint32_t *tttp, void **arg) { struct icl_cxgbei_conn *icc = ic_to_icc(ic); struct toepcb *toep = icc->toep; struct ctl_scsiio *ctsio = &io->scsiio; struct adapter *sc = icc->sc; struct cxgbei_data *ci = sc->iscsi_ulp_softc; struct ppod_region *pr = &ci->pr; struct cxgbei_ddp_state *ddp; struct ppod_reservation *prsv; struct ctl_sg_entry *sgl, sg_entry; + struct inpcb *inp; + struct mbufq mq; int sg_entries = ctsio->kern_sg_entries; uint32_t ttt; int xferlen, rc = 0, alias; /* This is for the offload driver's state. Must not be set already. */ MPASS(arg != NULL); MPASS(*arg == NULL); if (ctsio->ext_data_filled == 0) { int first_burst; struct icl_pdu *ip = io_to_request_pdu(io); #ifdef INVARIANTS struct icl_cxgbei_pdu *icp = ip_to_icp(ip); MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); MPASS(ic == ip->ip_conn); MPASS(ip->ip_bhs_mbuf != NULL); #endif first_burst = icl_pdu_data_segment_length(ip); /* * Note that ICL calls conn_transfer_setup even if the first * burst had everything and there's nothing left to transfer. * * NB: The CTL frontend might have provided a buffer * whose length (kern_data_len) is smaller than the * FirstBurstLength of unsolicited data. Treat those * as an empty transfer. */ xferlen = ctsio->kern_data_len; if (xferlen < first_burst || xferlen - first_burst < ci->ddp_threshold) { no_ddp: /* * No DDP for this transfer. Allocate a TTT (based on * the one passed in) that cannot be a valid hardware * DDP tag in the iSCSI region. */ ttt = *tttp & M_PPOD_TAG; ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit; *tttp = htobe32(ttt); MPASS(io_to_ddp_state(io) == NULL); if (rc != 0) counter_u64_add( toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1); return (0); } if (sg_entries == 0) { sgl = &sg_entry; sgl->len = xferlen; sgl->addr = (void *)ctsio->kern_data_ptr; sg_entries = 1; } else sgl = (void *)ctsio->kern_data_ptr; if (!ddp_sgl_check(sgl, sg_entries, xferlen)) goto no_ddp; /* * Reserve resources for DDP, update the ttt that should be used * in the PDU, and save DDP specific state for this I/O. */ MPASS(io_to_ddp_state(io) == NULL); ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO); if (ddp == NULL) { rc = ENOMEM; goto no_ddp; } prsv = &ddp->prsv; rc = t4_alloc_page_pods_for_sgl(pr, sgl, sg_entries, prsv); if (rc != 0) { free(ddp, M_CXGBEI); goto no_ddp; } + mbufq_init(&mq, INT_MAX); rc = t4_write_page_pods_for_sgl(sc, toep, prsv, sgl, sg_entries, - xferlen); + xferlen, &mq); if (__predict_false(rc != 0)) { + mbufq_drain(&mq); t4_free_page_pods(prsv); free(ddp, M_CXGBEI); goto no_ddp; } + /* + * Do not get inp from toep->inp as the toepcb might + * have detached already. + */ + inp = sotoinpcb(ic->ic_socket); + INP_WLOCK(inp); + if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) { + INP_WUNLOCK(inp); + mbufq_drain(&mq); + t4_free_page_pods(prsv); + free(ddp, M_CXGBEI); + return (ECONNRESET); + } + mbufq_concat(&toep->ulp_pduq, &mq); + INP_WUNLOCK(inp); + ddp->cmp.next_buffer_offset = ctsio->kern_rel_offset + first_burst; ddp->cmp.last_datasn = -1; cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); *tttp = htobe32(prsv->prsv_tag); io_to_ddp_state(io) = ddp; *arg = ctsio; counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1); return (0); } /* * In the middle of an I/O. A non-NULL page pod reservation indicates * that a DDP buffer is being used for the I/O. */ ddp = io_to_ddp_state(ctsio); if (ddp == NULL) goto no_ddp; prsv = &ddp->prsv; alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift; alias++; prsv->prsv_tag &= ~pr->pr_alias_mask; prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask; ddp->cmp.next_datasn = 0; ddp->cmp.last_datasn = -1; cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); *tttp = htobe32(prsv->prsv_tag); *arg = ctsio; return (0); } void icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg) { struct ctl_scsiio *ctsio = arg; if (ctsio != NULL) { struct cxgbei_ddp_state *ddp; ddp = io_to_ddp_state(ctsio); MPASS(ddp != NULL); cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp); if (ctsio->kern_data_len == ctsio->ext_data_filled || ic->ic_disconnecting) { t4_free_page_pods(&ddp->prsv); free(ddp, M_CXGBEI); io_to_ddp_state(ctsio) = NULL; } } } static void cxgbei_limits(struct adapter *sc, void *arg) { struct icl_drv_limits *idl = arg; struct cxgbei_data *ci; int max_dsl; if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims") != 0) return; if (uld_active(sc, ULD_ISCSI)) { ci = sc->iscsi_ulp_softc; MPASS(ci != NULL); /* * AHS is not supported by the kernel so we'll not account for * it either in our PDU len -> data segment len conversions. */ max_dsl = ci->max_rx_pdu_len - ISCSI_BHS_SIZE - ISCSI_HEADER_DIGEST_SIZE - ISCSI_DATA_DIGEST_SIZE; if (idl->idl_max_recv_data_segment_length > max_dsl) idl->idl_max_recv_data_segment_length = max_dsl; max_dsl = ci->max_tx_pdu_len - ISCSI_BHS_SIZE - ISCSI_HEADER_DIGEST_SIZE - ISCSI_DATA_DIGEST_SIZE; if (idl->idl_max_send_data_segment_length > max_dsl) idl->idl_max_send_data_segment_length = max_dsl; } end_synchronized_op(sc, LOCK_HELD); } static int icl_cxgbei_limits(struct icl_drv_limits *idl) { /* Maximum allowed by the RFC. cxgbei_limits will clip them. */ idl->idl_max_recv_data_segment_length = (1 << 24) - 1; idl->idl_max_send_data_segment_length = (1 << 24) - 1; /* These are somewhat arbitrary. */ idl->idl_max_burst_length = max_burst_length; idl->idl_first_burst_length = first_burst_length; t4_iterate(cxgbei_limits, idl); return (0); } int icl_cxgbei_mod_load(void) { int rc; refcount_init(&icl_cxgbei_ncons, 0); rc = icl_register("cxgbei", false, -100, icl_cxgbei_limits, icl_cxgbei_new_conn); return (rc); } int icl_cxgbei_mod_unload(void) { if (icl_cxgbei_ncons != 0) return (EBUSY); icl_unregister("cxgbei", false); return (0); } #endif diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c index 34c01674659a..2b58cb60d4fd 100644 --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -1,2157 +1,2140 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_inet.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define TCPSTATES #include #include #include #include #include #include #include #include #include #include #include #ifdef TCP_OFFLOAD #include "common/common.h" #include "common/t4_msg.h" #include "common/t4_regs.h" #include "common/t4_tcb.h" #include "tom/t4_tom.h" /* * Use the 'backend3' field in AIO jobs to store the amount of data * received by the AIO job so far. */ #define aio_received backend3 static void aio_ddp_requeue_task(void *context, int pending); static void ddp_complete_all(struct toepcb *toep, int error); static void t4_aio_cancel_active(struct kaiocb *job); static void t4_aio_cancel_queued(struct kaiocb *job); static TAILQ_HEAD(, pageset) ddp_orphan_pagesets; static struct mtx ddp_orphan_pagesets_lock; static struct task ddp_orphan_task; #define MAX_DDP_BUFFER_SIZE (M_TCB_RX_DDP_BUF0_LEN) /* * A page set holds information about a buffer used for DDP. The page * set holds resources such as the VM pages backing the buffer (either * held or wired) and the page pods associated with the buffer. * Recently used page sets are cached to allow for efficient reuse of * buffers (avoiding the need to re-fault in pages, hold them, etc.). * Note that cached page sets keep the backing pages wired. The * number of wired pages is capped by only allowing for two wired * pagesets per connection. This is not a perfect cap, but is a * trade-off for performance. * * If an application ping-pongs two buffers for a connection via * aio_read(2) then those buffers should remain wired and expensive VM * fault lookups should be avoided after each buffer has been used * once. If an application uses more than two buffers then this will * fall back to doing expensive VM fault lookups for each operation. */ static void free_pageset(struct tom_data *td, struct pageset *ps) { vm_page_t p; int i; if (ps->prsv.prsv_nppods > 0) t4_free_page_pods(&ps->prsv); for (i = 0; i < ps->npages; i++) { p = ps->pages[i]; vm_page_unwire(p, PQ_INACTIVE); } mtx_lock(&ddp_orphan_pagesets_lock); TAILQ_INSERT_TAIL(&ddp_orphan_pagesets, ps, link); taskqueue_enqueue(taskqueue_thread, &ddp_orphan_task); mtx_unlock(&ddp_orphan_pagesets_lock); } static void ddp_free_orphan_pagesets(void *context, int pending) { struct pageset *ps; mtx_lock(&ddp_orphan_pagesets_lock); while (!TAILQ_EMPTY(&ddp_orphan_pagesets)) { ps = TAILQ_FIRST(&ddp_orphan_pagesets); TAILQ_REMOVE(&ddp_orphan_pagesets, ps, link); mtx_unlock(&ddp_orphan_pagesets_lock); if (ps->vm) vmspace_free(ps->vm); free(ps, M_CXGBE); mtx_lock(&ddp_orphan_pagesets_lock); } mtx_unlock(&ddp_orphan_pagesets_lock); } static void recycle_pageset(struct toepcb *toep, struct pageset *ps) { DDP_ASSERT_LOCKED(toep); if (!(toep->ddp.flags & DDP_DEAD)) { KASSERT(toep->ddp.cached_count + toep->ddp.active_count < nitems(toep->ddp.db), ("too many wired pagesets")); TAILQ_INSERT_HEAD(&toep->ddp.cached_pagesets, ps, link); toep->ddp.cached_count++; } else free_pageset(toep->td, ps); } static void ddp_complete_one(struct kaiocb *job, int error) { long copied; /* * If this job had copied data out of the socket buffer before * it was cancelled, report it as a short read rather than an * error. */ copied = job->aio_received; if (copied != 0 || error == 0) aio_complete(job, copied, 0); else aio_complete(job, -1, error); } static void free_ddp_buffer(struct tom_data *td, struct ddp_buffer *db) { if (db->job) { /* * XXX: If we are un-offloading the socket then we * should requeue these on the socket somehow. If we * got a FIN from the remote end, then this completes * any remaining requests with an EOF read. */ if (!aio_clear_cancel_function(db->job)) ddp_complete_one(db->job, 0); } if (db->ps) free_pageset(td, db->ps); } void ddp_init_toep(struct toepcb *toep) { TAILQ_INIT(&toep->ddp.aiojobq); TASK_INIT(&toep->ddp.requeue_task, 0, aio_ddp_requeue_task, toep); toep->ddp.flags = DDP_OK; toep->ddp.active_id = -1; mtx_init(&toep->ddp.lock, "t4 ddp", NULL, MTX_DEF); } void ddp_uninit_toep(struct toepcb *toep) { mtx_destroy(&toep->ddp.lock); } void release_ddp_resources(struct toepcb *toep) { struct pageset *ps; int i; DDP_LOCK(toep); toep->ddp.flags |= DDP_DEAD; for (i = 0; i < nitems(toep->ddp.db); i++) { free_ddp_buffer(toep->td, &toep->ddp.db[i]); } while ((ps = TAILQ_FIRST(&toep->ddp.cached_pagesets)) != NULL) { TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link); free_pageset(toep->td, ps); } ddp_complete_all(toep, 0); DDP_UNLOCK(toep); } #ifdef INVARIANTS void ddp_assert_empty(struct toepcb *toep) { int i; MPASS(!(toep->ddp.flags & DDP_TASK_ACTIVE)); for (i = 0; i < nitems(toep->ddp.db); i++) { MPASS(toep->ddp.db[i].job == NULL); MPASS(toep->ddp.db[i].ps == NULL); } MPASS(TAILQ_EMPTY(&toep->ddp.cached_pagesets)); MPASS(TAILQ_EMPTY(&toep->ddp.aiojobq)); } #endif static void complete_ddp_buffer(struct toepcb *toep, struct ddp_buffer *db, unsigned int db_idx) { unsigned int db_flag; toep->ddp.active_count--; if (toep->ddp.active_id == db_idx) { if (toep->ddp.active_count == 0) { KASSERT(toep->ddp.db[db_idx ^ 1].job == NULL, ("%s: active_count mismatch", __func__)); toep->ddp.active_id = -1; } else toep->ddp.active_id ^= 1; #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: tid %u, ddp_active_id = %d", __func__, toep->tid, toep->ddp.active_id); #endif } else { KASSERT(toep->ddp.active_count != 0 && toep->ddp.active_id != -1, ("%s: active count mismatch", __func__)); } db->cancel_pending = 0; db->job = NULL; recycle_pageset(toep, db->ps); db->ps = NULL; db_flag = db_idx == 1 ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE; KASSERT(toep->ddp.flags & db_flag, ("%s: DDP buffer not active. toep %p, ddp_flags 0x%x", __func__, toep, toep->ddp.flags)); toep->ddp.flags &= ~db_flag; } /* XXX: handle_ddp_data code duplication */ void insert_ddp_data(struct toepcb *toep, uint32_t n) { struct inpcb *inp = toep->inp; struct tcpcb *tp = intotcpcb(inp); struct ddp_buffer *db; struct kaiocb *job; size_t placed; long copied; unsigned int db_flag, db_idx; INP_WLOCK_ASSERT(inp); DDP_ASSERT_LOCKED(toep); tp->rcv_nxt += n; #ifndef USE_DDP_RX_FLOW_CONTROL KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__)); tp->rcv_wnd -= n; #endif CTR2(KTR_CXGBE, "%s: placed %u bytes before falling out of DDP", __func__, n); while (toep->ddp.active_count > 0) { MPASS(toep->ddp.active_id != -1); db_idx = toep->ddp.active_id; db_flag = db_idx == 1 ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE; MPASS((toep->ddp.flags & db_flag) != 0); db = &toep->ddp.db[db_idx]; job = db->job; copied = job->aio_received; placed = n; if (placed > job->uaiocb.aio_nbytes - copied) placed = job->uaiocb.aio_nbytes - copied; if (placed > 0) job->msgrcv = 1; if (!aio_clear_cancel_function(job)) { /* * Update the copied length for when * t4_aio_cancel_active() completes this * request. */ job->aio_received += placed; } else if (copied + placed != 0) { CTR4(KTR_CXGBE, "%s: completing %p (copied %ld, placed %lu)", __func__, job, copied, placed); /* XXX: This always completes if there is some data. */ aio_complete(job, copied + placed, 0); } else if (aio_set_cancel_function(job, t4_aio_cancel_queued)) { TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count++; } else aio_cancel(job); n -= placed; complete_ddp_buffer(toep, db, db_idx); } MPASS(n == 0); } /* SET_TCB_FIELD sent as a ULP command looks like this */ #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) /* RX_DATA_ACK sent as a ULP command looks like this */ #define LEN__RX_DATA_ACK_ULP (sizeof(struct ulp_txpkt) + \ sizeof(struct ulptx_idata) + sizeof(struct cpl_rx_data_ack_core)) static inline void * mk_set_tcb_field_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep, uint64_t word, uint64_t mask, uint64_t val) { struct ulptx_idata *ulpsc; struct cpl_set_tcb_field_core *req; ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); ulpmc->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(sizeof(*req)); req = (struct cpl_set_tcb_field_core *)(ulpsc + 1); OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tid)); req->reply_ctrl = htobe16(V_NO_REPLY(1) | V_QUEUENO(toep->ofld_rxq->iq.abs_id)); req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0)); req->mask = htobe64(mask); req->val = htobe64(val); ulpsc = (struct ulptx_idata *)(req + 1); if (LEN__SET_TCB_FIELD_ULP % 16) { ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); ulpsc->len = htobe32(0); return (ulpsc + 1); } return (ulpsc); } static inline void * mk_rx_data_ack_ulp(struct ulp_txpkt *ulpmc, struct toepcb *toep) { struct ulptx_idata *ulpsc; struct cpl_rx_data_ack_core *req; ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); ulpmc->len = htobe32(howmany(LEN__RX_DATA_ACK_ULP, 16)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(sizeof(*req)); req = (struct cpl_rx_data_ack_core *)(ulpsc + 1); OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tid)); req->credit_dack = htobe32(F_RX_MODULATE_RX); ulpsc = (struct ulptx_idata *)(req + 1); if (LEN__RX_DATA_ACK_ULP % 16) { ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); ulpsc->len = htobe32(0); return (ulpsc + 1); } return (ulpsc); } static struct wrqe * mk_update_tcb_for_ddp(struct adapter *sc, struct toepcb *toep, int db_idx, struct pageset *ps, int offset, uint64_t ddp_flags, uint64_t ddp_flags_mask) { struct wrqe *wr; struct work_request_hdr *wrh; struct ulp_txpkt *ulpmc; int len; KASSERT(db_idx == 0 || db_idx == 1, ("%s: bad DDP buffer index %d", __func__, db_idx)); /* * We'll send a compound work request that has 3 SET_TCB_FIELDs and an * RX_DATA_ACK (with RX_MODULATE to speed up delivery). * * The work request header is 16B and always ends at a 16B boundary. * The ULPTX master commands that follow must all end at 16B boundaries * too so we round up the size to 16. */ len = sizeof(*wrh) + 3 * roundup2(LEN__SET_TCB_FIELD_ULP, 16) + roundup2(LEN__RX_DATA_ACK_ULP, 16); wr = alloc_wrqe(len, toep->ctrlq); if (wr == NULL) return (NULL); wrh = wrtod(wr); INIT_ULPTX_WRH(wrh, len, 1, 0); /* atomic */ ulpmc = (struct ulp_txpkt *)(wrh + 1); /* Write the buffer's tag */ ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_BUF0_TAG + db_idx, V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG), V_TCB_RX_DDP_BUF0_TAG(ps->prsv.prsv_tag)); /* Update the current offset in the DDP buffer and its total length */ if (db_idx == 0) ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_BUF0_OFFSET, V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) | V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), V_TCB_RX_DDP_BUF0_OFFSET(offset) | V_TCB_RX_DDP_BUF0_LEN(ps->len)); else ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_BUF1_OFFSET, V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) | V_TCB_RX_DDP_BUF1_LEN((u64)M_TCB_RX_DDP_BUF1_LEN << 32), V_TCB_RX_DDP_BUF1_OFFSET(offset) | V_TCB_RX_DDP_BUF1_LEN((u64)ps->len << 32)); /* Update DDP flags */ ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_RX_DDP_FLAGS, ddp_flags_mask, ddp_flags); /* Gratuitous RX_DATA_ACK with RX_MODULATE set to speed up delivery. */ ulpmc = mk_rx_data_ack_ulp(ulpmc, toep); return (wr); } static int handle_ddp_data(struct toepcb *toep, __be32 ddp_report, __be32 rcv_nxt, int len) { uint32_t report = be32toh(ddp_report); unsigned int db_idx; struct inpcb *inp = toep->inp; struct ddp_buffer *db; struct tcpcb *tp; struct socket *so; struct sockbuf *sb; struct kaiocb *job; long copied; db_idx = report & F_DDP_BUF_IDX ? 1 : 0; if (__predict_false(!(report & F_DDP_INV))) CXGBE_UNIMPLEMENTED("DDP buffer still valid"); INP_WLOCK(inp); so = inp_inpcbtosocket(inp); sb = &so->so_rcv; DDP_LOCK(toep); KASSERT(toep->ddp.active_id == db_idx, ("completed DDP buffer (%d) != active_id (%d) for tid %d", db_idx, toep->ddp.active_id, toep->tid)); db = &toep->ddp.db[db_idx]; job = db->job; if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) { /* * This can happen due to an administrative tcpdrop(8). * Just fail the request with ECONNRESET. */ CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x", __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags); if (aio_clear_cancel_function(job)) ddp_complete_one(job, ECONNRESET); goto completed; } tp = intotcpcb(inp); /* * For RX_DDP_COMPLETE, len will be zero and rcv_nxt is the * sequence number of the next byte to receive. The length of * the data received for this message must be computed by * comparing the new and old values of rcv_nxt. * * For RX_DATA_DDP, len might be non-zero, but it is only the * length of the most recent DMA. It does not include the * total length of the data received since the previous update * for this DDP buffer. rcv_nxt is the sequence number of the * first received byte from the most recent DMA. */ len += be32toh(rcv_nxt) - tp->rcv_nxt; tp->rcv_nxt += len; tp->t_rcvtime = ticks; #ifndef USE_DDP_RX_FLOW_CONTROL KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); tp->rcv_wnd -= len; #endif #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %u, DDP[%d] placed %d bytes (%#x)", __func__, toep->tid, db_idx, len, report); #endif /* receive buffer autosize */ MPASS(toep->vnet == so->so_vnet); CURVNET_SET(toep->vnet); SOCKBUF_LOCK(sb); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autorcvbuf && sb->sb_hiwat < V_tcp_autorcvbuf_max && len > (sbspace(sb) / 8 * 7)) { struct adapter *sc = td_adapter(toep->td); unsigned int hiwat = sb->sb_hiwat; unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc, V_tcp_autorcvbuf_max); if (!sbreserve_locked(sb, newsize, so, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; } SOCKBUF_UNLOCK(sb); CURVNET_RESTORE(); job->msgrcv = 1; if (db->cancel_pending) { /* * Update the job's length but defer completion to the * TCB_RPL callback. */ job->aio_received += len; goto out; } else if (!aio_clear_cancel_function(job)) { /* * Update the copied length for when * t4_aio_cancel_active() completes this request. */ job->aio_received += len; } else { copied = job->aio_received; #ifdef VERBOSE_TRACES CTR5(KTR_CXGBE, "%s: tid %u, completing %p (copied %ld, placed %d)", __func__, toep->tid, job, copied, len); #endif aio_complete(job, copied + len, 0); t4_rcvd(&toep->td->tod, tp); } completed: complete_ddp_buffer(toep, db, db_idx); if (toep->ddp.waiting_count > 0) ddp_queue_toep(toep); out: DDP_UNLOCK(toep); INP_WUNLOCK(inp); return (0); } void handle_ddp_indicate(struct toepcb *toep) { DDP_ASSERT_LOCKED(toep); MPASS(toep->ddp.active_count == 0); MPASS((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0); if (toep->ddp.waiting_count == 0) { /* * The pending requests that triggered the request for an * an indicate were cancelled. Those cancels should have * already disabled DDP. Just ignore this as the data is * going into the socket buffer anyway. */ return; } CTR3(KTR_CXGBE, "%s: tid %d indicated (%d waiting)", __func__, toep->tid, toep->ddp.waiting_count); ddp_queue_toep(toep); } CTASSERT(CPL_COOKIE_DDP0 + 1 == CPL_COOKIE_DDP1); static int do_ddp_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); unsigned int db_idx; struct toepcb *toep; struct inpcb *inp; struct ddp_buffer *db; struct kaiocb *job; long copied; if (cpl->status != CPL_ERR_NONE) panic("XXX: tcp_rpl failed: %d", cpl->status); toep = lookup_tid(sc, tid); inp = toep->inp; switch (cpl->cookie) { case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(CPL_COOKIE_DDP0): case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(CPL_COOKIE_DDP1): /* * XXX: This duplicates a lot of code with handle_ddp_data(). */ db_idx = G_COOKIE(cpl->cookie) - CPL_COOKIE_DDP0; MPASS(db_idx < nitems(toep->ddp.db)); INP_WLOCK(inp); DDP_LOCK(toep); db = &toep->ddp.db[db_idx]; /* * handle_ddp_data() should leave the job around until * this callback runs once a cancel is pending. */ MPASS(db != NULL); MPASS(db->job != NULL); MPASS(db->cancel_pending); /* * XXX: It's not clear what happens if there is data * placed when the buffer is invalidated. I suspect we * need to read the TCB to see how much data was placed. * * For now this just pretends like nothing was placed. * * XXX: Note that if we did check the PCB we would need to * also take care of updating the tp, etc. */ job = db->job; copied = job->aio_received; if (copied == 0) { CTR2(KTR_CXGBE, "%s: cancelling %p", __func__, job); aio_cancel(job); } else { CTR3(KTR_CXGBE, "%s: completing %p (copied %ld)", __func__, job, copied); aio_complete(job, copied, 0); t4_rcvd(&toep->td->tod, intotcpcb(inp)); } complete_ddp_buffer(toep, db, db_idx); if (toep->ddp.waiting_count > 0) ddp_queue_toep(toep); DDP_UNLOCK(toep); INP_WUNLOCK(inp); break; default: panic("XXX: unknown tcb_rpl offset %#x, cookie %#x", G_WORD(cpl->cookie), G_COOKIE(cpl->cookie)); } return (0); } void handle_ddp_close(struct toepcb *toep, struct tcpcb *tp, __be32 rcv_nxt) { struct ddp_buffer *db; struct kaiocb *job; long copied; unsigned int db_flag, db_idx; int len, placed; INP_WLOCK_ASSERT(toep->inp); DDP_ASSERT_LOCKED(toep); len = be32toh(rcv_nxt) - tp->rcv_nxt; tp->rcv_nxt += len; while (toep->ddp.active_count > 0) { MPASS(toep->ddp.active_id != -1); db_idx = toep->ddp.active_id; db_flag = db_idx == 1 ? DDP_BUF1_ACTIVE : DDP_BUF0_ACTIVE; MPASS((toep->ddp.flags & db_flag) != 0); db = &toep->ddp.db[db_idx]; job = db->job; copied = job->aio_received; placed = len; if (placed > job->uaiocb.aio_nbytes - copied) placed = job->uaiocb.aio_nbytes - copied; if (placed > 0) job->msgrcv = 1; if (!aio_clear_cancel_function(job)) { /* * Update the copied length for when * t4_aio_cancel_active() completes this * request. */ job->aio_received += placed; } else { CTR4(KTR_CXGBE, "%s: tid %d completed buf %d len %d", __func__, toep->tid, db_idx, placed); aio_complete(job, copied + placed, 0); } len -= placed; complete_ddp_buffer(toep, db, db_idx); } MPASS(len == 0); ddp_complete_all(toep, 0); } #define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\ F_DDP_PPOD_PARITY_ERR | F_DDP_PADDING_ERR | F_DDP_OFFSET_ERR |\ F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\ F_DDP_INVALID_PPOD | F_DDP_HDRCRC_ERR | F_DDP_DATACRC_ERR) extern cpl_handler_t t4_cpl_handler[]; static int do_rx_data_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); uint32_t vld; struct toepcb *toep = lookup_tid(sc, tid); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); KASSERT(!(toep->flags & TPF_SYNQE), ("%s: toep %p claims to be a synq entry", __func__, toep)); vld = be32toh(cpl->ddpvld); if (__predict_false(vld & DDP_ERR)) { panic("%s: DDP error 0x%x (tid %d, toep %p)", __func__, vld, tid, toep); } if (ulp_mode(toep) == ULP_MODE_ISCSI) { t4_cpl_handler[CPL_RX_ISCSI_DDP](iq, rss, m); return (0); } handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len)); return (0); } static int do_rx_ddp_complete(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { struct adapter *sc = iq->adapter; const struct cpl_rx_ddp_complete *cpl = (const void *)(rss + 1); unsigned int tid = GET_TID(cpl); struct toepcb *toep = lookup_tid(sc, tid); KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); KASSERT(!(toep->flags & TPF_SYNQE), ("%s: toep %p claims to be a synq entry", __func__, toep)); handle_ddp_data(toep, cpl->ddp_report, cpl->rcv_nxt, 0); return (0); } static void enable_ddp(struct adapter *sc, struct toepcb *toep) { KASSERT((toep->ddp.flags & (DDP_ON | DDP_OK | DDP_SC_REQ)) == DDP_OK, ("%s: toep %p has bad ddp_flags 0x%x", __func__, toep, toep->ddp.flags)); CTR3(KTR_CXGBE, "%s: tid %u (time %u)", __func__, toep->tid, time_uptime); DDP_ASSERT_LOCKED(toep); toep->ddp.flags |= DDP_SC_REQ; t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1) | V_TF_DDP_INDICATE_OUT(1) | V_TF_DDP_BUF0_INDICATE(1) | V_TF_DDP_BUF1_INDICATE(1) | V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_BUF1_VALID(1), V_TF_DDP_BUF0_INDICATE(1) | V_TF_DDP_BUF1_INDICATE(1), 0, 0); t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, V_TF_RCV_COALESCE_ENABLE(1), 0, 0, 0); } static int calculate_hcf(int n1, int n2) { int a, b, t; if (n1 <= n2) { a = n1; b = n2; } else { a = n2; b = n1; } while (a != 0) { t = a; a = b % a; b = t; } return (b); } static inline int pages_to_nppods(int npages, int ddp_page_shift) { MPASS(ddp_page_shift >= PAGE_SHIFT); return (howmany(npages >> (ddp_page_shift - PAGE_SHIFT), PPOD_PAGES)); } static int alloc_page_pods(struct ppod_region *pr, u_int nppods, u_int pgsz_idx, struct ppod_reservation *prsv) { vmem_addr_t addr; /* relative to start of region */ if (vmem_alloc(pr->pr_arena, PPOD_SZ(nppods), M_NOWAIT | M_FIRSTFIT, &addr) != 0) return (ENOMEM); CTR5(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d, pgsz %d", __func__, pr->pr_arena, (uint32_t)addr & pr->pr_tag_mask, nppods, 1 << pr->pr_page_shift[pgsz_idx]); /* * The hardware tagmask includes an extra invalid bit but the arena was * seeded with valid values only. An allocation out of this arena will * fit inside the tagmask but won't have the invalid bit set. */ MPASS((addr & pr->pr_tag_mask) == addr); MPASS((addr & pr->pr_invalid_bit) == 0); prsv->prsv_pr = pr; prsv->prsv_tag = V_PPOD_PGSZ(pgsz_idx) | addr; prsv->prsv_nppods = nppods; return (0); } int t4_alloc_page_pods_for_ps(struct ppod_region *pr, struct pageset *ps) { int i, hcf, seglen, idx, nppods; struct ppod_reservation *prsv = &ps->prsv; KASSERT(prsv->prsv_nppods == 0, ("%s: page pods already allocated", __func__)); /* * The DDP page size is unrelated to the VM page size. We combine * contiguous physical pages into larger segments to get the best DDP * page size possible. This is the largest of the four sizes in * A_ULP_RX_TDDP_PSZ that evenly divides the HCF of the segment sizes in * the page list. */ hcf = 0; for (i = 0; i < ps->npages; i++) { seglen = PAGE_SIZE; while (i < ps->npages - 1 && ps->pages[i]->phys_addr + PAGE_SIZE == ps->pages[i + 1]->phys_addr) { seglen += PAGE_SIZE; i++; } hcf = calculate_hcf(hcf, seglen); if (hcf < (1 << pr->pr_page_shift[1])) { idx = 0; goto have_pgsz; /* give up, short circuit */ } } #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1) MPASS((hcf & PR_PAGE_MASK(0)) == 0); /* PAGE_SIZE is >= 4K everywhere */ for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) { if ((hcf & PR_PAGE_MASK(idx)) == 0) break; } #undef PR_PAGE_MASK have_pgsz: MPASS(idx <= M_PPOD_PGSZ); nppods = pages_to_nppods(ps->npages, pr->pr_page_shift[idx]); if (alloc_page_pods(pr, nppods, idx, prsv) != 0) return (0); MPASS(prsv->prsv_nppods > 0); return (1); } int t4_alloc_page_pods_for_buf(struct ppod_region *pr, vm_offset_t buf, int len, struct ppod_reservation *prsv) { int hcf, seglen, idx, npages, nppods; uintptr_t start_pva, end_pva, pva, p1; MPASS(buf > 0); MPASS(len > 0); /* * The DDP page size is unrelated to the VM page size. We combine * contiguous physical pages into larger segments to get the best DDP * page size possible. This is the largest of the four sizes in * A_ULP_RX_ISCSI_PSZ that evenly divides the HCF of the segment sizes * in the page list. */ hcf = 0; start_pva = trunc_page(buf); end_pva = trunc_page(buf + len - 1); pva = start_pva; while (pva <= end_pva) { seglen = PAGE_SIZE; p1 = pmap_kextract(pva); pva += PAGE_SIZE; while (pva <= end_pva && p1 + seglen == pmap_kextract(pva)) { seglen += PAGE_SIZE; pva += PAGE_SIZE; } hcf = calculate_hcf(hcf, seglen); if (hcf < (1 << pr->pr_page_shift[1])) { idx = 0; goto have_pgsz; /* give up, short circuit */ } } #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1) MPASS((hcf & PR_PAGE_MASK(0)) == 0); /* PAGE_SIZE is >= 4K everywhere */ for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) { if ((hcf & PR_PAGE_MASK(idx)) == 0) break; } #undef PR_PAGE_MASK have_pgsz: MPASS(idx <= M_PPOD_PGSZ); npages = 1; npages += (end_pva - start_pva) >> pr->pr_page_shift[idx]; nppods = howmany(npages, PPOD_PAGES); if (alloc_page_pods(pr, nppods, idx, prsv) != 0) return (ENOMEM); MPASS(prsv->prsv_nppods > 0); return (0); } int t4_alloc_page_pods_for_sgl(struct ppod_region *pr, struct ctl_sg_entry *sgl, int entries, struct ppod_reservation *prsv) { int hcf, seglen, idx = 0, npages, nppods, i, len; uintptr_t start_pva, end_pva, pva, p1 ; vm_offset_t buf; struct ctl_sg_entry *sge; MPASS(entries > 0); MPASS(sgl); /* * The DDP page size is unrelated to the VM page size. We combine * contiguous physical pages into larger segments to get the best DDP * page size possible. This is the largest of the four sizes in * A_ULP_RX_ISCSI_PSZ that evenly divides the HCF of the segment sizes * in the page list. */ hcf = 0; for (i = entries - 1; i >= 0; i--) { sge = sgl + i; buf = (vm_offset_t)sge->addr; len = sge->len; start_pva = trunc_page(buf); end_pva = trunc_page(buf + len - 1); pva = start_pva; while (pva <= end_pva) { seglen = PAGE_SIZE; p1 = pmap_kextract(pva); pva += PAGE_SIZE; while (pva <= end_pva && p1 + seglen == pmap_kextract(pva)) { seglen += PAGE_SIZE; pva += PAGE_SIZE; } hcf = calculate_hcf(hcf, seglen); if (hcf < (1 << pr->pr_page_shift[1])) { idx = 0; goto have_pgsz; /* give up, short circuit */ } } } #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1) MPASS((hcf & PR_PAGE_MASK(0)) == 0); /* PAGE_SIZE is >= 4K everywhere */ for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) { if ((hcf & PR_PAGE_MASK(idx)) == 0) break; } #undef PR_PAGE_MASK have_pgsz: MPASS(idx <= M_PPOD_PGSZ); npages = 0; while (entries--) { npages++; start_pva = trunc_page((vm_offset_t)sgl->addr); end_pva = trunc_page((vm_offset_t)sgl->addr + sgl->len - 1); npages += (end_pva - start_pva) >> pr->pr_page_shift[idx]; sgl = sgl + 1; } nppods = howmany(npages, PPOD_PAGES); if (alloc_page_pods(pr, nppods, idx, prsv) != 0) return (ENOMEM); MPASS(prsv->prsv_nppods > 0); return (0); } void t4_free_page_pods(struct ppod_reservation *prsv) { struct ppod_region *pr = prsv->prsv_pr; vmem_addr_t addr; MPASS(prsv != NULL); MPASS(prsv->prsv_nppods != 0); addr = prsv->prsv_tag & pr->pr_tag_mask; MPASS((addr & pr->pr_invalid_bit) == 0); CTR4(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d", __func__, pr->pr_arena, addr, prsv->prsv_nppods); vmem_free(pr->pr_arena, addr, PPOD_SZ(prsv->prsv_nppods)); prsv->prsv_nppods = 0; } #define NUM_ULP_TX_SC_IMM_PPODS (256 / PPOD_SIZE) int t4_write_page_pods_for_ps(struct adapter *sc, struct sge_wrq *wrq, int tid, struct pageset *ps) { struct wrqe *wr; struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; struct pagepod *ppod; int i, j, k, n, chunk, len, ddp_pgsz, idx; u_int ppod_addr; uint32_t cmd; struct ppod_reservation *prsv = &ps->prsv; struct ppod_region *pr = prsv->prsv_pr; KASSERT(!(ps->flags & PS_PPODS_WRITTEN), ("%s: page pods already written", __func__)); MPASS(prsv->prsv_nppods > 0); cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); if (is_t4(sc)) cmd |= htobe32(F_ULP_MEMIO_ORDER); else cmd |= htobe32(F_T5_ULP_MEMIO_IMM); ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { /* How many page pods are we writing in this cycle */ n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); chunk = PPOD_SZ(n); len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); wr = alloc_wrqe(len, wrq); if (wr == NULL) return (ENOMEM); /* ok to just bail out */ ulpmc = wrtod(wr); INIT_ULPTX_WR(ulpmc, len, 0, 0); ulpmc->cmd = cmd; ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(chunk); ppod = (struct pagepod *)(ulpsc + 1); for (j = 0; j < n; i++, j++, ppod++) { ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | V_PPOD_TID(tid) | prsv->prsv_tag); ppod->len_offset = htobe64(V_PPOD_LEN(ps->len) | V_PPOD_OFST(ps->offset)); ppod->rsvd = 0; idx = i * PPOD_PAGES * (ddp_pgsz / PAGE_SIZE); for (k = 0; k < nitems(ppod->addr); k++) { if (idx < ps->npages) { ppod->addr[k] = htobe64(ps->pages[idx]->phys_addr); idx += ddp_pgsz / PAGE_SIZE; } else ppod->addr[k] = 0; #if 0 CTR5(KTR_CXGBE, "%s: tid %d ppod[%d]->addr[%d] = %p", __func__, toep->tid, i, k, htobe64(ppod->addr[k])); #endif } } t4_wrq_tx(sc, wr); } ps->flags |= PS_PPODS_WRITTEN; return (0); } static struct mbuf * alloc_raw_wr_mbuf(int len) { struct mbuf *m; if (len <= MHLEN) m = m_gethdr(M_NOWAIT, MT_DATA); else if (len <= MCLBYTES) m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); else m = NULL; if (m == NULL) return (NULL); m->m_pkthdr.len = len; m->m_len = len; set_mbuf_raw_wr(m, true); return (m); } int t4_write_page_pods_for_buf(struct adapter *sc, struct toepcb *toep, - struct ppod_reservation *prsv, vm_offset_t buf, int buflen) + struct ppod_reservation *prsv, vm_offset_t buf, int buflen, + struct mbufq *wrq) { - struct inpcb *inp = toep->inp; struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; struct pagepod *ppod; int i, j, k, n, chunk, len, ddp_pgsz; u_int ppod_addr, offset; uint32_t cmd; struct ppod_region *pr = prsv->prsv_pr; uintptr_t end_pva, pva, pa; struct mbuf *m; - struct mbufq wrq; cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); if (is_t4(sc)) cmd |= htobe32(F_ULP_MEMIO_ORDER); else cmd |= htobe32(F_T5_ULP_MEMIO_IMM); ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; offset = buf & PAGE_MASK; ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); pva = trunc_page(buf); end_pva = trunc_page(buf + buflen - 1); - mbufq_init(&wrq, INT_MAX); for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { /* How many page pods are we writing in this cycle */ n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); MPASS(n > 0); chunk = PPOD_SZ(n); len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); m = alloc_raw_wr_mbuf(len); - if (m == NULL) { - mbufq_drain(&wrq); + if (m == NULL) return (ENOMEM); - } ulpmc = mtod(m, struct ulp_mem_io *); INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); ulpmc->cmd = cmd; ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(chunk); ppod = (struct pagepod *)(ulpsc + 1); for (j = 0; j < n; i++, j++, ppod++) { ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | V_PPOD_TID(toep->tid) | (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ))); ppod->len_offset = htobe64(V_PPOD_LEN(buflen) | V_PPOD_OFST(offset)); ppod->rsvd = 0; for (k = 0; k < nitems(ppod->addr); k++) { if (pva > end_pva) ppod->addr[k] = 0; else { pa = pmap_kextract(pva); ppod->addr[k] = htobe64(pa); pva += ddp_pgsz; } #if 0 CTR5(KTR_CXGBE, "%s: tid %d ppod[%d]->addr[%d] = %p", __func__, toep->tid, i, k, htobe64(ppod->addr[k])); #endif } /* * Walk back 1 segment so that the first address in the * next pod is the same as the last one in the current * pod. */ pva -= ddp_pgsz; } - mbufq_enqueue(&wrq, m); + mbufq_enqueue(wrq, m); } - INP_WLOCK(inp); - mbufq_concat(&toep->ulp_pduq, &wrq); - INP_WUNLOCK(inp); - MPASS(pva <= end_pva); return (0); } int t4_write_page_pods_for_sgl(struct adapter *sc, struct toepcb *toep, struct ppod_reservation *prsv, struct ctl_sg_entry *sgl, int entries, - int xferlen) + int xferlen, struct mbufq *wrq) { - struct inpcb *inp = toep->inp; struct ulp_mem_io *ulpmc; struct ulptx_idata *ulpsc; struct pagepod *ppod; int i, j, k, n, chunk, len, ddp_pgsz; u_int ppod_addr, offset, sg_offset = 0; uint32_t cmd; struct ppod_region *pr = prsv->prsv_pr; uintptr_t pva, pa; struct mbuf *m; - struct mbufq wrq; MPASS(sgl != NULL); MPASS(entries > 0); cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE)); if (is_t4(sc)) cmd |= htobe32(F_ULP_MEMIO_ORDER); else cmd |= htobe32(F_T5_ULP_MEMIO_IMM); ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)]; offset = (vm_offset_t)sgl->addr & PAGE_MASK; ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask); pva = trunc_page((vm_offset_t)sgl->addr); - mbufq_init(&wrq, INT_MAX); for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) { /* How many page pods are we writing in this cycle */ n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS); MPASS(n > 0); chunk = PPOD_SZ(n); len = roundup2(sizeof(*ulpmc) + sizeof(*ulpsc) + chunk, 16); m = alloc_raw_wr_mbuf(len); - if (m == NULL) { - mbufq_drain(&wrq); + if (m == NULL) return (ENOMEM); - } ulpmc = mtod(m, struct ulp_mem_io *); INIT_ULPTX_WR(ulpmc, len, 0, toep->tid); ulpmc->cmd = cmd; ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32)); ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16)); ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5)); ulpsc = (struct ulptx_idata *)(ulpmc + 1); ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); ulpsc->len = htobe32(chunk); ppod = (struct pagepod *)(ulpsc + 1); for (j = 0; j < n; i++, j++, ppod++) { ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID | V_PPOD_TID(toep->tid) | (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ))); ppod->len_offset = htobe64(V_PPOD_LEN(xferlen) | V_PPOD_OFST(offset)); ppod->rsvd = 0; for (k = 0; k < nitems(ppod->addr); k++) { if (entries != 0) { pa = pmap_kextract(pva + sg_offset); ppod->addr[k] = htobe64(pa); } else ppod->addr[k] = 0; #if 0 CTR5(KTR_CXGBE, "%s: tid %d ppod[%d]->addr[%d] = %p", __func__, toep->tid, i, k, htobe64(ppod->addr[k])); #endif /* * If this is the last entry in a pod, * reuse the same entry for first address * in the next pod. */ if (k + 1 == nitems(ppod->addr)) break; /* * Don't move to the next DDP page if the * sgl is already finished. */ if (entries == 0) continue; sg_offset += ddp_pgsz; if (sg_offset == sgl->len) { /* * This sgl entry is done. Go * to the next. */ entries--; sgl++; sg_offset = 0; if (entries != 0) pva = trunc_page( (vm_offset_t)sgl->addr); } } } - mbufq_enqueue(&wrq, m); + mbufq_enqueue(wrq, m); } - INP_WLOCK(inp); - mbufq_concat(&toep->ulp_pduq, &wrq); - INP_WUNLOCK(inp); - return (0); } /* * Prepare a pageset for DDP. This sets up page pods. */ static int prep_pageset(struct adapter *sc, struct toepcb *toep, struct pageset *ps) { struct tom_data *td = sc->tom_softc; if (ps->prsv.prsv_nppods == 0 && !t4_alloc_page_pods_for_ps(&td->pr, ps)) { return (0); } if (!(ps->flags & PS_PPODS_WRITTEN) && t4_write_page_pods_for_ps(sc, toep->ctrlq, toep->tid, ps) != 0) { return (0); } return (1); } int t4_init_ppod_region(struct ppod_region *pr, struct t4_range *r, u_int psz, const char *name) { int i; MPASS(pr != NULL); MPASS(r->size > 0); pr->pr_start = r->start; pr->pr_len = r->size; pr->pr_page_shift[0] = 12 + G_HPZ0(psz); pr->pr_page_shift[1] = 12 + G_HPZ1(psz); pr->pr_page_shift[2] = 12 + G_HPZ2(psz); pr->pr_page_shift[3] = 12 + G_HPZ3(psz); /* The SGL -> page pod algorithm requires the sizes to be in order. */ for (i = 1; i < nitems(pr->pr_page_shift); i++) { if (pr->pr_page_shift[i] <= pr->pr_page_shift[i - 1]) return (ENXIO); } pr->pr_tag_mask = ((1 << fls(r->size)) - 1) & V_PPOD_TAG(M_PPOD_TAG); pr->pr_alias_mask = V_PPOD_TAG(M_PPOD_TAG) & ~pr->pr_tag_mask; if (pr->pr_tag_mask == 0 || pr->pr_alias_mask == 0) return (ENXIO); pr->pr_alias_shift = fls(pr->pr_tag_mask); pr->pr_invalid_bit = 1 << (pr->pr_alias_shift - 1); pr->pr_arena = vmem_create(name, 0, pr->pr_len, PPOD_SIZE, 0, M_FIRSTFIT | M_NOWAIT); if (pr->pr_arena == NULL) return (ENOMEM); return (0); } void t4_free_ppod_region(struct ppod_region *pr) { MPASS(pr != NULL); if (pr->pr_arena) vmem_destroy(pr->pr_arena); bzero(pr, sizeof(*pr)); } static int pscmp(struct pageset *ps, struct vmspace *vm, vm_offset_t start, int npages, int pgoff, int len) { if (ps->start != start || ps->npages != npages || ps->offset != pgoff || ps->len != len) return (1); return (ps->vm != vm || ps->vm_timestamp != vm->vm_map.timestamp); } static int hold_aio(struct toepcb *toep, struct kaiocb *job, struct pageset **pps) { struct vmspace *vm; vm_map_t map; vm_offset_t start, end, pgoff; struct pageset *ps; int n; DDP_ASSERT_LOCKED(toep); /* * The AIO subsystem will cancel and drain all requests before * permitting a process to exit or exec, so p_vmspace should * be stable here. */ vm = job->userproc->p_vmspace; map = &vm->vm_map; start = (uintptr_t)job->uaiocb.aio_buf; pgoff = start & PAGE_MASK; end = round_page(start + job->uaiocb.aio_nbytes); start = trunc_page(start); if (end - start > MAX_DDP_BUFFER_SIZE) { /* * Truncate the request to a short read. * Alternatively, we could DDP in chunks to the larger * buffer, but that would be quite a bit more work. * * When truncating, round the request down to avoid * crossing a cache line on the final transaction. */ end = rounddown2(start + MAX_DDP_BUFFER_SIZE, CACHE_LINE_SIZE); #ifdef VERBOSE_TRACES CTR4(KTR_CXGBE, "%s: tid %d, truncating size from %lu to %lu", __func__, toep->tid, (unsigned long)job->uaiocb.aio_nbytes, (unsigned long)(end - (start + pgoff))); job->uaiocb.aio_nbytes = end - (start + pgoff); #endif end = round_page(end); } n = atop(end - start); /* * Try to reuse a cached pageset. */ TAILQ_FOREACH(ps, &toep->ddp.cached_pagesets, link) { if (pscmp(ps, vm, start, n, pgoff, job->uaiocb.aio_nbytes) == 0) { TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link); toep->ddp.cached_count--; *pps = ps; return (0); } } /* * If there are too many cached pagesets to create a new one, * free a pageset before creating a new one. */ KASSERT(toep->ddp.active_count + toep->ddp.cached_count <= nitems(toep->ddp.db), ("%s: too many wired pagesets", __func__)); if (toep->ddp.active_count + toep->ddp.cached_count == nitems(toep->ddp.db)) { KASSERT(toep->ddp.cached_count > 0, ("no cached pageset to free")); ps = TAILQ_LAST(&toep->ddp.cached_pagesets, pagesetq); TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link); toep->ddp.cached_count--; free_pageset(toep->td, ps); } DDP_UNLOCK(toep); /* Create a new pageset. */ ps = malloc(sizeof(*ps) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK | M_ZERO); ps->pages = (vm_page_t *)(ps + 1); ps->vm_timestamp = map->timestamp; ps->npages = vm_fault_quick_hold_pages(map, start, end - start, VM_PROT_WRITE, ps->pages, n); DDP_LOCK(toep); if (ps->npages < 0) { free(ps, M_CXGBE); return (EFAULT); } KASSERT(ps->npages == n, ("hold_aio: page count mismatch: %d vs %d", ps->npages, n)); ps->offset = pgoff; ps->len = job->uaiocb.aio_nbytes; refcount_acquire(&vm->vm_refcnt); ps->vm = vm; ps->start = start; CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d", __func__, toep->tid, ps, job, ps->npages); *pps = ps; return (0); } static void ddp_complete_all(struct toepcb *toep, int error) { struct kaiocb *job; DDP_ASSERT_LOCKED(toep); while (!TAILQ_EMPTY(&toep->ddp.aiojobq)) { job = TAILQ_FIRST(&toep->ddp.aiojobq); TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count--; if (aio_clear_cancel_function(job)) ddp_complete_one(job, error); } } static void aio_ddp_cancel_one(struct kaiocb *job) { long copied; /* * If this job had copied data out of the socket buffer before * it was cancelled, report it as a short read rather than an * error. */ copied = job->aio_received; if (copied != 0) aio_complete(job, copied, 0); else aio_cancel(job); } /* * Called when the main loop wants to requeue a job to retry it later. * Deals with the race of the job being cancelled while it was being * examined. */ static void aio_ddp_requeue_one(struct toepcb *toep, struct kaiocb *job) { DDP_ASSERT_LOCKED(toep); if (!(toep->ddp.flags & DDP_DEAD) && aio_set_cancel_function(job, t4_aio_cancel_queued)) { TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count++; } else aio_ddp_cancel_one(job); } static void aio_ddp_requeue(struct toepcb *toep) { struct adapter *sc = td_adapter(toep->td); struct socket *so; struct sockbuf *sb; struct inpcb *inp; struct kaiocb *job; struct ddp_buffer *db; size_t copied, offset, resid; struct pageset *ps; struct mbuf *m; uint64_t ddp_flags, ddp_flags_mask; struct wrqe *wr; int buf_flag, db_idx, error; DDP_ASSERT_LOCKED(toep); restart: if (toep->ddp.flags & DDP_DEAD) { MPASS(toep->ddp.waiting_count == 0); MPASS(toep->ddp.active_count == 0); return; } if (toep->ddp.waiting_count == 0 || toep->ddp.active_count == nitems(toep->ddp.db)) { return; } job = TAILQ_FIRST(&toep->ddp.aiojobq); so = job->fd_file->f_data; sb = &so->so_rcv; SOCKBUF_LOCK(sb); /* We will never get anything unless we are or were connected. */ if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { SOCKBUF_UNLOCK(sb); ddp_complete_all(toep, ENOTCONN); return; } KASSERT(toep->ddp.active_count == 0 || sbavail(sb) == 0, ("%s: pending sockbuf data and DDP is active", __func__)); /* Abort if socket has reported problems. */ /* XXX: Wait for any queued DDP's to finish and/or flush them? */ if (so->so_error && sbavail(sb) == 0) { toep->ddp.waiting_count--; TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); if (!aio_clear_cancel_function(job)) { SOCKBUF_UNLOCK(sb); goto restart; } /* * If this job has previously copied some data, report * a short read and leave the error to be reported by * a future request. */ copied = job->aio_received; if (copied != 0) { SOCKBUF_UNLOCK(sb); aio_complete(job, copied, 0); goto restart; } error = so->so_error; so->so_error = 0; SOCKBUF_UNLOCK(sb); aio_complete(job, -1, error); goto restart; } /* * Door is closed. If there is pending data in the socket buffer, * deliver it. If there are pending DDP requests, wait for those * to complete. Once they have completed, return EOF reads. */ if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) { SOCKBUF_UNLOCK(sb); if (toep->ddp.active_count != 0) return; ddp_complete_all(toep, 0); return; } /* * If DDP is not enabled and there is no pending socket buffer * data, try to enable DDP. */ if (sbavail(sb) == 0 && (toep->ddp.flags & DDP_ON) == 0) { SOCKBUF_UNLOCK(sb); /* * Wait for the card to ACK that DDP is enabled before * queueing any buffers. Currently this waits for an * indicate to arrive. This could use a TCB_SET_FIELD_RPL * message to know that DDP was enabled instead of waiting * for the indicate which would avoid copying the indicate * if no data is pending. * * XXX: Might want to limit the indicate size to the size * of the first queued request. */ if ((toep->ddp.flags & DDP_SC_REQ) == 0) enable_ddp(sc, toep); return; } SOCKBUF_UNLOCK(sb); /* * If another thread is queueing a buffer for DDP, let it * drain any work and return. */ if (toep->ddp.queueing != NULL) return; /* Take the next job to prep it for DDP. */ toep->ddp.waiting_count--; TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); if (!aio_clear_cancel_function(job)) goto restart; toep->ddp.queueing = job; /* NB: This drops DDP_LOCK while it holds the backing VM pages. */ error = hold_aio(toep, job, &ps); if (error != 0) { ddp_complete_one(job, error); toep->ddp.queueing = NULL; goto restart; } SOCKBUF_LOCK(sb); if (so->so_error && sbavail(sb) == 0) { copied = job->aio_received; if (copied != 0) { SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_complete(job, copied, 0); toep->ddp.queueing = NULL; goto restart; } error = so->so_error; so->so_error = 0; SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_complete(job, -1, error); toep->ddp.queueing = NULL; goto restart; } if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) { SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); if (toep->ddp.active_count != 0) { /* * The door is closed, but there are still pending * DDP buffers. Requeue. These jobs will all be * completed once those buffers drain. */ aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; return; } ddp_complete_one(job, 0); ddp_complete_all(toep, 0); toep->ddp.queueing = NULL; return; } sbcopy: /* * If the toep is dead, there shouldn't be any data in the socket * buffer, so the above case should have handled this. */ MPASS(!(toep->ddp.flags & DDP_DEAD)); /* * If there is pending data in the socket buffer (either * from before the requests were queued or a DDP indicate), * copy those mbufs out directly. */ copied = 0; offset = ps->offset + job->aio_received; MPASS(job->aio_received <= job->uaiocb.aio_nbytes); resid = job->uaiocb.aio_nbytes - job->aio_received; m = sb->sb_mb; KASSERT(m == NULL || toep->ddp.active_count == 0, ("%s: sockbuf data with active DDP", __func__)); while (m != NULL && resid > 0) { struct iovec iov[1]; struct uio uio; int error; iov[0].iov_base = mtod(m, void *); iov[0].iov_len = m->m_len; if (iov[0].iov_len > resid) iov[0].iov_len = resid; uio.uio_iov = iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = iov[0].iov_len; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; error = uiomove_fromphys(ps->pages, offset + copied, uio.uio_resid, &uio); MPASS(error == 0 && uio.uio_resid == 0); copied += uio.uio_offset; resid -= uio.uio_offset; m = m->m_next; } if (copied != 0) { sbdrop_locked(sb, copied); job->aio_received += copied; job->msgrcv = 1; copied = job->aio_received; inp = sotoinpcb(so); if (!INP_TRY_WLOCK(inp)) { /* * The reference on the socket file descriptor in * the AIO job should keep 'sb' and 'inp' stable. * Our caller has a reference on the 'toep' that * keeps it stable. */ SOCKBUF_UNLOCK(sb); DDP_UNLOCK(toep); INP_WLOCK(inp); DDP_LOCK(toep); SOCKBUF_LOCK(sb); /* * If the socket has been closed, we should detect * that and complete this request if needed on * the next trip around the loop. */ } t4_rcvd_locked(&toep->td->tod, intotcpcb(inp)); INP_WUNLOCK(inp); if (resid == 0 || toep->ddp.flags & DDP_DEAD) { /* * We filled the entire buffer with socket * data, DDP is not being used, or the socket * is being shut down, so complete the * request. */ SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_complete(job, copied, 0); toep->ddp.queueing = NULL; goto restart; } /* * If DDP is not enabled, requeue this request and restart. * This will either enable DDP or wait for more data to * arrive on the socket buffer. */ if ((toep->ddp.flags & (DDP_ON | DDP_SC_REQ)) != DDP_ON) { SOCKBUF_UNLOCK(sb); recycle_pageset(toep, ps); aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; goto restart; } /* * An indicate might have arrived and been added to * the socket buffer while it was unlocked after the * copy to lock the INP. If so, restart the copy. */ if (sbavail(sb) != 0) goto sbcopy; } SOCKBUF_UNLOCK(sb); if (prep_pageset(sc, toep, ps) == 0) { recycle_pageset(toep, ps); aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; /* * XXX: Need to retry this later. Mostly need a trigger * when page pods are freed up. */ printf("%s: prep_pageset failed\n", __func__); return; } /* Determine which DDP buffer to use. */ if (toep->ddp.db[0].job == NULL) { db_idx = 0; } else { MPASS(toep->ddp.db[1].job == NULL); db_idx = 1; } ddp_flags = 0; ddp_flags_mask = 0; if (db_idx == 0) { ddp_flags |= V_TF_DDP_BUF0_VALID(1); if (so->so_state & SS_NBIO) ddp_flags |= V_TF_DDP_BUF0_FLUSH(1); ddp_flags_mask |= V_TF_DDP_PSH_NO_INVALIDATE0(1) | V_TF_DDP_PUSH_DISABLE_0(1) | V_TF_DDP_PSHF_ENABLE_0(1) | V_TF_DDP_BUF0_FLUSH(1) | V_TF_DDP_BUF0_VALID(1); buf_flag = DDP_BUF0_ACTIVE; } else { ddp_flags |= V_TF_DDP_BUF1_VALID(1); if (so->so_state & SS_NBIO) ddp_flags |= V_TF_DDP_BUF1_FLUSH(1); ddp_flags_mask |= V_TF_DDP_PSH_NO_INVALIDATE1(1) | V_TF_DDP_PUSH_DISABLE_1(1) | V_TF_DDP_PSHF_ENABLE_1(1) | V_TF_DDP_BUF1_FLUSH(1) | V_TF_DDP_BUF1_VALID(1); buf_flag = DDP_BUF1_ACTIVE; } MPASS((toep->ddp.flags & buf_flag) == 0); if ((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0) { MPASS(db_idx == 0); MPASS(toep->ddp.active_id == -1); MPASS(toep->ddp.active_count == 0); ddp_flags_mask |= V_TF_DDP_ACTIVE_BUF(1); } /* * The TID for this connection should still be valid. If DDP_DEAD * is set, SBS_CANTRCVMORE should be set, so we shouldn't be * this far anyway. Even if the socket is closing on the other * end, the AIO job holds a reference on this end of the socket * which will keep it open and keep the TCP PCB attached until * after the job is completed. */ wr = mk_update_tcb_for_ddp(sc, toep, db_idx, ps, job->aio_received, ddp_flags, ddp_flags_mask); if (wr == NULL) { recycle_pageset(toep, ps); aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; /* * XXX: Need a way to kick a retry here. * * XXX: We know the fixed size needed and could * preallocate this using a blocking request at the * start of the task to avoid having to handle this * edge case. */ printf("%s: mk_update_tcb_for_ddp failed\n", __func__); return; } if (!aio_set_cancel_function(job, t4_aio_cancel_active)) { free_wrqe(wr); recycle_pageset(toep, ps); aio_ddp_cancel_one(job); toep->ddp.queueing = NULL; goto restart; } #ifdef VERBOSE_TRACES CTR6(KTR_CXGBE, "%s: tid %u, scheduling %p for DDP[%d] (flags %#lx/%#lx)", __func__, toep->tid, job, db_idx, ddp_flags, ddp_flags_mask); #endif /* Give the chip the go-ahead. */ t4_wrq_tx(sc, wr); db = &toep->ddp.db[db_idx]; db->cancel_pending = 0; db->job = job; db->ps = ps; toep->ddp.queueing = NULL; toep->ddp.flags |= buf_flag; toep->ddp.active_count++; if (toep->ddp.active_count == 1) { MPASS(toep->ddp.active_id == -1); toep->ddp.active_id = db_idx; CTR2(KTR_CXGBE, "%s: ddp_active_id = %d", __func__, toep->ddp.active_id); } goto restart; } void ddp_queue_toep(struct toepcb *toep) { DDP_ASSERT_LOCKED(toep); if (toep->ddp.flags & DDP_TASK_ACTIVE) return; toep->ddp.flags |= DDP_TASK_ACTIVE; hold_toepcb(toep); soaio_enqueue(&toep->ddp.requeue_task); } static void aio_ddp_requeue_task(void *context, int pending) { struct toepcb *toep = context; DDP_LOCK(toep); aio_ddp_requeue(toep); toep->ddp.flags &= ~DDP_TASK_ACTIVE; DDP_UNLOCK(toep); free_toepcb(toep); } static void t4_aio_cancel_active(struct kaiocb *job) { struct socket *so = job->fd_file->f_data; struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; struct adapter *sc = td_adapter(toep->td); uint64_t valid_flag; int i; DDP_LOCK(toep); if (aio_cancel_cleared(job)) { DDP_UNLOCK(toep); aio_ddp_cancel_one(job); return; } for (i = 0; i < nitems(toep->ddp.db); i++) { if (toep->ddp.db[i].job == job) { /* Should only ever get one cancel request for a job. */ MPASS(toep->ddp.db[i].cancel_pending == 0); /* * Invalidate this buffer. It will be * cancelled or partially completed once the * card ACKs the invalidate. */ valid_flag = i == 0 ? V_TF_DDP_BUF0_VALID(1) : V_TF_DDP_BUF1_VALID(1); t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_RX_DDP_FLAGS, valid_flag, 0, 1, CPL_COOKIE_DDP0 + i); toep->ddp.db[i].cancel_pending = 1; CTR2(KTR_CXGBE, "%s: request %p marked pending", __func__, job); break; } } DDP_UNLOCK(toep); } static void t4_aio_cancel_queued(struct kaiocb *job) { struct socket *so = job->fd_file->f_data; struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; DDP_LOCK(toep); if (!aio_cancel_cleared(job)) { TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count--; if (toep->ddp.waiting_count == 0) ddp_queue_toep(toep); } CTR2(KTR_CXGBE, "%s: request %p cancelled", __func__, job); DDP_UNLOCK(toep); aio_ddp_cancel_one(job); } int t4_aio_queue_ddp(struct socket *so, struct kaiocb *job) { struct tcpcb *tp = so_sototcpcb(so); struct toepcb *toep = tp->t_toe; /* Ignore writes. */ if (job->uaiocb.aio_lio_opcode != LIO_READ) return (EOPNOTSUPP); DDP_LOCK(toep); /* * XXX: Think about possibly returning errors for ENOTCONN, * etc. Perhaps the caller would only queue the request * if it failed with EOPNOTSUPP? */ #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid); #endif if (!aio_set_cancel_function(job, t4_aio_cancel_queued)) panic("new job was cancelled"); TAILQ_INSERT_TAIL(&toep->ddp.aiojobq, job, list); toep->ddp.waiting_count++; toep->ddp.flags |= DDP_OK; /* * Try to handle this request synchronously. If this has * to block because the task is running, it will just bail * and let the task handle it instead. */ aio_ddp_requeue(toep); DDP_UNLOCK(toep); return (0); } void t4_ddp_mod_load(void) { t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl, CPL_COOKIE_DDP0); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl, CPL_COOKIE_DDP1); t4_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp); t4_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete); TAILQ_INIT(&ddp_orphan_pagesets); mtx_init(&ddp_orphan_pagesets_lock, "ddp orphans", NULL, MTX_DEF); TASK_INIT(&ddp_orphan_task, 0, ddp_free_orphan_pagesets, NULL); } void t4_ddp_mod_unload(void) { taskqueue_drain(taskqueue_thread, &ddp_orphan_task); MPASS(TAILQ_EMPTY(&ddp_orphan_pagesets)); mtx_destroy(&ddp_orphan_pagesets_lock); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, NULL, CPL_COOKIE_DDP0); t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, NULL, CPL_COOKIE_DDP1); t4_register_cpl_handler(CPL_RX_DATA_DDP, NULL); t4_register_cpl_handler(CPL_RX_DDP_COMPLETE, NULL); } #endif diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h index c7984f838735..21cfb1df6e16 100644 --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -1,483 +1,483 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2015 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __T4_TOM_H__ #define __T4_TOM_H__ #include #include "common/t4_hw.h" #include "common/t4_msg.h" #include "tom/t4_tls.h" #define LISTEN_HASH_SIZE 32 /* * Min receive window. We want it to be large enough to accommodate receive * coalescing, handle jumbo frames, and not trigger sender SWS avoidance. */ #define MIN_RCV_WND (24 * 1024U) /* * Max receive window supported by HW in bytes. Only a small part of it can * be set through option0, the rest needs to be set through RX_DATA_ACK. */ #define MAX_RCV_WND ((1U << 27) - 1) #define DDP_RSVD_WIN (16 * 1024U) #define SB_DDP_INDICATE SB_IN_TOE /* soreceive must respond to indicate */ #define USE_DDP_RX_FLOW_CONTROL #define PPOD_SZ(n) ((n) * sizeof(struct pagepod)) #define PPOD_SIZE (PPOD_SZ(1)) /* TOE PCB flags */ enum { TPF_ATTACHED = (1 << 0), /* a tcpcb refers to this toepcb */ TPF_FLOWC_WR_SENT = (1 << 1), /* firmware flow context WR sent */ TPF_TX_DATA_SENT = (1 << 2), /* some data sent */ TPF_TX_SUSPENDED = (1 << 3), /* tx suspended for lack of resources */ TPF_SEND_FIN = (1 << 4), /* send FIN after all pending data */ TPF_FIN_SENT = (1 << 5), /* FIN has been sent */ TPF_ABORT_SHUTDOWN = (1 << 6), /* connection abort is in progress */ TPF_CPL_PENDING = (1 << 7), /* haven't received the last CPL */ TPF_SYNQE = (1 << 8), /* synq_entry, not really a toepcb */ TPF_SYNQE_EXPANDED = (1 << 9), /* toepcb ready, tid context updated */ TPF_FORCE_CREDITS = (1 << 10), /* always send credits */ TPF_KTLS = (1 << 11), /* send TLS records from KTLS */ TPF_INITIALIZED = (1 << 12), /* init_toepcb has been called */ TPF_TLS_RECEIVE = (1 << 13), /* should receive TLS records */ TPF_TLS_ESTABLISHED = (1 << 14), /* TLS handshake timer initialized */ }; enum { DDP_OK = (1 << 0), /* OK to turn on DDP */ DDP_SC_REQ = (1 << 1), /* state change (on/off) requested */ DDP_ON = (1 << 2), /* DDP is turned on */ DDP_BUF0_ACTIVE = (1 << 3), /* buffer 0 in use (not invalidated) */ DDP_BUF1_ACTIVE = (1 << 4), /* buffer 1 in use (not invalidated) */ DDP_TASK_ACTIVE = (1 << 5), /* requeue task is queued / running */ DDP_DEAD = (1 << 6), /* toepcb is shutting down */ }; struct ctl_sg_entry; struct sockopt; struct offload_settings; /* * Connection parameters for an offloaded connection. These are mostly (but not * all) hardware TOE parameters. */ struct conn_params { int8_t rx_coalesce; int8_t cong_algo; int8_t tc_idx; int8_t tstamp; int8_t sack; int8_t nagle; int8_t keepalive; int8_t wscale; int8_t ecn; int8_t mtu_idx; int8_t ulp_mode; int8_t tx_align; int16_t txq_idx; /* ofld_txq = &sc->sge.ofld_txq[txq_idx] */ int16_t rxq_idx; /* ofld_rxq = &sc->sge.ofld_rxq[rxq_idx] */ int16_t l2t_idx; uint16_t emss; uint16_t opt0_bufsize; u_int sndbuf; /* controls TP tx pages */ }; struct ofld_tx_sdesc { uint32_t plen; /* payload length */ uint8_t tx_credits; /* firmware tx credits (unit is 16B) */ void *iv_buffer; /* optional buffer holding IVs for TLS */ }; struct ppod_region { u_int pr_start; u_int pr_len; u_int pr_page_shift[4]; uint32_t pr_tag_mask; /* hardware tagmask for this region. */ uint32_t pr_invalid_bit; /* OR with this to invalidate tag. */ uint32_t pr_alias_mask; /* AND with tag to get alias bits. */ u_int pr_alias_shift; /* shift this much for first alias bit. */ vmem_t *pr_arena; }; struct ppod_reservation { struct ppod_region *prsv_pr; uint32_t prsv_tag; /* Full tag: pgsz, alias, tag, color */ u_int prsv_nppods; }; struct pageset { TAILQ_ENTRY(pageset) link; vm_page_t *pages; int npages; int flags; int offset; /* offset in first page */ int len; struct ppod_reservation prsv; struct vmspace *vm; vm_offset_t start; u_int vm_timestamp; }; TAILQ_HEAD(pagesetq, pageset); #define PS_PPODS_WRITTEN 0x0001 /* Page pods written to the card. */ struct ddp_buffer { struct pageset *ps; struct kaiocb *job; int cancel_pending; }; struct ddp_pcb { u_int flags; struct ddp_buffer db[2]; TAILQ_HEAD(, pageset) cached_pagesets; TAILQ_HEAD(, kaiocb) aiojobq; u_int waiting_count; u_int active_count; u_int cached_count; int active_id; /* the currently active DDP buffer */ struct task requeue_task; struct kaiocb *queueing; struct mtx lock; }; struct toepcb { struct tom_data *td; struct inpcb *inp; /* backpointer to host stack's PCB */ u_int flags; /* miscellaneous flags */ TAILQ_ENTRY(toepcb) link; /* toep_list */ int refcount; struct vnet *vnet; struct vi_info *vi; /* virtual interface */ struct sge_ofld_txq *ofld_txq; struct sge_ofld_rxq *ofld_rxq; struct sge_wrq *ctrlq; struct l2t_entry *l2te; /* L2 table entry used by this connection */ struct clip_entry *ce; /* CLIP table entry used by this tid */ int tid; /* Connection identifier */ /* tx credit handling */ u_int tx_total; /* total tx WR credits (in 16B units) */ u_int tx_credits; /* tx WR credits (in 16B units) available */ u_int tx_nocompl; /* tx WR credits since last compl request */ u_int plen_nocompl; /* payload since last compl request */ struct conn_params params; void *ulpcb; void *ulpcb2; struct mbufq ulp_pduq; /* PDUs waiting to be sent out. */ struct mbufq ulp_pdu_reclaimq; struct ddp_pcb ddp; struct tls_ofld_info tls; TAILQ_HEAD(, kaiocb) aiotx_jobq; struct task aiotx_task; struct socket *aiotx_so; /* Tx software descriptor */ uint8_t txsd_total; uint8_t txsd_pidx; uint8_t txsd_cidx; uint8_t txsd_avail; struct ofld_tx_sdesc txsd[]; }; static inline int ulp_mode(struct toepcb *toep) { return (toep->params.ulp_mode); } #define DDP_LOCK(toep) mtx_lock(&(toep)->ddp.lock) #define DDP_UNLOCK(toep) mtx_unlock(&(toep)->ddp.lock) #define DDP_ASSERT_LOCKED(toep) mtx_assert(&(toep)->ddp.lock, MA_OWNED) /* * Compressed state for embryonic connections for a listener. */ struct synq_entry { struct listen_ctx *lctx; /* backpointer to listen ctx */ struct mbuf *syn; int flags; /* same as toepcb's tp_flags */ volatile int ok_to_respond; volatile u_int refcnt; int tid; uint32_t iss; uint32_t irs; uint32_t ts; uint32_t rss_hash; __be16 tcp_opt; /* from cpl_pass_establish */ struct toepcb *toep; struct conn_params params; }; /* listen_ctx flags */ #define LCTX_RPL_PENDING 1 /* waiting for a CPL_PASS_OPEN_RPL */ struct listen_ctx { LIST_ENTRY(listen_ctx) link; /* listen hash linkage */ volatile int refcount; int stid; struct stid_region stid_region; int flags; struct inpcb *inp; /* listening socket's inp */ struct vnet *vnet; struct sge_wrq *ctrlq; struct sge_ofld_rxq *ofld_rxq; struct clip_entry *ce; }; /* tcb_histent flags */ #define TE_RPL_PENDING 1 #define TE_ACTIVE 2 /* bits in one 8b tcb_histent sample. */ #define TS_RTO (1 << 0) #define TS_DUPACKS (1 << 1) #define TS_FASTREXMT (1 << 2) #define TS_SND_BACKLOGGED (1 << 3) #define TS_CWND_LIMITED (1 << 4) #define TS_ECN_ECE (1 << 5) #define TS_ECN_CWR (1 << 6) #define TS_RESERVED (1 << 7) /* Unused. */ struct tcb_histent { struct mtx te_lock; struct callout te_callout; uint64_t te_tcb[TCB_SIZE / sizeof(uint64_t)]; struct adapter *te_adapter; u_int te_flags; u_int te_tid; uint8_t te_pidx; uint8_t te_sample[100]; }; struct tom_data { struct toedev tod; /* toepcb's associated with this TOE device */ struct mtx toep_list_lock; TAILQ_HEAD(, toepcb) toep_list; struct mtx lctx_hash_lock; LIST_HEAD(, listen_ctx) *listen_hash; u_long listen_mask; int lctx_count; /* # of lctx in the hash table */ struct ppod_region pr; struct rwlock tcb_history_lock __aligned(CACHE_LINE_SIZE); struct tcb_histent **tcb_history; int dupack_threshold; /* WRs that will not be sent to the chip because L2 resolution failed */ struct mtx unsent_wr_lock; STAILQ_HEAD(, wrqe) unsent_wr_list; struct task reclaim_wr_resources; }; static inline struct tom_data * tod_td(struct toedev *tod) { return (__containerof(tod, struct tom_data, tod)); } static inline struct adapter * td_adapter(struct tom_data *td) { return (td->tod.tod_softc); } static inline void set_mbuf_raw_wr(struct mbuf *m, bool raw) { M_ASSERTPKTHDR(m); m->m_pkthdr.PH_per.eight[6] = raw; } static inline bool mbuf_raw_wr(struct mbuf *m) { M_ASSERTPKTHDR(m); return (m->m_pkthdr.PH_per.eight[6]); } static inline void set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode) { M_ASSERTPKTHDR(m); m->m_pkthdr.PH_per.eight[0] = ulp_submode; } static inline uint8_t mbuf_ulp_submode(struct mbuf *m) { M_ASSERTPKTHDR(m); return (m->m_pkthdr.PH_per.eight[0]); } /* t4_tom.c */ struct toepcb *alloc_toepcb(struct vi_info *, int); int init_toepcb(struct vi_info *, struct toepcb *); struct toepcb *hold_toepcb(struct toepcb *); void free_toepcb(struct toepcb *); void offload_socket(struct socket *, struct toepcb *); void restore_so_proto(struct socket *, bool); void undo_offload_socket(struct socket *); void final_cpl_received(struct toepcb *); void insert_tid(struct adapter *, int, void *, int); void *lookup_tid(struct adapter *, int); void update_tid(struct adapter *, int, void *); void remove_tid(struct adapter *, int, int); u_long select_rcv_wnd(struct socket *); int select_rcv_wscale(void); void init_conn_params(struct vi_info *, struct offload_settings *, struct in_conninfo *, struct socket *, const struct tcp_options *, int16_t, struct conn_params *cp); __be64 calc_options0(struct vi_info *, struct conn_params *); __be32 calc_options2(struct vi_info *, struct conn_params *); uint64_t select_ntuple(struct vi_info *, struct l2t_entry *); int negative_advice(int); int add_tid_to_history(struct adapter *, u_int); /* t4_connect.c */ void t4_init_connect_cpl_handlers(void); void t4_uninit_connect_cpl_handlers(void); int t4_connect(struct toedev *, struct socket *, struct nhop_object *, struct sockaddr *); void act_open_failure_cleanup(struct adapter *, u_int, u_int); /* t4_listen.c */ void t4_init_listen_cpl_handlers(void); void t4_uninit_listen_cpl_handlers(void); int t4_listen_start(struct toedev *, struct tcpcb *); int t4_listen_stop(struct toedev *, struct tcpcb *); void t4_syncache_added(struct toedev *, void *); void t4_syncache_removed(struct toedev *, void *); int t4_syncache_respond(struct toedev *, void *, struct mbuf *); int do_abort_req_synqe(struct sge_iq *, const struct rss_header *, struct mbuf *); int do_abort_rpl_synqe(struct sge_iq *, const struct rss_header *, struct mbuf *); void t4_offload_socket(struct toedev *, void *, struct socket *); void synack_failure_cleanup(struct adapter *, int); /* t4_cpl_io.c */ void aiotx_init_toep(struct toepcb *); int t4_aio_queue_aiotx(struct socket *, struct kaiocb *); void t4_init_cpl_io_handlers(void); void t4_uninit_cpl_io_handlers(void); void send_abort_rpl(struct adapter *, struct sge_ofld_txq *, int , int); void send_flowc_wr(struct toepcb *, struct tcpcb *); void send_reset(struct adapter *, struct toepcb *, uint32_t); int send_rx_credits(struct adapter *, struct toepcb *, int); void send_rx_modulate(struct adapter *, struct toepcb *); void make_established(struct toepcb *, uint32_t, uint32_t, uint16_t); int t4_close_conn(struct adapter *, struct toepcb *); void t4_rcvd(struct toedev *, struct tcpcb *); void t4_rcvd_locked(struct toedev *, struct tcpcb *); int t4_tod_output(struct toedev *, struct tcpcb *); int t4_send_fin(struct toedev *, struct tcpcb *); int t4_send_rst(struct toedev *, struct tcpcb *); void t4_set_tcb_field(struct adapter *, struct sge_wrq *, struct toepcb *, uint16_t, uint64_t, uint64_t, int, int); void t4_push_frames(struct adapter *, struct toepcb *, int); void t4_push_pdus(struct adapter *, struct toepcb *, int); /* t4_ddp.c */ int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int, const char *); void t4_free_ppod_region(struct ppod_region *); int t4_alloc_page_pods_for_ps(struct ppod_region *, struct pageset *); int t4_alloc_page_pods_for_buf(struct ppod_region *, vm_offset_t, int, struct ppod_reservation *); int t4_alloc_page_pods_for_sgl(struct ppod_region *, struct ctl_sg_entry *, int, struct ppod_reservation *); int t4_write_page_pods_for_ps(struct adapter *, struct sge_wrq *, int, struct pageset *); int t4_write_page_pods_for_buf(struct adapter *, struct toepcb *, - struct ppod_reservation *, vm_offset_t, int); + struct ppod_reservation *, vm_offset_t, int, struct mbufq *); int t4_write_page_pods_for_sgl(struct adapter *, struct toepcb *, - struct ppod_reservation *, struct ctl_sg_entry *, int, int); + struct ppod_reservation *, struct ctl_sg_entry *, int, int, struct mbufq *); void t4_free_page_pods(struct ppod_reservation *); int t4_soreceive_ddp(struct socket *, struct sockaddr **, struct uio *, struct mbuf **, struct mbuf **, int *); int t4_aio_queue_ddp(struct socket *, struct kaiocb *); void t4_ddp_mod_load(void); void t4_ddp_mod_unload(void); void ddp_assert_empty(struct toepcb *); void ddp_init_toep(struct toepcb *); void ddp_uninit_toep(struct toepcb *); void ddp_queue_toep(struct toepcb *); void release_ddp_resources(struct toepcb *toep); void handle_ddp_close(struct toepcb *, struct tcpcb *, uint32_t); void handle_ddp_indicate(struct toepcb *); void insert_ddp_data(struct toepcb *, uint32_t); const struct offload_settings *lookup_offload_policy(struct adapter *, int, struct mbuf *, uint16_t, struct inpcb *); /* t4_tls.c */ bool can_tls_offload(struct adapter *); void do_rx_data_tls(const struct cpl_rx_data *, struct toepcb *, struct mbuf *); int t4_ctloutput_tls(struct socket *, struct sockopt *); void t4_push_tls_records(struct adapter *, struct toepcb *, int); void t4_push_ktls(struct adapter *, struct toepcb *, int); void t4_tls_mod_load(void); void t4_tls_mod_unload(void); void tls_detach(struct toepcb *); void tls_establish(struct toepcb *); void tls_init_toep(struct toepcb *); int tls_rx_key(struct toepcb *); void tls_stop_handshake_timer(struct toepcb *); int tls_tx_key(struct toepcb *); void tls_uninit_toep(struct toepcb *); int tls_alloc_ktls(struct toepcb *, struct ktls_session *, int); #endif