Changeset View
Changeset View
Standalone View
Standalone View
head/sys/dev/cxgbe/tom/t4_listen.c
Show First 20 Lines • Show All 342 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
send_reset_synqe(struct toedev *tod, struct synq_entry *synqe) | send_reset_synqe(struct toedev *tod, struct synq_entry *synqe) | ||||
{ | { | ||||
struct adapter *sc = tod->tod_softc; | struct adapter *sc = tod->tod_softc; | ||||
struct mbuf *m = synqe->syn; | struct mbuf *m = synqe->syn; | ||||
struct ifnet *ifp = m->m_pkthdr.rcvif; | struct ifnet *ifp = m->m_pkthdr.rcvif; | ||||
struct vi_info *vi = ifp->if_softc; | struct vi_info *vi = ifp->if_softc; | ||||
struct port_info *pi = vi->pi; | struct port_info *pi = vi->pi; | ||||
struct l2t_entry *e = &sc->l2t->l2tab[synqe->l2e_idx]; | struct l2t_entry *e = &sc->l2t->l2tab[synqe->params.l2t_idx]; | ||||
struct wrqe *wr; | struct wrqe *wr; | ||||
struct fw_flowc_wr *flowc; | struct fw_flowc_wr *flowc; | ||||
struct cpl_abort_req *req; | struct cpl_abort_req *req; | ||||
int flowclen; | int flowclen; | ||||
struct sge_wrq *ofld_txq; | struct sge_wrq *ofld_txq; | ||||
struct sge_ofld_rxq *ofld_rxq; | struct sge_ofld_rxq *ofld_rxq; | ||||
const int nparams = 6; | const int nparams = 6; | ||||
const u_int pfvf = sc->pf << S_FW_VIID_PFN; | const u_int pfvf = sc->pf << S_FW_VIID_PFN; | ||||
INP_WLOCK_ASSERT(synqe->lctx->inp); | INP_WLOCK_ASSERT(synqe->lctx->inp); | ||||
CTR5(KTR_CXGBE, "%s: synqe %p (0x%x), tid %d%s", | CTR5(KTR_CXGBE, "%s: synqe %p (0x%x), tid %d%s", | ||||
__func__, synqe, synqe->flags, synqe->tid, | __func__, synqe, synqe->flags, synqe->tid, | ||||
synqe->flags & TPF_ABORT_SHUTDOWN ? | synqe->flags & TPF_ABORT_SHUTDOWN ? | ||||
" (abort already in progress)" : ""); | " (abort already in progress)" : ""); | ||||
if (synqe->flags & TPF_ABORT_SHUTDOWN) | if (synqe->flags & TPF_ABORT_SHUTDOWN) | ||||
return; /* abort already in progress */ | return; /* abort already in progress */ | ||||
synqe->flags |= TPF_ABORT_SHUTDOWN; | synqe->flags |= TPF_ABORT_SHUTDOWN; | ||||
ofld_txq = &sc->sge.ofld_txq[synqe->txqid]; | ofld_txq = &sc->sge.ofld_txq[synqe->params.txq_idx]; | ||||
ofld_rxq = &sc->sge.ofld_rxq[synqe->rxqid]; | ofld_rxq = &sc->sge.ofld_rxq[synqe->params.rxq_idx]; | ||||
/* The wrqe will have two WRs - a flowc followed by an abort_req */ | /* The wrqe will have two WRs - a flowc followed by an abort_req */ | ||||
flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); | flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); | ||||
wr = alloc_wrqe(roundup2(flowclen, EQ_ESIZE) + sizeof(*req), ofld_txq); | wr = alloc_wrqe(roundup2(flowclen, EQ_ESIZE) + sizeof(*req), ofld_txq); | ||||
if (wr == NULL) { | if (wr == NULL) { | ||||
/* XXX */ | /* XXX */ | ||||
panic("%s: allocation failure.", __func__); | panic("%s: allocation failure.", __func__); | ||||
▲ Show 20 Lines • Show All 450 Lines • ▼ Show 20 Lines | #endif | ||||
return (status); | return (status); | ||||
} | } | ||||
static void | static void | ||||
done_with_synqe(struct adapter *sc, struct synq_entry *synqe) | done_with_synqe(struct adapter *sc, struct synq_entry *synqe) | ||||
{ | { | ||||
struct listen_ctx *lctx = synqe->lctx; | struct listen_ctx *lctx = synqe->lctx; | ||||
struct inpcb *inp = lctx->inp; | struct inpcb *inp = lctx->inp; | ||||
struct l2t_entry *e = &sc->l2t->l2tab[synqe->l2e_idx]; | struct l2t_entry *e = &sc->l2t->l2tab[synqe->params.l2t_idx]; | ||||
int ntids; | int ntids; | ||||
INP_WLOCK_ASSERT(inp); | INP_WLOCK_ASSERT(inp); | ||||
ntids = inp->inp_vflag & INP_IPV6 ? 2 : 1; | ntids = inp->inp_vflag & INP_IPV6 ? 2 : 1; | ||||
remove_tid(sc, synqe->tid, ntids); | remove_tid(sc, synqe->tid, ntids); | ||||
release_tid(sc, synqe->tid, lctx->ctrlq); | release_tid(sc, synqe->tid, lctx->ctrlq); | ||||
t4_l2t_release(e); | t4_l2t_release(e); | ||||
Show All 34 Lines | #endif | ||||
CTR6(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x), lctx %p, status %d", | CTR6(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x), lctx %p, status %d", | ||||
__func__, tid, synqe, synqe->flags, synqe->lctx, cpl->status); | __func__, tid, synqe, synqe->flags, synqe->lctx, cpl->status); | ||||
if (negative_advice(cpl->status)) | if (negative_advice(cpl->status)) | ||||
return (0); /* Ignore negative advice */ | return (0); /* Ignore negative advice */ | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
ofld_txq = &sc->sge.ofld_txq[synqe->txqid]; | ofld_txq = &sc->sge.ofld_txq[synqe->params.txq_idx]; | ||||
/* | /* | ||||
* If we'd initiated an abort earlier the reply to it is responsible for | * If we'd initiated an abort earlier the reply to it is responsible for | ||||
* cleaning up resources. Otherwise we tear everything down right here | * cleaning up resources. Otherwise we tear everything down right here | ||||
* right now. We owe the T4 a CPL_ABORT_RPL no matter what. | * right now. We owe the T4 a CPL_ABORT_RPL no matter what. | ||||
*/ | */ | ||||
if (synqe->flags & TPF_ABORT_SHUTDOWN) { | if (synqe->flags & TPF_ABORT_SHUTDOWN) { | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | #endif | ||||
offload_socket(so, toep); | offload_socket(so, toep); | ||||
make_established(toep, synqe->iss, synqe->irs, synqe->tcp_opt); | make_established(toep, synqe->iss, synqe->irs, synqe->tcp_opt); | ||||
toep->flags |= TPF_CPL_PENDING; | toep->flags |= TPF_CPL_PENDING; | ||||
update_tid(sc, synqe->tid, toep); | update_tid(sc, synqe->tid, toep); | ||||
synqe->flags |= TPF_SYNQE_EXPANDED; | synqe->flags |= TPF_SYNQE_EXPANDED; | ||||
} | } | ||||
static inline void | |||||
save_qids_in_synqe(struct synq_entry *synqe, struct vi_info *vi, | |||||
struct offload_settings *s) | |||||
{ | |||||
uint32_t txqid, rxqid; | |||||
if (s->txq >= 0 && s->txq < vi->nofldtxq) | |||||
txqid = s->txq; | |||||
else | |||||
txqid = arc4random() % vi->nofldtxq; | |||||
txqid += vi->first_ofld_txq; | |||||
if (s->rxq >= 0 && s->rxq < vi->nofldrxq) | |||||
rxqid = s->rxq; | |||||
else | |||||
rxqid = arc4random() % vi->nofldrxq; | |||||
rxqid += vi->first_ofld_rxq; | |||||
synqe->txqid = txqid; | |||||
synqe->rxqid = rxqid; | |||||
} | |||||
static void | static void | ||||
t4opt_to_tcpopt(const struct tcp_options *t4opt, struct tcpopt *to) | t4opt_to_tcpopt(const struct tcp_options *t4opt, struct tcpopt *to) | ||||
{ | { | ||||
bzero(to, sizeof(*to)); | bzero(to, sizeof(*to)); | ||||
if (t4opt->mss) { | if (t4opt->mss) { | ||||
to->to_flags |= TOF_MSS; | to->to_flags |= TOF_MSS; | ||||
to->to_mss = be16toh(t4opt->mss); | to->to_mss = be16toh(t4opt->mss); | ||||
} | } | ||||
if (t4opt->wsf > 0 && t4opt->wsf < 15) { | if (t4opt->wsf > 0 && t4opt->wsf < 15) { | ||||
to->to_flags |= TOF_SCALE; | to->to_flags |= TOF_SCALE; | ||||
to->to_wscale = t4opt->wsf; | to->to_wscale = t4opt->wsf; | ||||
} | } | ||||
if (t4opt->tstamp) | if (t4opt->tstamp) | ||||
to->to_flags |= TOF_TS; | to->to_flags |= TOF_TS; | ||||
if (t4opt->sack) | if (t4opt->sack) | ||||
to->to_flags |= TOF_SACKPERM; | to->to_flags |= TOF_SACKPERM; | ||||
} | } | ||||
/* | |||||
* Options2 for passive open. | |||||
*/ | |||||
static uint32_t | |||||
calc_opt2p(struct adapter *sc, struct port_info *pi, int rxqid, | |||||
const struct tcp_options *tcpopt, struct tcphdr *th, int ulp_mode, | |||||
struct cc_algo *cc, const struct offload_settings *s) | |||||
{ | |||||
struct sge_ofld_rxq *ofld_rxq = &sc->sge.ofld_rxq[rxqid]; | |||||
uint32_t opt2 = 0; | |||||
/* | |||||
* rx flow control, rx coalesce, congestion control, and tx pace are all | |||||
* explicitly set by the driver. On T5+ the ISS is also set by the | |||||
* driver to the value picked by the kernel. | |||||
*/ | |||||
if (is_t4(sc)) { | |||||
opt2 |= F_RX_FC_VALID | F_RX_COALESCE_VALID; | |||||
opt2 |= F_CONG_CNTRL_VALID | F_PACE_VALID; | |||||
} else { | |||||
opt2 |= F_T5_OPT_2_VALID; /* all 4 valid */ | |||||
opt2 |= F_T5_ISS; /* ISS provided in CPL */ | |||||
} | |||||
if (tcpopt->sack && (s->sack > 0 || (s->sack < 0 && V_tcp_do_rfc1323))) | |||||
opt2 |= F_SACK_EN; | |||||
if (tcpopt->tstamp && | |||||
(s->tstamp > 0 || (s->tstamp < 0 && V_tcp_do_rfc1323))) | |||||
opt2 |= F_TSTAMPS_EN; | |||||
if (tcpopt->wsf < 15 && V_tcp_do_rfc1323) | |||||
opt2 |= F_WND_SCALE_EN; | |||||
if (th->th_flags & (TH_ECE | TH_CWR) && | |||||
(s->ecn > 0 || (s->ecn < 0 && V_tcp_do_ecn))) | |||||
opt2 |= F_CCTRL_ECN; | |||||
/* XXX: F_RX_CHANNEL for multiple rx c-chan support goes here. */ | |||||
opt2 |= V_TX_QUEUE(sc->params.tp.tx_modq[pi->tx_chan]); | |||||
/* These defaults are subject to ULP specific fixups later. */ | |||||
opt2 |= V_RX_FC_DDP(0) | V_RX_FC_DISABLE(0); | |||||
opt2 |= V_PACE(0); | |||||
if (s->cong_algo >= 0) | |||||
opt2 |= V_CONG_CNTRL(s->cong_algo); | |||||
else if (sc->tt.cong_algorithm >= 0) | |||||
opt2 |= V_CONG_CNTRL(sc->tt.cong_algorithm & M_CONG_CNTRL); | |||||
else { | |||||
if (strcasecmp(cc->name, "reno") == 0) | |||||
opt2 |= V_CONG_CNTRL(CONG_ALG_RENO); | |||||
else if (strcasecmp(cc->name, "tahoe") == 0) | |||||
opt2 |= V_CONG_CNTRL(CONG_ALG_TAHOE); | |||||
if (strcasecmp(cc->name, "newreno") == 0) | |||||
opt2 |= V_CONG_CNTRL(CONG_ALG_NEWRENO); | |||||
if (strcasecmp(cc->name, "highspeed") == 0) | |||||
opt2 |= V_CONG_CNTRL(CONG_ALG_HIGHSPEED); | |||||
else { | |||||
/* | |||||
* Use newreno in case the algorithm selected by the | |||||
* host stack is not supported by the hardware. | |||||
*/ | |||||
opt2 |= V_CONG_CNTRL(CONG_ALG_NEWRENO); | |||||
} | |||||
} | |||||
if (s->rx_coalesce > 0 || (s->rx_coalesce < 0 && sc->tt.rx_coalesce)) | |||||
opt2 |= V_RX_COALESCE(M_RX_COALESCE); | |||||
/* Note that ofld_rxq is already set according to s->rxq. */ | |||||
opt2 |= F_RSS_QUEUE_VALID; | |||||
opt2 |= V_RSS_QUEUE(ofld_rxq->iq.abs_id); | |||||
#ifdef USE_DDP_RX_FLOW_CONTROL | |||||
if (ulp_mode == ULP_MODE_TCPDDP) | |||||
opt2 |= F_RX_FC_DDP; | |||||
#endif | |||||
if (ulp_mode == ULP_MODE_TLS) { | |||||
opt2 &= ~V_RX_COALESCE(M_RX_COALESCE); | |||||
opt2 |= F_RX_FC_DISABLE; | |||||
} | |||||
return (htobe32(opt2)); | |||||
} | |||||
static void | static void | ||||
pass_accept_req_to_protohdrs(struct adapter *sc, const struct mbuf *m, | pass_accept_req_to_protohdrs(struct adapter *sc, const struct mbuf *m, | ||||
struct in_conninfo *inc, struct tcphdr *th) | struct in_conninfo *inc, struct tcphdr *th) | ||||
{ | { | ||||
const struct cpl_pass_accept_req *cpl = mtod(m, const void *); | const struct cpl_pass_accept_req *cpl = mtod(m, const void *); | ||||
const struct ether_header *eh; | const struct ether_header *eh; | ||||
unsigned int hlen = be32toh(cpl->hdr_len); | unsigned int hlen = be32toh(cpl->hdr_len); | ||||
uintptr_t l3hdr; | uintptr_t l3hdr; | ||||
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
static int | static int | ||||
send_synack(struct adapter *sc, struct synq_entry *synqe, uint64_t opt0, | send_synack(struct adapter *sc, struct synq_entry *synqe, uint64_t opt0, | ||||
uint32_t opt2, int tid) | uint32_t opt2, int tid) | ||||
{ | { | ||||
struct wrqe *wr; | struct wrqe *wr; | ||||
struct cpl_pass_accept_rpl *rpl; | struct cpl_pass_accept_rpl *rpl; | ||||
struct l2t_entry *e = &sc->l2t->l2tab[synqe->l2e_idx]; | struct l2t_entry *e = &sc->l2t->l2tab[synqe->params.l2t_idx]; | ||||
wr = alloc_wrqe(is_t4(sc) ? sizeof(struct cpl_pass_accept_rpl) : | wr = alloc_wrqe(is_t4(sc) ? sizeof(struct cpl_pass_accept_rpl) : | ||||
sizeof(struct cpl_t5_pass_accept_rpl), &sc->sge.ctrlq[0]); | sizeof(struct cpl_t5_pass_accept_rpl), &sc->sge.ctrlq[0]); | ||||
if (wr == NULL) | if (wr == NULL) | ||||
return (ENOMEM); | return (ENOMEM); | ||||
rpl = wrtod(wr); | rpl = wrtod(wr); | ||||
if (is_t4(sc)) | if (is_t4(sc)) | ||||
▲ Show 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | found: | ||||
synqe = alloc_synqe(sc, lctx, M_NOWAIT); | synqe = alloc_synqe(sc, lctx, M_NOWAIT); | ||||
if (synqe == NULL) { | if (synqe == NULL) { | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
REJECT_PASS_ACCEPT_REQ(true); | REJECT_PASS_ACCEPT_REQ(true); | ||||
} | } | ||||
atomic_store_int(&synqe->ok_to_respond, 0); | atomic_store_int(&synqe->ok_to_respond, 0); | ||||
init_conn_params(vi, &settings, &inc, so, &cpl->tcpopt, e->idx, | |||||
&synqe->params); | |||||
/* | /* | ||||
* If all goes well t4_syncache_respond will get called during | * If all goes well t4_syncache_respond will get called during | ||||
* syncache_add. Note that syncache_add releases the pcb lock. | * syncache_add. Note that syncache_add releases the pcb lock. | ||||
*/ | */ | ||||
t4opt_to_tcpopt(&cpl->tcpopt, &to); | t4opt_to_tcpopt(&cpl->tcpopt, &to); | ||||
toe_syncache_add(&inc, &to, &th, inp, tod, synqe); | toe_syncache_add(&inc, &to, &th, inp, tod, synqe); | ||||
if (atomic_load_int(&synqe->ok_to_respond) > 0) { | if (atomic_load_int(&synqe->ok_to_respond) > 0) { | ||||
uint64_t opt0; | uint64_t opt0; | ||||
uint32_t opt2; | uint32_t opt2; | ||||
u_int wnd; | |||||
int rscale, mtu_idx, rx_credits; | |||||
mtu_idx = find_best_mtu_idx(sc, &inc, &settings); | opt0 = calc_options0(vi, &synqe->params); | ||||
rscale = cpl->tcpopt.wsf && V_tcp_do_rfc1323 ? select_rcv_wscale() : 0; | opt2 = calc_options2(vi, &synqe->params); | ||||
wnd = max(so->sol_sbrcv_hiwat, MIN_RCV_WND); | |||||
wnd = min(wnd, MAX_RCV_WND); | |||||
rx_credits = min(wnd >> 10, M_RCV_BUFSIZ); | |||||
save_qids_in_synqe(synqe, vi, &settings); | |||||
synqe->ulp_mode = select_ulp_mode(so, sc, &settings); | |||||
opt0 = calc_opt0(so, vi, e, mtu_idx, rscale, rx_credits, | |||||
synqe->ulp_mode, &settings); | |||||
opt2 = calc_opt2p(sc, pi, synqe->rxqid, &cpl->tcpopt, &th, | |||||
synqe->ulp_mode, CC_ALGO(intotcpcb(inp)), &settings); | |||||
insert_tid(sc, tid, synqe, ntids); | insert_tid(sc, tid, synqe, ntids); | ||||
synqe->tid = tid; | synqe->tid = tid; | ||||
synqe->l2e_idx = e->idx; | |||||
synqe->rcv_bufsize = rx_credits; | |||||
synqe->syn = m; | synqe->syn = m; | ||||
m = NULL; | m = NULL; | ||||
if (send_synack(sc, synqe, opt0, opt2, tid) != 0) { | if (send_synack(sc, synqe, opt0, opt2, tid) != 0) { | ||||
remove_tid(sc, tid, ntids); | remove_tid(sc, tid, ntids); | ||||
m = synqe->syn; | m = synqe->syn; | ||||
synqe->syn = NULL; | synqe->syn = NULL; | ||||
REJECT_PASS_ACCEPT_REQ(true); | REJECT_PASS_ACCEPT_REQ(true); | ||||
} | } | ||||
CTR6(KTR_CXGBE, | CTR6(KTR_CXGBE, | ||||
"%s: stid %u, tid %u, lctx %p, synqe %p, mode %d, SYNACK", | "%s: stid %u, tid %u, synqe %p, opt0 %#016lx, opt2 %#08x", | ||||
__func__, stid, tid, lctx, synqe, synqe->ulp_mode); | __func__, stid, tid, synqe, be64toh(opt0), be32toh(opt2)); | ||||
} else | } else | ||||
REJECT_PASS_ACCEPT_REQ(false); | REJECT_PASS_ACCEPT_REQ(false); | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
return (0); | return (0); | ||||
reject: | reject: | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
CTR4(KTR_CXGBE, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid, | CTR4(KTR_CXGBE, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid, | ||||
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Lines | |||||
reset: | reset: | ||||
send_reset_synqe(TOEDEV(ifp), synqe); | send_reset_synqe(TOEDEV(ifp), synqe); | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); | INP_INFO_RUNLOCK_ET(&V_tcbinfo, et); | ||||
CURVNET_RESTORE(); | CURVNET_RESTORE(); | ||||
return (0); | return (0); | ||||
} | } | ||||
KASSERT(synqe->rxqid == iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0], | KASSERT(synqe->params.rxq_idx == iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0], | ||||
("%s: CPL arrived on unexpected rxq. %d %d", __func__, | ("%s: CPL arrived on unexpected rxq. %d %d", __func__, | ||||
synqe->rxqid, (int)(iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0]))); | synqe->params.rxq_idx, | ||||
(int)(iq_to_ofld_rxq(iq) - &sc->sge.ofld_rxq[0]))); | |||||
toep = alloc_toepcb(vi, synqe->txqid, synqe->rxqid, M_NOWAIT); | toep = alloc_toepcb(vi, M_NOWAIT); | ||||
if (toep == NULL) | if (toep == NULL) | ||||
goto reset; | goto reset; | ||||
toep->tid = tid; | toep->tid = tid; | ||||
toep->l2te = &sc->l2t->l2tab[synqe->l2e_idx]; | toep->l2te = &sc->l2t->l2tab[synqe->params.l2t_idx]; | ||||
toep->vnet = lctx->vnet; | toep->vnet = lctx->vnet; | ||||
set_ulp_mode(toep, synqe->ulp_mode); | bcopy(&synqe->params, &toep->params, sizeof(toep->params)); | ||||
toep->opt0_rcv_bufsize = synqe->rcv_bufsize; | init_toepcb(vi, toep); | ||||
MPASS(be32toh(cpl->snd_isn) - 1 == synqe->iss); | MPASS(be32toh(cpl->snd_isn) - 1 == synqe->iss); | ||||
MPASS(be32toh(cpl->rcv_isn) - 1 == synqe->irs); | MPASS(be32toh(cpl->rcv_isn) - 1 == synqe->irs); | ||||
synqe->tcp_opt = cpl->tcp_opt; | synqe->tcp_opt = cpl->tcp_opt; | ||||
synqe->toep = toep; | synqe->toep = toep; | ||||
/* Come up with something that syncache_expand should be ok with. */ | /* Come up with something that syncache_expand should be ok with. */ | ||||
synqe_to_protohdrs(sc, synqe, cpl, &inc, &th, &to); | synqe_to_protohdrs(sc, synqe, cpl, &inc, &th, &to); | ||||
▲ Show 20 Lines • Show All 59 Lines • Show Last 20 Lines |