Differential D21387 Diff 61334 head/sys/dev/cxgbe/tom/t4_cpl_io.c

Changeset View

Standalone View

head/sys/dev/cxgbe/tom/t4_cpl_io.c

Show First 20 Lines • Show All 71 Lines • ▼ Show 20 Lines
#include "common/t4_tcb.h"		#include "common/t4_tcb.h"
#include "tom/t4_tom_l2t.h"		#include "tom/t4_tom_l2t.h"
#include "tom/t4_tom.h"		#include "tom/t4_tom.h"

static void t4_aiotx_cancel(struct kaiocb *job);		static void t4_aiotx_cancel(struct kaiocb *job);
static void t4_aiotx_queue_toep(struct socket so, struct toepcb toep);		static void t4_aiotx_queue_toep(struct socket so, struct toepcb toep);

void		void
send_flowc_wr(struct toepcb toep, struct flowc_tx_params ftxp)		send_flowc_wr(struct toepcb toep, struct tcpcb tp)
{		{
struct wrqe *wr;		struct wrqe *wr;
struct fw_flowc_wr *flowc;		struct fw_flowc_wr *flowc;
unsigned int nparams, flowclen, paramidx;		unsigned int nparams, flowclen, paramidx;
struct vi_info *vi = toep->vi;		struct vi_info *vi = toep->vi;
struct port_info *pi = vi->pi;		struct port_info *pi = vi->pi;
struct adapter *sc = pi->adapter;		struct adapter *sc = pi->adapter;
unsigned int pfvf = sc->pf << S_FW_VIID_PFN;		unsigned int pfvf = sc->pf << S_FW_VIID_PFN;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];		struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];

KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),		KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
("%s: flowc for tid %u sent already", __func__, toep->tid));		("%s: flowc for tid %u sent already", __func__, toep->tid));

if (ftxp != NULL)		if (tp != NULL)
nparams = 8;		nparams = 8;
else		else
nparams = 6;		nparams = 6;
if (toep->ulp_mode == ULP_MODE_TLS)		if (ulp_mode(toep) == ULP_MODE_TLS)
nparams++;		nparams++;
if (toep->tls.fcplenmax != 0)		if (toep->tls.fcplenmax != 0)
nparams++;		nparams++;
if (toep->tc_idx != -1) {		if (toep->params.tc_idx != -1) {
MPASS(toep->tc_idx >= 0 &&		MPASS(toep->params.tc_idx >= 0 &&
toep->tc_idx < sc->chip_params->nsched_cls);		toep->params.tc_idx < sc->chip_params->nsched_cls);
nparams++;		nparams++;
}		}

flowclen = sizeof(flowc) + nparams sizeof(struct fw_flowc_mnemval);		flowclen = sizeof(flowc) + nparams sizeof(struct fw_flowc_mnemval);

wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);		wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
if (wr == NULL) {		if (wr == NULL) {
/* XXX */		/* XXX */
Show All 15 Lines	#define FLOWC_PARAM(__m, __v) \
} while (0)		} while (0)

paramidx = 0;		paramidx = 0;

FLOWC_PARAM(PFNVFN, pfvf);		FLOWC_PARAM(PFNVFN, pfvf);
FLOWC_PARAM(CH, pi->tx_chan);		FLOWC_PARAM(CH, pi->tx_chan);
FLOWC_PARAM(PORT, pi->tx_chan);		FLOWC_PARAM(PORT, pi->tx_chan);
FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id);		FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id);
if (ftxp) {		FLOWC_PARAM(SNDBUF, toep->params.sndbuf);
uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);		FLOWC_PARAM(MSS, toep->params.emss);
		if (tp) {
FLOWC_PARAM(SNDNXT, ftxp->snd_nxt);		FLOWC_PARAM(SNDNXT, tp->snd_nxt);
FLOWC_PARAM(RCVNXT, ftxp->rcv_nxt);		FLOWC_PARAM(RCVNXT, tp->rcv_nxt);
FLOWC_PARAM(SNDBUF, sndbuf);		}
FLOWC_PARAM(MSS, ftxp->mss);

CTR6(KTR_CXGBE,		CTR6(KTR_CXGBE,
"%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",		"%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
__func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt,		__func__, toep->tid, toep->params.emss, toep->params.sndbuf,
ftxp->rcv_nxt);		tp ? tp->snd_nxt : 0, tp ? tp->rcv_nxt : 0);
} else {
FLOWC_PARAM(SNDBUF, 512);
FLOWC_PARAM(MSS, 512);

CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);		if (ulp_mode(toep) == ULP_MODE_TLS)
}		FLOWC_PARAM(ULP_MODE, ulp_mode(toep));
if (toep->ulp_mode == ULP_MODE_TLS)
FLOWC_PARAM(ULP_MODE, toep->ulp_mode);
if (toep->tls.fcplenmax != 0)		if (toep->tls.fcplenmax != 0)
FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax);		FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax);
if (toep->tc_idx != -1)		if (toep->params.tc_idx != -1)
FLOWC_PARAM(SCHEDCLASS, toep->tc_idx);		FLOWC_PARAM(SCHEDCLASS, toep->params.tc_idx);
#undef FLOWC_PARAM		#undef FLOWC_PARAM

KASSERT(paramidx == nparams, ("nparams mismatch"));		KASSERT(paramidx == nparams, ("nparams mismatch"));

txsd->tx_credits = howmany(flowclen, 16);		txsd->tx_credits = howmany(flowclen, 16);
txsd->plen = 0;		txsd->plen = 0;
KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,		KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
("%s: not enough credits (%d)", __func__, toep->tx_credits));		("%s: not enough credits (%d)", __func__, toep->tx_credits));
Show All 24 Lines	if (kbps == 0) {
tc_idx = -1;		tc_idx = -1;
} else {		} else {
rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx);		rc = t4_reserve_cl_rl_kbps(sc, port_id, kbps, &tc_idx);
if (rc != 0)		if (rc != 0)
return (rc);		return (rc);
MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls);		MPASS(tc_idx >= 0 && tc_idx < sc->chip_params->nsched_cls);
}		}

if (toep->tc_idx != tc_idx) {		if (toep->params.tc_idx != tc_idx) {
struct wrqe *wr;		struct wrqe *wr;
struct fw_flowc_wr *flowc;		struct fw_flowc_wr *flowc;
int nparams = 1, flowclen, flowclen16;		int nparams = 1, flowclen, flowclen16;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];		struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];

flowclen = sizeof(flowc) + nparams sizeof(struct		flowclen = sizeof(flowc) + nparams sizeof(struct
fw_flowc_mnemval);		fw_flowc_mnemval);
flowclen16 = howmany(flowclen, 16);		flowclen16 = howmany(flowclen, 16);
Show All 22 Lines	if (toep->params.tc_idx != tc_idx) {
txsd->plen = 0;		txsd->plen = 0;
toep->tx_credits -= txsd->tx_credits;		toep->tx_credits -= txsd->tx_credits;
if (__predict_false(++toep->txsd_pidx == toep->txsd_total))		if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
toep->txsd_pidx = 0;		toep->txsd_pidx = 0;
toep->txsd_avail--;		toep->txsd_avail--;
t4_wrq_tx(sc, wr);		t4_wrq_tx(sc, wr);
}		}

if (toep->tc_idx >= 0)		if (toep->params.tc_idx >= 0)
t4_release_cl_rl(sc, port_id, toep->tc_idx);		t4_release_cl_rl(sc, port_id, toep->params.tc_idx);
toep->tc_idx = tc_idx;		toep->params.tc_idx = tc_idx;

return (0);		return (0);
}		}
#endif		#endif

void		void
send_reset(struct adapter sc, struct toepcb toep, uint32_t snd_nxt)		send_reset(struct adapter sc, struct toepcb toep, uint32_t snd_nxt)
{		{
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines
assign_rxopt(struct tcpcb *tp, uint16_t opt)		assign_rxopt(struct tcpcb *tp, uint16_t opt)
{		{
struct toepcb *toep = tp->t_toe;		struct toepcb *toep = tp->t_toe;
struct inpcb *inp = tp->t_inpcb;		struct inpcb *inp = tp->t_inpcb;
struct adapter *sc = td_adapter(toep->td);		struct adapter *sc = td_adapter(toep->td);

INP_LOCK_ASSERT(inp);		INP_LOCK_ASSERT(inp);

toep->tcp_opt = opt;		toep->params.mtu_idx = G_TCPOPT_MSS(opt);
toep->mtu_idx = G_TCPOPT_MSS(opt);		tp->t_maxseg = sc->params.mtus[toep->params.mtu_idx];
tp->t_maxseg = sc->params.mtus[toep->mtu_idx];
if (inp->inp_inc.inc_flags & INC_ISIPV6)		if (inp->inp_inc.inc_flags & INC_ISIPV6)
tp->t_maxseg -= sizeof(struct ip6_hdr) + sizeof(struct tcphdr);		tp->t_maxseg -= sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
else		else
tp->t_maxseg -= sizeof(struct ip) + sizeof(struct tcphdr);		tp->t_maxseg -= sizeof(struct ip) + sizeof(struct tcphdr);

toep->emss = tp->t_maxseg;		toep->params.emss = tp->t_maxseg;
if (G_TCPOPT_TSTAMP(opt)) {		if (G_TCPOPT_TSTAMP(opt)) {
		toep->params.tstamp = 1;
		toep->params.emss -= TCPOLEN_TSTAMP_APPA;
tp->t_flags \|= TF_RCVD_TSTMP; /* timestamps ok */		tp->t_flags \|= TF_RCVD_TSTMP; /* timestamps ok */
tp->ts_recent = 0; /* hmmm */		tp->ts_recent = 0; /* hmmm */
tp->ts_recent_age = tcp_ts_getticks();		tp->ts_recent_age = tcp_ts_getticks();
toep->emss -= TCPOLEN_TSTAMP_APPA;		} else
}		toep->params.tstamp = 0;

CTR6(KTR_CXGBE, "%s: tid %d, mtu_idx %u (%u), t_maxseg %u, emss %u",		if (G_TCPOPT_SACK(opt)) {
__func__, toep->tid, toep->mtu_idx,		toep->params.sack = 1;
sc->params.mtus[G_TCPOPT_MSS(opt)], tp->t_maxseg, toep->emss);

if (G_TCPOPT_SACK(opt))
tp->t_flags \|= TF_SACK_PERMIT; /* should already be set */		tp->t_flags \|= TF_SACK_PERMIT; /* should already be set */
else		} else {
		toep->params.sack = 0;
tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */		tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */
		}

if (G_TCPOPT_WSCALE_OK(opt))		if (G_TCPOPT_WSCALE_OK(opt))
tp->t_flags \|= TF_RCVD_SCALE;		tp->t_flags \|= TF_RCVD_SCALE;

/* Doing window scaling? */		/* Doing window scaling? */
if ((tp->t_flags & (TF_RCVD_SCALE \| TF_REQ_SCALE)) ==		if ((tp->t_flags & (TF_RCVD_SCALE \| TF_REQ_SCALE)) ==
(TF_RCVD_SCALE \| TF_REQ_SCALE)) {		(TF_RCVD_SCALE \| TF_REQ_SCALE)) {
tp->rcv_scale = tp->request_r_scale;		tp->rcv_scale = tp->request_r_scale;
tp->snd_scale = G_TCPOPT_SND_WSCALE(opt);		tp->snd_scale = G_TCPOPT_SND_WSCALE(opt);
		} else
		toep->params.wscale = 0;

		CTR6(KTR_CXGBE,
		"assign_rxopt: tid %d, mtu_idx %u, emss %u, ts %u, sack %u, wscale %u",
		toep->tid, toep->params.mtu_idx, toep->params.emss,
		toep->params.tstamp, toep->params.sack, toep->params.wscale);
}		}
}

/*		/*
* Completes some final bits of initialization for just established connections		* Completes some final bits of initialization for just established connections
* and changes their state to TCPS_ESTABLISHED.		* and changes their state to TCPS_ESTABLISHED.
*		*
* The ISNs are from the exchange of SYNs.		* The ISNs are from the exchange of SYNs.
*/		*/
void		void
make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt)		make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt)
{		{
struct inpcb *inp = toep->inp;		struct inpcb *inp = toep->inp;
struct socket *so = inp->inp_socket;		struct socket *so = inp->inp_socket;
struct tcpcb *tp = intotcpcb(inp);		struct tcpcb *tp = intotcpcb(inp);
long bufsize;
uint16_t tcpopt = be16toh(opt);		uint16_t tcpopt = be16toh(opt);
struct flowc_tx_params ftxp;

INP_WLOCK_ASSERT(inp);		INP_WLOCK_ASSERT(inp);
KASSERT(tp->t_state == TCPS_SYN_SENT \|\|		KASSERT(tp->t_state == TCPS_SYN_SENT \|\|
tp->t_state == TCPS_SYN_RECEIVED,		tp->t_state == TCPS_SYN_RECEIVED,
("%s: TCP state %s", __func__, tcpstates[tp->t_state]));		("%s: TCP state %s", __func__, tcpstates[tp->t_state]));

CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p",		CTR6(KTR_CXGBE, "%s: tid %d, so %p, inp %p, tp %p, toep %p",
__func__, toep->tid, so, inp, tp, toep);		__func__, toep->tid, so, inp, tp, toep);

tcp_state_change(tp, TCPS_ESTABLISHED);		tcp_state_change(tp, TCPS_ESTABLISHED);
tp->t_starttime = ticks;		tp->t_starttime = ticks;
TCPSTAT_INC(tcps_connects);		TCPSTAT_INC(tcps_connects);

tp->irs = irs;		tp->irs = irs;
tcp_rcvseqinit(tp);		tcp_rcvseqinit(tp);
tp->rcv_wnd = (u_int)toep->opt0_rcv_bufsize << 10;		tp->rcv_wnd = (u_int)toep->params.opt0_bufsize << 10;
tp->rcv_adv += tp->rcv_wnd;		tp->rcv_adv += tp->rcv_wnd;
tp->last_ack_sent = tp->rcv_nxt;		tp->last_ack_sent = tp->rcv_nxt;

tp->iss = iss;		tp->iss = iss;
tcp_sendseqinit(tp);		tcp_sendseqinit(tp);
tp->snd_una = iss + 1;		tp->snd_una = iss + 1;
tp->snd_nxt = iss + 1;		tp->snd_nxt = iss + 1;
tp->snd_max = iss + 1;		tp->snd_max = iss + 1;

assign_rxopt(tp, tcpopt);		assign_rxopt(tp, tcpopt);
		send_flowc_wr(toep, tp);

SOCKBUF_LOCK(&so->so_snd);
if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf)
bufsize = V_tcp_autosndbuf_max;
else
bufsize = sbspace(&so->so_snd);
SOCKBUF_UNLOCK(&so->so_snd);

ftxp.snd_nxt = tp->snd_nxt;
ftxp.rcv_nxt = tp->rcv_nxt;
ftxp.snd_space = bufsize;
ftxp.mss = toep->emss;
send_flowc_wr(toep, &ftxp);

soisconnected(so);		soisconnected(so);
}		}

int		int
send_rx_credits(struct adapter sc, struct toepcb toep, int credits)		send_rx_credits(struct adapter sc, struct toepcb toep, int credits)
{		{
struct wrqe *wr;		struct wrqe *wr;
struct cpl_rx_data_ack *req;		struct cpl_rx_data_ack *req;
Show All 39 Lines	t4_rcvd_locked(struct toedev tod, struct tcpcb tp)
struct sockbuf *sb = &so->so_rcv;		struct sockbuf *sb = &so->so_rcv;
struct toepcb *toep = tp->t_toe;		struct toepcb *toep = tp->t_toe;
int rx_credits;		int rx_credits;

INP_WLOCK_ASSERT(inp);		INP_WLOCK_ASSERT(inp);
SOCKBUF_LOCK_ASSERT(sb);		SOCKBUF_LOCK_ASSERT(sb);

rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;		rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
if (toep->ulp_mode == ULP_MODE_TLS) {		if (ulp_mode(toep) == ULP_MODE_TLS) {
if (toep->tls.rcv_over >= rx_credits) {		if (toep->tls.rcv_over >= rx_credits) {
toep->tls.rcv_over -= rx_credits;		toep->tls.rcv_over -= rx_credits;
rx_credits = 0;		rx_credits = 0;
} else {		} else {
rx_credits -= toep->tls.rcv_over;		rx_credits -= toep->tls.rcv_over;
toep->tls.rcv_over = 0;		toep->tls.rcv_over = 0;
}		}
}		}
▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Lines	max_dsgl_nsegs(int tx_credits)
if ((sge_pair_credits * 16) % 24 == 16)		if ((sge_pair_credits * 16) % 24 == 16)
nseg++;		nseg++;

return (nseg);		return (nseg);
}		}

static inline void		static inline void
write_tx_wr(void dst, struct toepcb toep, unsigned int immdlen,		write_tx_wr(void dst, struct toepcb toep, unsigned int immdlen,
unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign)		unsigned int plen, uint8_t credits, int shove, int ulp_submode)
{		{
struct fw_ofld_tx_data_wr *txwr = dst;		struct fw_ofld_tx_data_wr *txwr = dst;

txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) \|		txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) \|
V_FW_WR_IMMDLEN(immdlen));		V_FW_WR_IMMDLEN(immdlen));
txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) \|		txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) \|
V_FW_WR_LEN16(credits));		V_FW_WR_LEN16(credits));
txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) \|		txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ulp_mode(toep)) \|
V_TX_ULP_SUBMODE(ulp_submode) \| V_TX_URG(0) \| V_TX_SHOVE(shove));		V_TX_ULP_SUBMODE(ulp_submode) \| V_TX_URG(0) \| V_TX_SHOVE(shove));
txwr->plen = htobe32(plen);		txwr->plen = htobe32(plen);

if (txalign > 0) {		if (toep->params.tx_align > 0) {
struct tcpcb *tp = intotcpcb(toep->inp);		if (plen < 2 * toep->params.emss)

if (plen < 2 * toep->emss)
txwr->lsodisable_to_flags \|=		txwr->lsodisable_to_flags \|=
htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE);		htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE);
else		else
txwr->lsodisable_to_flags \|=		txwr->lsodisable_to_flags \|=
htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD \|		htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD \|
(tp->t_flags & TF_NODELAY ? 0 :		(toep->params.nagle == 0 ? 0 :
F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE));		F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE));
}		}
}		}

/*		/*
* Generate a DSGL from a starting mbuf. The total number of segments and the		* Generate a DSGL from a starting mbuf. The total number of segments and the
* maximum segments in any one mbuf are provided.		* maximum segments in any one mbuf are provided.
*/		*/
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines	t4_push_frames(struct adapter sc, struct toepcb toep, int drop)
int tx_credits, shove, compl, sowwakeup;		int tx_credits, shove, compl, sowwakeup;
struct ofld_tx_sdesc *txsd;		struct ofld_tx_sdesc *txsd;
bool nomap_mbuf_seen;		bool nomap_mbuf_seen;

INP_WLOCK_ASSERT(inp);		INP_WLOCK_ASSERT(inp);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,		KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));		("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));

KASSERT(toep->ulp_mode == ULP_MODE_NONE \|\|		KASSERT(ulp_mode(toep) == ULP_MODE_NONE \|\|
toep->ulp_mode == ULP_MODE_TCPDDP \|\|		ulp_mode(toep) == ULP_MODE_TCPDDP \|\|
toep->ulp_mode == ULP_MODE_TLS \|\|		ulp_mode(toep) == ULP_MODE_TLS \|\|
toep->ulp_mode == ULP_MODE_RDMA,		ulp_mode(toep) == ULP_MODE_RDMA,
("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));		("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));

#ifdef VERBOSE_TRACES		#ifdef VERBOSE_TRACES
CTR5(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",		CTR5(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
__func__, toep->tid, toep->flags, tp->t_flags, drop);		__func__, toep->tid, toep->flags, tp->t_flags, drop);
#endif		#endif
if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))		if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
return;		return;

▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Lines	if (plen <= max_imm && !nomap_mbuf_seen) {
toep->ofld_txq);		toep->ofld_txq);
if (wr == NULL) {		if (wr == NULL) {
/* XXX: how will we recover from this? */		/* XXX: how will we recover from this? */
toep->flags \|= TPF_TX_SUSPENDED;		toep->flags \|= TPF_TX_SUSPENDED;
return;		return;
}		}
txwr = wrtod(wr);		txwr = wrtod(wr);
credits = howmany(wr->wr_len, 16);		credits = howmany(wr->wr_len, 16);
write_tx_wr(txwr, toep, plen, plen, credits, shove, 0,		write_tx_wr(txwr, toep, plen, plen, credits, shove, 0);
sc->tt.tx_align);
m_copydata(sndptr, 0, plen, (void *)(txwr + 1));		m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
nsegs = 0;		nsegs = 0;
} else {		} else {
int wr_len;		int wr_len;

/* DSGL tx */		/* DSGL tx */

wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +		wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;		((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);		wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
if (wr == NULL) {		if (wr == NULL) {
/* XXX: how will we recover from this? */		/* XXX: how will we recover from this? */
toep->flags \|= TPF_TX_SUSPENDED;		toep->flags \|= TPF_TX_SUSPENDED;
return;		return;
}		}
txwr = wrtod(wr);		txwr = wrtod(wr);
credits = howmany(wr_len, 16);		credits = howmany(wr_len, 16);
write_tx_wr(txwr, toep, 0, plen, credits, shove, 0,		write_tx_wr(txwr, toep, 0, plen, credits, shove, 0);
sc->tt.tx_align);
write_tx_sgl(txwr + 1, sndptr, m, nsegs,		write_tx_sgl(txwr + 1, sndptr, m, nsegs,
max_nsegs_1mbuf);		max_nsegs_1mbuf);
if (wr_len & 0xf) {		if (wr_len & 0xf) {
uint64_t pad = (uint64_t )		uint64_t pad = (uint64_t )
((uintptr_t)txwr + wr_len);		((uintptr_t)txwr + wr_len);
*pad = 0;		*pad = 0;
}		}
}		}

KASSERT(toep->tx_credits >= credits,		KASSERT(toep->tx_credits >= credits,
("%s: not enough credits", __func__));		("%s: not enough credits", __func__));

toep->tx_credits -= credits;		toep->tx_credits -= credits;
toep->tx_nocompl += credits;		toep->tx_nocompl += credits;
toep->plen_nocompl += plen;		toep->plen_nocompl += plen;
if (toep->tx_credits <= toep->tx_total * 3 / 8 &&		if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
toep->tx_nocompl >= toep->tx_total / 4)		toep->tx_nocompl >= toep->tx_total / 4)
compl = 1;		compl = 1;

if (compl \|\| toep->ulp_mode == ULP_MODE_RDMA) {		if (compl \|\| ulp_mode(toep) == ULP_MODE_RDMA) {
txwr->op_to_immdlen \|= htobe32(F_FW_WR_COMPL);		txwr->op_to_immdlen \|= htobe32(F_FW_WR_COMPL);
toep->tx_nocompl = 0;		toep->tx_nocompl = 0;
toep->plen_nocompl = 0;		toep->plen_nocompl = 0;
}		}

tp->snd_nxt += plen;		tp->snd_nxt += plen;
tp->snd_max += plen;		tp->snd_max += plen;

▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines	t4_push_pdus(struct adapter sc, struct toepcb toep, int drop)
int tx_credits, shove;		int tx_credits, shove;
struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];		struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
struct mbufq *pduq = &toep->ulp_pduq;		struct mbufq *pduq = &toep->ulp_pduq;
static const u_int ulp_extra_len[] = {0, 4, 4, 8};		static const u_int ulp_extra_len[] = {0, 4, 4, 8};

INP_WLOCK_ASSERT(inp);		INP_WLOCK_ASSERT(inp);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT,		KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));		("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
KASSERT(toep->ulp_mode == ULP_MODE_ISCSI,		KASSERT(ulp_mode(toep) == ULP_MODE_ISCSI,
("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));		("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));

if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))		if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
return;		return;

/*		/*
* This function doesn't resume by itself. Someone else must clear the		* This function doesn't resume by itself. Someone else must clear the
* flag and call this function.		* flag and call this function.
*/		*/
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines	if (plen <= max_imm) {
if (wr == NULL) {		if (wr == NULL) {
/* XXX: how will we recover from this? */		/* XXX: how will we recover from this? */
toep->flags \|= TPF_TX_SUSPENDED;		toep->flags \|= TPF_TX_SUSPENDED;
return;		return;
}		}
txwr = wrtod(wr);		txwr = wrtod(wr);
credits = howmany(wr->wr_len, 16);		credits = howmany(wr->wr_len, 16);
write_tx_wr(txwr, toep, plen, adjusted_plen, credits,		write_tx_wr(txwr, toep, plen, adjusted_plen, credits,
shove, ulp_submode, sc->tt.tx_align);		shove, ulp_submode);
m_copydata(sndptr, 0, plen, (void *)(txwr + 1));		m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
nsegs = 0;		nsegs = 0;
} else {		} else {
int wr_len;		int wr_len;

/* DSGL tx */		/* DSGL tx */
wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +		wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;		((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);		wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
if (wr == NULL) {		if (wr == NULL) {
/* XXX: how will we recover from this? */		/* XXX: how will we recover from this? */
toep->flags \|= TPF_TX_SUSPENDED;		toep->flags \|= TPF_TX_SUSPENDED;
return;		return;
}		}
txwr = wrtod(wr);		txwr = wrtod(wr);
credits = howmany(wr_len, 16);		credits = howmany(wr_len, 16);
write_tx_wr(txwr, toep, 0, adjusted_plen, credits,		write_tx_wr(txwr, toep, 0, adjusted_plen, credits,
shove, ulp_submode, sc->tt.tx_align);		shove, ulp_submode);
write_tx_sgl(txwr + 1, sndptr, m, nsegs,		write_tx_sgl(txwr + 1, sndptr, m, nsegs,
max_nsegs_1mbuf);		max_nsegs_1mbuf);
if (wr_len & 0xf) {		if (wr_len & 0xf) {
uint64_t pad = (uint64_t )		uint64_t pad = (uint64_t )
((uintptr_t)txwr + wr_len);		((uintptr_t)txwr + wr_len);
*pad = 0;		*pad = 0;
}		}
}		}
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines
#endif		#endif
struct toepcb *toep = tp->t_toe;		struct toepcb *toep = tp->t_toe;

INP_WLOCK_ASSERT(inp);		INP_WLOCK_ASSERT(inp);
KASSERT((inp->inp_flags & INP_DROPPED) == 0,		KASSERT((inp->inp_flags & INP_DROPPED) == 0,
("%s: inp %p dropped.", __func__, inp));		("%s: inp %p dropped.", __func__, inp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));		KASSERT(toep != NULL, ("%s: toep is NULL", __func__));

if (toep->ulp_mode == ULP_MODE_ISCSI)		if (ulp_mode(toep) == ULP_MODE_ISCSI)
t4_push_pdus(sc, toep, 0);		t4_push_pdus(sc, toep, 0);
else if (tls_tx_key(toep))		else if (tls_tx_key(toep))
t4_push_tls_records(sc, toep, 0);		t4_push_tls_records(sc, toep, 0);
else		else
t4_push_frames(sc, toep, 0);		t4_push_frames(sc, toep, 0);

return (0);		return (0);
}		}
Show All 9 Lines	#endif

INP_WLOCK_ASSERT(inp);		INP_WLOCK_ASSERT(inp);
KASSERT((inp->inp_flags & INP_DROPPED) == 0,		KASSERT((inp->inp_flags & INP_DROPPED) == 0,
("%s: inp %p dropped.", __func__, inp));		("%s: inp %p dropped.", __func__, inp));
KASSERT(toep != NULL, ("%s: toep is NULL", __func__));		KASSERT(toep != NULL, ("%s: toep is NULL", __func__));

toep->flags \|= TPF_SEND_FIN;		toep->flags \|= TPF_SEND_FIN;
if (tp->t_state >= TCPS_ESTABLISHED) {		if (tp->t_state >= TCPS_ESTABLISHED) {
if (toep->ulp_mode == ULP_MODE_ISCSI)		if (ulp_mode(toep) == ULP_MODE_ISCSI)
t4_push_pdus(sc, toep, 0);		t4_push_pdus(sc, toep, 0);
else if (tls_tx_key(toep))		else if (tls_tx_key(toep))
t4_push_tls_records(sc, toep, 0);		t4_push_tls_records(sc, toep, 0);
else		else
t4_push_frames(sc, toep, 0);		t4_push_frames(sc, toep, 0);
}		}

return (0);		return (0);
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines	#endif

if (toep->flags & TPF_ABORT_SHUTDOWN)		if (toep->flags & TPF_ABORT_SHUTDOWN)
goto done;		goto done;

tp->rcv_nxt++; /* FIN */		tp->rcv_nxt++; /* FIN */

so = inp->inp_socket;		so = inp->inp_socket;
socantrcvmore(so);		socantrcvmore(so);
if (toep->ulp_mode == ULP_MODE_TCPDDP) {		if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
DDP_LOCK(toep);		DDP_LOCK(toep);
if (__predict_false(toep->ddp.flags &		if (__predict_false(toep->ddp.flags &
(DDP_BUF0_ACTIVE \| DDP_BUF1_ACTIVE)))		(DDP_BUF0_ACTIVE \| DDP_BUF1_ACTIVE)))
handle_ddp_close(toep, tp, cpl->rcv_nxt);		handle_ddp_close(toep, tp, cpl->rcv_nxt);
DDP_UNLOCK(toep);		DDP_UNLOCK(toep);
}		}

if (toep->ulp_mode != ULP_MODE_RDMA) {		if (ulp_mode(toep) != ULP_MODE_RDMA) {
KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),		KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt),
("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,		("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt,
be32toh(cpl->rcv_nxt)));		be32toh(cpl->rcv_nxt)));
}		}

switch (tp->t_state) {		switch (tp->t_state) {
case TCPS_SYN_RECEIVED:		case TCPS_SYN_RECEIVED:
tp->t_starttime = ticks;		tp->t_starttime = ticks;
▲ Show 20 Lines • Show All 294 Lines • ▼ Show 20 Lines	do_rx_data(struct sge_iq iq, const struct rss_header rss, struct mbuf *m)

tp = intotcpcb(inp);		tp = intotcpcb(inp);

if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq)))		if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq)))
ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt;		ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt;

tp->rcv_nxt += len;		tp->rcv_nxt += len;
if (tp->rcv_wnd < len) {		if (tp->rcv_wnd < len) {
KASSERT(toep->ulp_mode == ULP_MODE_RDMA,		KASSERT(ulp_mode(toep) == ULP_MODE_RDMA,
("%s: negative window size", __func__));		("%s: negative window size", __func__));
}		}

tp->rcv_wnd -= len;		tp->rcv_wnd -= len;
tp->t_rcvtime = ticks;		tp->t_rcvtime = ticks;

if (toep->ulp_mode == ULP_MODE_TCPDDP)		if (ulp_mode(toep) == ULP_MODE_TCPDDP)
DDP_LOCK(toep);		DDP_LOCK(toep);
so = inp_inpcbtosocket(inp);		so = inp_inpcbtosocket(inp);
sb = &so->so_rcv;		sb = &so->so_rcv;
SOCKBUF_LOCK(sb);		SOCKBUF_LOCK(sb);

if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {		if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",		CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
__func__, tid, len);		__func__, tid, len);
m_freem(m);		m_freem(m);
SOCKBUF_UNLOCK(sb);		SOCKBUF_UNLOCK(sb);
if (toep->ulp_mode == ULP_MODE_TCPDDP)		if (ulp_mode(toep) == ULP_MODE_TCPDDP)
DDP_UNLOCK(toep);		DDP_UNLOCK(toep);
INP_WUNLOCK(inp);		INP_WUNLOCK(inp);

CURVNET_SET(toep->vnet);		CURVNET_SET(toep->vnet);
INP_INFO_RLOCK_ET(&V_tcbinfo, et);		INP_INFO_RLOCK_ET(&V_tcbinfo, et);
INP_WLOCK(inp);		INP_WLOCK(inp);
tp = tcp_drop(tp, ECONNRESET);		tp = tcp_drop(tp, ECONNRESET);
if (tp)		if (tp)
Show All 14 Lines	if (sb->sb_flags & SB_AUTOSIZE &&
unsigned int hiwat = sb->sb_hiwat;		unsigned int hiwat = sb->sb_hiwat;
unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,		unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
V_tcp_autorcvbuf_max);		V_tcp_autorcvbuf_max);

if (!sbreserve_locked(sb, newsize, so, NULL))		if (!sbreserve_locked(sb, newsize, so, NULL))
sb->sb_flags &= ~SB_AUTOSIZE;		sb->sb_flags &= ~SB_AUTOSIZE;
}		}

if (toep->ulp_mode == ULP_MODE_TCPDDP) {		if (ulp_mode(toep) == ULP_MODE_TCPDDP) {
int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off;		int changed = !(toep->ddp.flags & DDP_ON) ^ cpl->ddp_off;

if (toep->ddp.waiting_count != 0 \|\| toep->ddp.active_count != 0)		if (toep->ddp.waiting_count != 0 \|\| toep->ddp.active_count != 0)
CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)",		CTR3(KTR_CXGBE, "%s: tid %u, non-ddp rx (%d bytes)",
__func__, tid, len);		__func__, tid, len);

if (changed) {		if (changed) {
if (toep->ddp.flags & DDP_SC_REQ)		if (toep->ddp.flags & DDP_SC_REQ)
Show All 26 Lines	do_rx_data(struct sge_iq iq, const struct rss_header rss, struct mbuf *m)
sbappendstream_locked(sb, m, 0);		sbappendstream_locked(sb, m, 0);
rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;		rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {		if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {
rx_credits = send_rx_credits(sc, toep, rx_credits);		rx_credits = send_rx_credits(sc, toep, rx_credits);
tp->rcv_wnd += rx_credits;		tp->rcv_wnd += rx_credits;
tp->rcv_adv += rx_credits;		tp->rcv_adv += rx_credits;
}		}

if (toep->ulp_mode == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 &&		if (ulp_mode(toep) == ULP_MODE_TCPDDP && toep->ddp.waiting_count > 0 &&
sbavail(sb) != 0) {		sbavail(sb) != 0) {
CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__,		CTR2(KTR_CXGBE, "%s: tid %u queueing AIO task", __func__,
tid);		tid);
ddp_queue_toep(toep);		ddp_queue_toep(toep);
}		}
sorwakeup_locked(so);		sorwakeup_locked(so);
SOCKBUF_UNLOCK_ASSERT(sb);		SOCKBUF_UNLOCK_ASSERT(sb);
if (toep->ulp_mode == ULP_MODE_TCPDDP)		if (ulp_mode(toep) == ULP_MODE_TCPDDP)
DDP_UNLOCK(toep);		DDP_UNLOCK(toep);

INP_WUNLOCK(inp);		INP_WUNLOCK(inp);
CURVNET_RESTORE();		CURVNET_RESTORE();
return (0);		return (0);
}		}

static int		static int
▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines	#endif
if (toep->flags & TPF_TX_SUSPENDED &&		if (toep->flags & TPF_TX_SUSPENDED &&
toep->tx_credits >= toep->tx_total / 4) {		toep->tx_credits >= toep->tx_total / 4) {
#ifdef VERBOSE_TRACES		#ifdef VERBOSE_TRACES
CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__,		CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__,
tid);		tid);
#endif		#endif
toep->flags &= ~TPF_TX_SUSPENDED;		toep->flags &= ~TPF_TX_SUSPENDED;
CURVNET_SET(toep->vnet);		CURVNET_SET(toep->vnet);
if (toep->ulp_mode == ULP_MODE_ISCSI)		if (ulp_mode(toep) == ULP_MODE_ISCSI)
t4_push_pdus(sc, toep, plen);		t4_push_pdus(sc, toep, plen);
else if (tls_tx_key(toep))		else if (tls_tx_key(toep))
t4_push_tls_records(sc, toep, plen);		t4_push_tls_records(sc, toep, plen);
else		else
t4_push_frames(sc, toep, plen);		t4_push_frames(sc, toep, plen);
CURVNET_RESTORE();		CURVNET_RESTORE();
} else if (plen > 0) {		} else if (plen > 0) {
struct sockbuf *sb = &so->so_snd;		struct sockbuf *sb = &so->so_snd;
int sbu;		int sbu;

SOCKBUF_LOCK(sb);		SOCKBUF_LOCK(sb);
sbu = sbused(sb);		sbu = sbused(sb);
if (toep->ulp_mode == ULP_MODE_ISCSI) {		if (ulp_mode(toep) == ULP_MODE_ISCSI) {

if (__predict_false(sbu > 0)) {		if (__predict_false(sbu > 0)) {
/*		/*
* The data trasmitted before the tid's ULP mode		* The data trasmitted before the tid's ULP mode
* changed to ISCSI is still in so_snd.		* changed to ISCSI is still in so_snd.
* Incoming credits should account for so_snd		* Incoming credits should account for so_snd
* first.		* first.
*/		*/
▲ Show 20 Lines • Show All 533 Lines • Show Last 20 Lines