diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -1753,17 +1753,18 @@ if (changed) { if (toep->ddp.flags & DDP_SC_REQ) toep->ddp.flags ^= DDP_ON | DDP_SC_REQ; - else { - KASSERT(cpl->ddp_off == 1, - ("%s: DDP switched on by itself.", - __func__)); - + else if (cpl->ddp_off == 1) { /* Fell out of DDP mode */ toep->ddp.flags &= ~DDP_ON; CTR1(KTR_CXGBE, "%s: fell out of DDP mode", __func__); insert_ddp_data(toep, ddp_placed); + } else { + /* + * Data was received while still + * ULP_MODE_NONE, just fall through. + */ } } diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -192,7 +192,7 @@ free_pageset(td, db->ps); } -void +static void ddp_init_toep(struct toepcb *toep) { @@ -810,6 +810,78 @@ return (0); } +static bool +set_ddp_ulp_mode(struct toepcb *toep) +{ + struct adapter *sc = toep->vi->adapter; + struct wrqe *wr; + struct work_request_hdr *wrh; + struct ulp_txpkt *ulpmc; + int fields, len; + + if (!sc->tt.ddp) + return (false); + + fields = 0; + + /* Overlay region including W_TCB_RX_DDP_FLAGS */ + fields += 3; + + /* W_TCB_ULP_TYPE */ + fields++; + +#ifdef USE_DDP_RX_FLOW_CONTROL + /* W_TCB_T_FLAGS */ + fields++; +#endif + + len = sizeof(*wrh) + fields * roundup2(LEN__SET_TCB_FIELD_ULP, 16); + KASSERT(len <= SGE_MAX_WR_LEN, + ("%s: WR with %d TCB field updates too large", __func__, fields)); + + wr = alloc_wrqe(len, toep->ctrlq); + if (wr == NULL) + return (false); + + CTR(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); + + wrh = wrtod(wr); + INIT_ULPTX_WRH(wrh, len, 1, 0); /* atomic */ + ulpmc = (struct ulp_txpkt *)(wrh + 1); + + /* + * Words 26/27 are zero except for the DDP_OFF flag in + * W_TCB_RX_DDP_FLAGS (27). + */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 26, + 0xffffffffffffffff, (uint64_t)V_TF_DDP_OFF(1) << 32); + + /* Words 28/29 are zero. */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 28, + 0xffffffffffffffff, 0); + + /* Words 30/31 are zero. */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, 30, + 0xffffffffffffffff, 0); + + /* Set the ULP mode to ULP_MODE_TCPDDP. */ + toep->params.ulp_mode = ULP_MODE_TCPDDP; + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_ULP_TYPE, + V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), + V_TCB_ULP_TYPE(ULP_MODE_TCPDDP)); + +#ifdef USE_DDP_RX_FLOW_CONTROL + /* Set TF_RX_FLOW_CONTROL_DDP. */ + ulpmc = mk_set_tcb_field_ulp(ulpmc, toep, W_TCB_T_FLAGS, + V_TF_RX_FLOW_CONTROL_DDP(1), V_TF_RX_FLOW_CONTROL_DDP(1)); +#endif + + ddp_init_toep(toep); + + t4_wrq_tx(sc, wr); + return (true); +} + static void enable_ddp(struct adapter *sc, struct toepcb *toep) { @@ -2203,7 +2275,8 @@ int t4_aio_queue_ddp(struct socket *so, struct kaiocb *job) { - struct tcpcb *tp = sototcpcb(so); + struct inpcb *inp = sotoinpcb(so); + struct tcpcb *tp = intotcpcb(inp); struct toepcb *toep = tp->t_toe; @@ -2211,6 +2284,15 @@ if (job->uaiocb.aio_lio_opcode != LIO_READ) return (EOPNOTSUPP); + INP_WLOCK(inp); + if (__predict_false(ulp_mode(toep) == ULP_MODE_NONE)) { + if (!set_ddp_ulp_mode(toep)) { + INP_WUNLOCK(inp); + return (EOPNOTSUPP); + } + } + INP_WUNLOCK(inp); + DDP_LOCK(toep); /* diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -505,7 +505,6 @@ void t4_ddp_mod_load(void); void t4_ddp_mod_unload(void); void ddp_assert_empty(struct toepcb *); -void ddp_init_toep(struct toepcb *); void ddp_uninit_toep(struct toepcb *); void ddp_queue_toep(struct toepcb *); void release_ddp_resources(struct toepcb *toep); diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c --- a/sys/dev/cxgbe/tom/t4_tom.c +++ b/sys/dev/cxgbe/tom/t4_tom.c @@ -179,8 +179,7 @@ toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; tls_init_toep(toep); - if (ulp_mode(toep) == ULP_MODE_TCPDDP) - ddp_init_toep(toep); + MPASS(ulp_mode(toep) != ULP_MODE_TCPDDP); toep->flags |= TPF_INITIALIZED; @@ -1216,10 +1215,7 @@ opt2 |= V_RX_COALESCE(M_RX_COALESCE); opt2 |= V_RX_FC_DDP(0) | V_RX_FC_DISABLE(0); -#ifdef USE_DDP_RX_FLOW_CONTROL - if (cp->ulp_mode == ULP_MODE_TCPDDP) - opt2 |= F_RX_FC_DDP; -#endif + MPASS(cp->ulp_mode != ULP_MODE_TCPDDP); return (htobe32(opt2)); } @@ -1327,11 +1323,7 @@ cp->tx_align = 0; /* ULP mode. */ - if (s->ddp > 0 || - (s->ddp < 0 && sc->tt.ddp && (so_options_get(so) & SO_NO_DDP) == 0)) - cp->ulp_mode = ULP_MODE_TCPDDP; - else - cp->ulp_mode = ULP_MODE_NONE; + cp->ulp_mode = ULP_MODE_NONE; /* Rx coalescing. */ if (s->rx_coalesce >= 0) @@ -1972,7 +1964,8 @@ if (SOLISTENING(so)) return (EINVAL); - if (ulp_mode(toep) == ULP_MODE_TCPDDP) { + if (ulp_mode(toep) == ULP_MODE_TCPDDP || + ulp_mode(toep) == ULP_MODE_NONE) { error = t4_aio_queue_ddp(so, job); if (error != EOPNOTSUPP) return (error);