Index: head/sys/dev/cxgbe/cxgbei/cxgbei.c =================================================================== --- head/sys/dev/cxgbe/cxgbei/cxgbei.c (nonexistent) +++ head/sys/dev/cxgbe/cxgbei/cxgbei.c (revision 292740) @@ -0,0 +1,1131 @@ +/*- + * Copyright (c) 2012 Chelsio Communications, Inc. + * All rights reserved. + * + * Chelsio T5xx iSCSI driver + * + * Written by: Sreenivasa Honnur + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include + +#ifdef TCP_OFFLOAD +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common.h" +#include "common/t4_msg.h" +#include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */ +#include "tom/t4_tom.h" +#include "cxgbei.h" +#include "cxgbei_ulp2_ddp.h" + +static int worker_thread_count; +static struct cxgbei_worker_thread_softc *cwt_softc; +static struct proc *cxgbei_proc; + +/* XXXNP some header instead. */ +struct icl_pdu *icl_cxgbei_new_pdu(int); +void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *); +void icl_cxgbei_conn_pdu_free(struct icl_conn *, struct icl_pdu *); + +/* + * Direct Data Placement - + * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted + * final destination host-memory buffers based on the Initiator Task Tag (ITT) + * in Data-In or Target Task Tag (TTT) in Data-Out PDUs. + * The host memory address is programmed into h/w in the format of pagepod + * entries. + * The location of the pagepod entry is encoded into ddp tag which is used as + * the base for ITT/TTT. + */ + +/* + * functions to program the pagepod in h/w + */ +static void inline +ppod_set(struct pagepod *ppod, + struct cxgbei_ulp2_pagepod_hdr *hdr, + struct cxgbei_ulp2_gather_list *gl, + unsigned int pidx) +{ + int i; + + memcpy(ppod, hdr, sizeof(*hdr)); + + for (i = 0; i < (PPOD_PAGES + 1); i++, pidx++) { + ppod->addr[i] = pidx < gl->nelem ? + cpu_to_be64(gl->dma_sg[pidx].phys_addr) : 0ULL; + } +} + +static void inline +ppod_clear(struct pagepod *ppod) +{ + memset(ppod, 0, sizeof(*ppod)); +} + +static inline void +ulp_mem_io_set_hdr(struct adapter *sc, int tid, struct ulp_mem_io *req, + unsigned int wr_len, unsigned int dlen, + unsigned int pm_addr) +{ + struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1); + + INIT_ULPTX_WR(req, wr_len, 0, 0); + req->cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | + V_ULP_MEMIO_ORDER(is_t4(sc)) | + V_T5_ULP_MEMIO_IMM(is_t5(sc))); + req->dlen = htonl(V_ULP_MEMIO_DATA_LEN(dlen >> 5)); + req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16) + | V_FW_WR_FLOWID(tid)); + req->lock_addr = htonl(V_ULP_MEMIO_ADDR(pm_addr >> 5)); + + idata->cmd_more = htonl(V_ULPTX_CMD(ULP_TX_SC_IMM)); + idata->len = htonl(dlen); +} + +#define PPOD_SIZE sizeof(struct pagepod) +#define ULPMEM_IDATA_MAX_NPPODS 1 /* 256/PPOD_SIZE */ +#define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */ + +static int +ppod_write_idata(struct cxgbei_data *ci, + struct cxgbei_ulp2_pagepod_hdr *hdr, + unsigned int idx, unsigned int npods, + struct cxgbei_ulp2_gather_list *gl, + unsigned int gl_pidx, struct toepcb *toep) +{ + u_int dlen = PPOD_SIZE * npods; + u_int pm_addr = idx * PPOD_SIZE + ci->llimit; + u_int wr_len = roundup(sizeof(struct ulp_mem_io) + + sizeof(struct ulptx_idata) + dlen, 16); + struct ulp_mem_io *req; + struct ulptx_idata *idata; + struct pagepod *ppod; + u_int i; + struct wrqe *wr; + struct adapter *sc = toep->vi->pi->adapter; + + wr = alloc_wrqe(wr_len, toep->ctrlq); + if (wr == NULL) { + CXGBE_UNIMPLEMENTED("ppod_write_idata: alloc_wrqe failure"); + return (ENOMEM); + } + + req = wrtod(wr); + memset(req, 0, wr_len); + ulp_mem_io_set_hdr(sc, toep->tid, req, wr_len, dlen, pm_addr); + idata = (struct ulptx_idata *)(req + 1); + + ppod = (struct pagepod *)(idata + 1); + for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES) { + if (!hdr) /* clear the pagepod */ + ppod_clear(ppod); + else /* set the pagepod */ + ppod_set(ppod, hdr, gl, gl_pidx); + } + + t4_wrq_tx(sc, wr); + return 0; +} + +int +t4_ddp_set_map(struct cxgbei_data *ci, void *iccp, + struct cxgbei_ulp2_pagepod_hdr *hdr, u_int idx, u_int npods, + struct cxgbei_ulp2_gather_list *gl, int reply) +{ + struct icl_cxgbei_conn *icc = (struct icl_cxgbei_conn *)iccp; + struct toepcb *toep = icc->toep; + int err; + unsigned int pidx = 0, w_npods = 0, cnt; + + /* + * on T4, if we use a mix of IMMD and DSGL with ULP_MEM_WRITE, + * the order would not be garanteed, so we will stick with IMMD + */ + gl->tid = toep->tid; + gl->port_id = toep->vi->pi->port_id; + gl->egress_dev = (void *)toep->vi->ifp; + + /* send via immediate data */ + for (; w_npods < npods; idx += cnt, w_npods += cnt, + pidx += PPOD_PAGES) { + cnt = npods - w_npods; + if (cnt > ULPMEM_IDATA_MAX_NPPODS) + cnt = ULPMEM_IDATA_MAX_NPPODS; + err = ppod_write_idata(ci, hdr, idx, cnt, gl, pidx, toep); + if (err) { + printf("%s: ppod_write_idata failed\n", __func__); + break; + } + } + return err; +} + +void +t4_ddp_clear_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl, + u_int tag, u_int idx, u_int npods, struct icl_cxgbei_conn *icc) +{ + struct toepcb *toep = icc->toep; + int err = -1; + u_int pidx = 0; + u_int w_npods = 0; + u_int cnt; + + for (; w_npods < npods; idx += cnt, w_npods += cnt, + pidx += PPOD_PAGES) { + cnt = npods - w_npods; + if (cnt > ULPMEM_IDATA_MAX_NPPODS) + cnt = ULPMEM_IDATA_MAX_NPPODS; + err = ppod_write_idata(ci, NULL, idx, cnt, gl, 0, toep); + if (err) + break; + } +} + +static int +cxgbei_map_sg(struct cxgbei_sgl *sgl, struct ccb_scsiio *csio) +{ + unsigned int data_len = csio->dxfer_len; + unsigned int sgoffset = (uint64_t)csio->data_ptr & PAGE_MASK; + unsigned int nsge; + unsigned char *sgaddr = csio->data_ptr; + unsigned int len = 0; + + nsge = (csio->dxfer_len + sgoffset + PAGE_SIZE - 1) >> PAGE_SHIFT; + sgl->sg_addr = sgaddr; + sgl->sg_offset = sgoffset; + if (data_len < (PAGE_SIZE - sgoffset)) + len = data_len; + else + len = PAGE_SIZE - sgoffset; + + sgl->sg_length = len; + + data_len -= len; + sgaddr += len; + sgl = sgl+1; + + while (data_len > 0) { + sgl->sg_addr = sgaddr; + len = (data_len < PAGE_SIZE)? data_len: PAGE_SIZE; + sgl->sg_length = len; + sgaddr += len; + data_len -= len; + sgl = sgl + 1; + } + + return nsge; +} + +static int +cxgbei_map_sg_tgt(struct cxgbei_sgl *sgl, union ctl_io *io) +{ + unsigned int data_len, sgoffset, nsge; + unsigned char *sgaddr; + unsigned int len = 0, index = 0, ctl_sg_count, i; + struct ctl_sg_entry ctl_sg_entry, *ctl_sglist; + + if (io->scsiio.kern_sg_entries > 0) { + ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr; + ctl_sg_count = io->scsiio.kern_sg_entries; + } else { + ctl_sglist = &ctl_sg_entry; + ctl_sglist->addr = io->scsiio.kern_data_ptr; + ctl_sglist->len = io->scsiio.kern_data_len; + ctl_sg_count = 1; + } + + sgaddr = sgl->sg_addr = ctl_sglist[index].addr; + sgoffset = sgl->sg_offset = (uint64_t)sgl->sg_addr & PAGE_MASK; + data_len = ctl_sglist[index].len; + + if (data_len < (PAGE_SIZE - sgoffset)) + len = data_len; + else + len = PAGE_SIZE - sgoffset; + + sgl->sg_length = len; + + data_len -= len; + sgaddr += len; + sgl = sgl+1; + + len = 0; + for (i = 0; i< ctl_sg_count; i++) + len += ctl_sglist[i].len; + nsge = (len + sgoffset + PAGE_SIZE -1) >> PAGE_SHIFT; + while (data_len > 0) { + sgl->sg_addr = sgaddr; + len = (data_len < PAGE_SIZE)? data_len: PAGE_SIZE; + sgl->sg_length = len; + sgaddr += len; + data_len -= len; + sgl = sgl + 1; + if (data_len == 0) { + if (index == ctl_sg_count - 1) + break; + index++; + sgaddr = ctl_sglist[index].addr; + data_len = ctl_sglist[index].len; + } + } + + return nsge; +} + +static int +t4_sk_ddp_tag_reserve(struct cxgbei_data *ci, struct icl_cxgbei_conn *icc, + u_int xferlen, struct cxgbei_sgl *sgl, u_int sgcnt, u_int *ddp_tag) +{ + struct cxgbei_ulp2_gather_list *gl; + int err = -EINVAL; + struct toepcb *toep = icc->toep; + + gl = cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(xferlen, sgl, sgcnt, ci, 0); + if (gl) { + err = cxgbei_ulp2_ddp_tag_reserve(ci, icc, toep->tid, + &ci->tag_format, ddp_tag, gl, 0, 0); + if (err) { + cxgbei_ulp2_ddp_release_gl(ci, gl); + } + } + + return err; +} + +static unsigned int +cxgbei_task_reserve_itt(struct icl_conn *ic, void **prv, + struct ccb_scsiio *scmd, unsigned int *itt) +{ + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + int xferlen = scmd->dxfer_len; + struct cxgbei_task_data *tdata = NULL; + struct cxgbei_sgl *sge = NULL; + struct toepcb *toep = icc->toep; + struct adapter *sc = td_adapter(toep->td); + struct cxgbei_data *ci = sc->iscsi_ulp_softc; + int err = -1; + + MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); + + tdata = (struct cxgbei_task_data *)*prv; + if (xferlen == 0 || tdata == NULL) + goto out; + if (xferlen < DDP_THRESHOLD) + goto out; + + if ((scmd->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { + tdata->nsge = cxgbei_map_sg(tdata->sgl, scmd); + if (tdata->nsge == 0) { + CTR1(KTR_CXGBE, "%s: map_sg failed", __func__); + return 0; + } + sge = tdata->sgl; + + tdata->sc_ddp_tag = *itt; + + CTR3(KTR_CXGBE, "%s: *itt:0x%x sc_ddp_tag:0x%x", + __func__, *itt, tdata->sc_ddp_tag); + if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, + tdata->sc_ddp_tag)) { + err = t4_sk_ddp_tag_reserve(ci, icc, scmd->dxfer_len, + sge, tdata->nsge, &tdata->sc_ddp_tag); + } else { + CTR3(KTR_CXGBE, + "%s: itt:0x%x sc_ddp_tag:0x%x not usable", + __func__, *itt, tdata->sc_ddp_tag); + } + } +out: + if (err < 0) + tdata->sc_ddp_tag = + cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *itt); + + return tdata->sc_ddp_tag; +} + +static unsigned int +cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io, + unsigned int *ttt) +{ + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + struct toepcb *toep = icc->toep; + struct adapter *sc = td_adapter(toep->td); + struct cxgbei_data *ci = sc->iscsi_ulp_softc; + struct cxgbei_task_data *tdata = NULL; + int xferlen, err = -1; + struct cxgbei_sgl *sge = NULL; + + MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); + + xferlen = (io->scsiio.kern_data_len - io->scsiio.ext_data_filled); + tdata = (struct cxgbei_task_data *)*prv; + if ((xferlen == 0) || (tdata == NULL)) + goto out; + if (xferlen < DDP_THRESHOLD) + goto out; + tdata->nsge = cxgbei_map_sg_tgt(tdata->sgl, io); + if (tdata->nsge == 0) { + CTR1(KTR_CXGBE, "%s: map_sg failed", __func__); + return 0; + } + sge = tdata->sgl; + + tdata->sc_ddp_tag = *ttt; + if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, tdata->sc_ddp_tag)) { + err = t4_sk_ddp_tag_reserve(ci, icc, xferlen, sge, + tdata->nsge, &tdata->sc_ddp_tag); + } else { + CTR2(KTR_CXGBE, "%s: sc_ddp_tag:0x%x not usable", + __func__, tdata->sc_ddp_tag); + } +out: + if (err < 0) + tdata->sc_ddp_tag = + cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *ttt); + return tdata->sc_ddp_tag; +} + +static int +t4_sk_ddp_tag_release(struct icl_cxgbei_conn *icc, unsigned int ddp_tag) +{ + struct toepcb *toep = icc->toep; + struct adapter *sc = td_adapter(toep->td); + struct cxgbei_data *ci = sc->iscsi_ulp_softc; + + cxgbei_ulp2_ddp_tag_release(ci, ddp_tag, icc); + + return (0); +} + +static int +cxgbei_ddp_init(struct adapter *sc, struct cxgbei_data *ci) +{ + int nppods, bits, max_sz, rc; + static const u_int pgsz_order[] = {0, 1, 2, 3}; + + MPASS(sc->vres.iscsi.size > 0); + + ci->llimit = sc->vres.iscsi.start; + ci->ulimit = sc->vres.iscsi.start + sc->vres.iscsi.size - 1; + max_sz = G_MAXRXDATA(t4_read_reg(sc, A_TP_PARA_REG2)); + + nppods = sc->vres.iscsi.size >> IPPOD_SIZE_SHIFT; + if (nppods <= 1024) + return (ENXIO); + + bits = fls(nppods); + if (bits > IPPOD_IDX_MAX_SIZE) + bits = IPPOD_IDX_MAX_SIZE; + nppods = (1 << (bits - 1)) - 1; + + rc = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR, + BUS_SPACE_MAXADDR, NULL, NULL, UINT32_MAX , 8, BUS_SPACE_MAXSIZE, + BUS_DMA_ALLOCNOW, NULL, NULL, &ci->ulp_ddp_tag); + if (rc != 0) { + device_printf(sc->dev, "%s: failed to create DMA tag: %u.\n", + __func__, rc); + return (rc); + } + + ci->colors = malloc(nppods * sizeof(char), M_CXGBE, M_NOWAIT | M_ZERO); + ci->gl_map = malloc(nppods * sizeof(struct cxgbei_ulp2_gather_list *), + M_CXGBE, M_NOWAIT | M_ZERO); + if (ci->colors == NULL || ci->gl_map == NULL) { + bus_dma_tag_destroy(ci->ulp_ddp_tag); + free(ci->colors, M_CXGBE); + free(ci->gl_map, M_CXGBE); + return (ENOMEM); + } + + mtx_init(&ci->map_lock, "ddp lock", NULL, MTX_DEF | MTX_DUPOK); + ci->max_txsz = ci->max_rxsz = min(max_sz, ULP2_MAX_PKT_SIZE); + ci->nppods = nppods; + ci->idx_last = nppods; + ci->idx_bits = bits; + ci->idx_mask = (1 << bits) - 1; + ci->rsvd_tag_mask = (1 << (bits + IPPOD_IDX_SHIFT)) - 1; + + ci->tag_format.sw_bits = bits; + ci->tag_format.rsvd_bits = bits; + ci->tag_format.rsvd_shift = IPPOD_IDX_SHIFT; + ci->tag_format.rsvd_mask = ci->idx_mask; + + t4_iscsi_init(sc, ci->idx_mask << IPPOD_IDX_SHIFT, pgsz_order); + + return (rc); +} + +static int +do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +{ + struct adapter *sc = iq->adapter; + struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *); + u_int tid = GET_TID(cpl); + struct toepcb *toep = lookup_tid(sc, tid); + struct icl_pdu *ip; + struct icl_cxgbei_pdu *icp; + + M_ASSERTPKTHDR(m); + + ip = icl_cxgbei_new_pdu(M_NOWAIT); + if (ip == NULL) + CXGBE_UNIMPLEMENTED("PDU allocation failure"); + icp = ip_to_icp(ip); + bcopy(mtod(m, caddr_t) + sizeof(*cpl), icp->ip.ip_bhs, sizeof(struct + iscsi_bhs)); + icp->pdu_seq = ntohl(cpl->seq); + icp->pdu_flags = SBUF_ULP_FLAG_HDR_RCVD; + + /* This is the start of a new PDU. There should be no old state. */ + MPASS(toep->ulpcb2 == NULL); + toep->ulpcb2 = icp; + +#if 0 + CTR4(KTR_CXGBE, "%s: tid %u, cpl->len hlen %u, m->m_len hlen %u", + __func__, tid, ntohs(cpl->len), m->m_len); +#endif + + m_freem(m); + return (0); +} + +static int +do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +{ + struct adapter *sc = iq->adapter; + struct cpl_iscsi_data *cpl = mtod(m, struct cpl_iscsi_data *); + u_int tid = GET_TID(cpl); + struct toepcb *toep = lookup_tid(sc, tid); + struct icl_cxgbei_pdu *icp = toep->ulpcb2; + + M_ASSERTPKTHDR(m); + + /* Must already have received the header (but not the data). */ + MPASS(icp != NULL); + MPASS(icp->pdu_flags == SBUF_ULP_FLAG_HDR_RCVD); + MPASS(icp->ip.ip_data_mbuf == NULL); + MPASS(icp->ip.ip_data_len == 0); + + m_adj(m, sizeof(*cpl)); + + icp->pdu_flags |= SBUF_ULP_FLAG_DATA_RCVD; + icp->ip.ip_data_mbuf = m; + icp->ip.ip_data_len = m->m_pkthdr.len; + +#if 0 + CTR4(KTR_CXGBE, "%s: tid %u, cpl->len dlen %u, m->m_len dlen %u", + __func__, tid, ntohs(cpl->len), m->m_len); +#endif + + return (0); +} + +static int +do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +{ + struct adapter *sc = iq->adapter; + const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1); + u_int tid = GET_TID(cpl); + struct toepcb *toep = lookup_tid(sc, tid); + struct inpcb *inp = toep->inp; + struct socket *so; + struct sockbuf *sb; + struct tcpcb *tp; + struct icl_cxgbei_conn *icc; + struct icl_conn *ic; + struct icl_cxgbei_pdu *icp = toep->ulpcb2; + struct icl_pdu *ip; + u_int pdu_len, val; + + MPASS(m == NULL); + + /* Must already be assembling a PDU. */ + MPASS(icp != NULL); + MPASS(icp->pdu_flags & SBUF_ULP_FLAG_HDR_RCVD); /* Data is optional. */ + ip = &icp->ip; + icp->pdu_flags |= SBUF_ULP_FLAG_STATUS_RCVD; + val = ntohl(cpl->ddpvld); + if (val & F_DDP_PADDING_ERR) + icp->pdu_flags |= SBUF_ULP_FLAG_PAD_ERROR; + if (val & F_DDP_HDRCRC_ERR) + icp->pdu_flags |= SBUF_ULP_FLAG_HCRC_ERROR; + if (val & F_DDP_DATACRC_ERR) + icp->pdu_flags |= SBUF_ULP_FLAG_DCRC_ERROR; + if (ip->ip_data_mbuf == NULL) { + /* XXXNP: what should ip->ip_data_len be, and why? */ + icp->pdu_flags |= SBUF_ULP_FLAG_DATA_DDPED; + } + pdu_len = ntohs(cpl->len); /* includes everything. */ + + INP_WLOCK(inp); + if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) { + CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", + __func__, tid, pdu_len, inp->inp_flags); + INP_WUNLOCK(inp); + icl_cxgbei_conn_pdu_free(NULL, ip); +#ifdef INVARIANTS + toep->ulpcb2 = NULL; +#endif + return (0); + } + + tp = intotcpcb(inp); + MPASS(icp->pdu_seq == tp->rcv_nxt); + MPASS(tp->rcv_wnd >= pdu_len); + tp->rcv_nxt += pdu_len; + tp->rcv_wnd -= pdu_len; + tp->t_rcvtime = ticks; + + /* update rx credits */ + toep->rx_credits += pdu_len; + t4_rcvd(&toep->td->tod, tp); /* XXX: sc->tom_softc.tod */ + + so = inp->inp_socket; + sb = &so->so_rcv; + SOCKBUF_LOCK(sb); + + icc = toep->ulpcb; + if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) { + CTR5(KTR_CXGBE, + "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x", + __func__, tid, pdu_len, icc, sb->sb_state); + SOCKBUF_UNLOCK(sb); + INP_WUNLOCK(inp); + + INP_INFO_RLOCK(&V_tcbinfo); + INP_WLOCK(inp); + tp = tcp_drop(tp, ECONNRESET); + if (tp) + INP_WUNLOCK(inp); + INP_INFO_RUNLOCK(&V_tcbinfo); + + icl_cxgbei_conn_pdu_free(NULL, ip); +#ifdef INVARIANTS + toep->ulpcb2 = NULL; +#endif + return (0); + } + MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); + ic = &icc->ic; + icl_cxgbei_new_pdu_set_conn(ip, ic); + + MPASS(m == NULL); /* was unused, we'll use it now. */ + m = sbcut_locked(sb, sbused(sb)); /* XXXNP: toep->sb_cc accounting? */ + if (__predict_false(m != NULL)) { + int len = m_length(m, NULL); + + /* + * PDUs were received before the tid transitioned to ULP mode. + * Convert them to icl_cxgbei_pdus and send them to ICL before + * the PDU in icp/ip. + */ + CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, tid, + len); + + /* XXXNP: needs to be rewritten. */ + if (len == sizeof(struct iscsi_bhs) || len == 4 + sizeof(struct + iscsi_bhs)) { + struct icl_cxgbei_pdu *icp0; + struct icl_pdu *ip0; + + ip0 = icl_cxgbei_new_pdu(M_NOWAIT); + icl_cxgbei_new_pdu_set_conn(ip0, ic); + if (ip0 == NULL) + CXGBE_UNIMPLEMENTED("PDU allocation failure"); + icp0 = ip_to_icp(ip0); + icp0->pdu_seq = 0; /* XXX */ + icp0->pdu_flags = SBUF_ULP_FLAG_HDR_RCVD | + SBUF_ULP_FLAG_STATUS_RCVD; + m_copydata(m, 0, sizeof(struct iscsi_bhs), (void *)ip0->ip_bhs); + STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip0, ip_next); + } + m_freem(m); + } + +#if 0 + CTR4(KTR_CXGBE, "%s: tid %u, pdu_len %u, pdu_flags 0x%x", + __func__, tid, pdu_len, icp->pdu_flags); +#endif + + STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next); + if ((icc->rx_flags & RXF_ACTIVE) == 0) { + struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt]; + + mtx_lock(&cwt->cwt_lock); + icc->rx_flags |= RXF_ACTIVE; + TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link); + if (cwt->cwt_state == CWT_SLEEPING) { + cwt->cwt_state = CWT_RUNNING; + cv_signal(&cwt->cwt_cv); + } + mtx_unlock(&cwt->cwt_lock); + } + SOCKBUF_UNLOCK(sb); + INP_WUNLOCK(inp); + +#ifdef INVARIANTS + toep->ulpcb2 = NULL; +#endif + + return (0); +} + +static void +t4_register_cpl_handler_with_tom(struct adapter *sc) +{ + + t4_register_cpl_handler(sc, CPL_ISCSI_HDR, do_rx_iscsi_hdr); + t4_register_cpl_handler(sc, CPL_ISCSI_DATA, do_rx_iscsi_data); + t4_register_cpl_handler(sc, CPL_RX_ISCSI_DDP, do_rx_iscsi_ddp); +} + +static void +t4_unregister_cpl_handler_with_tom(struct adapter *sc) +{ + + t4_register_cpl_handler(sc, CPL_ISCSI_HDR, NULL); + t4_register_cpl_handler(sc, CPL_ISCSI_DATA, NULL); + t4_register_cpl_handler(sc, CPL_RX_ISCSI_DDP, NULL); +} + +/* initiator */ +void +cxgbei_conn_task_reserve_itt(void *conn, void **prv, + void *scmd, unsigned int *itt) +{ + unsigned int tag; + tag = cxgbei_task_reserve_itt(conn, prv, scmd, itt); + if (tag) + *itt = htonl(tag); + return; +} + +/* target */ +void +cxgbei_conn_transfer_reserve_ttt(void *conn, void **prv, + void *scmd, unsigned int *ttt) +{ + unsigned int tag; + tag = cxgbei_task_reserve_ttt(conn, prv, scmd, ttt); + if (tag) + *ttt = htonl(tag); + return; +} + +void +cxgbei_cleanup_task(void *conn, void *ofld_priv) +{ + struct icl_conn *ic = (struct icl_conn *)conn; + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + struct cxgbei_task_data *tdata = ofld_priv; + struct adapter *sc = icc->sc; + struct cxgbei_data *ci = sc->iscsi_ulp_softc; + + MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); + MPASS(tdata != NULL); + + if (cxgbei_ulp2_is_ddp_tag(&ci->tag_format, tdata->sc_ddp_tag)) + t4_sk_ddp_tag_release(icc, tdata->sc_ddp_tag); + memset(tdata, 0, sizeof(*tdata)); +} + +static int +cxgbei_activate(struct adapter *sc) +{ + struct cxgbei_data *ci; + int rc; + + ASSERT_SYNCHRONIZED_OP(sc); + + if (uld_active(sc, ULD_ISCSI)) { + KASSERT(0, ("%s: iSCSI offload already enabled on adapter %p", + __func__, sc)); + return (0); + } + + if (sc->iscsicaps == 0 || sc->vres.iscsi.size == 0) { + device_printf(sc->dev, + "not iSCSI offload capable, or capability disabled.\n"); + return (ENOSYS); + } + + /* per-adapter softc for iSCSI */ + ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_NOWAIT); + if (ci == NULL) + return (ENOMEM); + + rc = cxgbei_ddp_init(sc, ci); + if (rc != 0) { + free(ci, M_CXGBE); + return (rc); + } + + t4_register_cpl_handler_with_tom(sc); + sc->iscsi_ulp_softc = ci; + + return (0); +} + +static int +cxgbei_deactivate(struct adapter *sc) +{ + + ASSERT_SYNCHRONIZED_OP(sc); + + if (sc->iscsi_ulp_softc != NULL) { + cxgbei_ddp_cleanup(sc->iscsi_ulp_softc); + t4_unregister_cpl_handler_with_tom(sc); + free(sc->iscsi_ulp_softc, M_CXGBE); + sc->iscsi_ulp_softc = NULL; + } + + return (0); +} + +static void +cxgbei_activate_all(struct adapter *sc, void *arg __unused) +{ + + if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isact") != 0) + return; + + /* Activate iSCSI if any port on this adapter has IFCAP_TOE enabled. */ + if (sc->offload_map && !uld_active(sc, ULD_ISCSI)) + (void) t4_activate_uld(sc, ULD_ISCSI); + + end_synchronized_op(sc, 0); +} + +static void +cxgbei_deactivate_all(struct adapter *sc, void *arg __unused) +{ + + if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isdea") != 0) + return; + + if (uld_active(sc, ULD_ISCSI)) + (void) t4_deactivate_uld(sc, ULD_ISCSI); + + end_synchronized_op(sc, 0); +} + +static struct uld_info cxgbei_uld_info = { + .uld_id = ULD_ISCSI, + .activate = cxgbei_activate, + .deactivate = cxgbei_deactivate, +}; + +static void +cwt_main(void *arg) +{ + struct cxgbei_worker_thread_softc *cwt = arg; + struct icl_cxgbei_conn *icc = NULL; + struct icl_conn *ic; + struct icl_pdu *ip; + struct sockbuf *sb; + STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus); + + MPASS(cwt != NULL); + + mtx_lock(&cwt->cwt_lock); + MPASS(cwt->cwt_state == 0); + cwt->cwt_state = CWT_RUNNING; + cv_signal(&cwt->cwt_cv); + + while (__predict_true(cwt->cwt_state != CWT_STOP)) { + cwt->cwt_state = CWT_RUNNING; + while ((icc = TAILQ_FIRST(&cwt->rx_head)) != NULL) { + TAILQ_REMOVE(&cwt->rx_head, icc, rx_link); + mtx_unlock(&cwt->cwt_lock); + + ic = &icc->ic; + sb = &ic->ic_socket->so_rcv; + + SOCKBUF_LOCK(sb); + MPASS(icc->rx_flags & RXF_ACTIVE); + if (__predict_true(!(sb->sb_state & SBS_CANTRCVMORE))) { + MPASS(STAILQ_EMPTY(&rx_pdus)); + STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu); + SOCKBUF_UNLOCK(sb); + + /* Hand over PDUs to ICL. */ + while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) { + STAILQ_REMOVE_HEAD(&rx_pdus, ip_next); + ic->ic_receive(ip); + } + + SOCKBUF_LOCK(sb); + MPASS(STAILQ_EMPTY(&rx_pdus)); + } + MPASS(icc->rx_flags & RXF_ACTIVE); + if (STAILQ_EMPTY(&icc->rcvd_pdus) || + __predict_false(sb->sb_state & SBS_CANTRCVMORE)) { + icc->rx_flags &= ~RXF_ACTIVE; + } else { + /* + * More PDUs were received while we were busy + * handing over the previous batch to ICL. + * Re-add this connection to the end of the + * queue. + */ + mtx_lock(&cwt->cwt_lock); + TAILQ_INSERT_TAIL(&cwt->rx_head, icc, + rx_link); + mtx_unlock(&cwt->cwt_lock); + } + SOCKBUF_UNLOCK(sb); + + mtx_lock(&cwt->cwt_lock); + } + + /* Inner loop doesn't check for CWT_STOP, do that first. */ + if (__predict_false(cwt->cwt_state == CWT_STOP)) + break; + cwt->cwt_state = CWT_SLEEPING; + cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); + } + + MPASS(TAILQ_FIRST(&cwt->rx_head) == NULL); + mtx_assert(&cwt->cwt_lock, MA_OWNED); + cwt->cwt_state = CWT_STOPPED; + cv_signal(&cwt->cwt_cv); + mtx_unlock(&cwt->cwt_lock); + kthread_exit(); +} + +static int +start_worker_threads(void) +{ + int i, rc; + struct cxgbei_worker_thread_softc *cwt; + + worker_thread_count = min(mp_ncpus, 32); + cwt_softc = malloc(worker_thread_count * sizeof(*cwt), M_CXGBE, + M_WAITOK | M_ZERO); + + MPASS(cxgbei_proc == NULL); + for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) { + mtx_init(&cwt->cwt_lock, "cwt lock", NULL, MTX_DEF); + cv_init(&cwt->cwt_cv, "cwt cv"); + TAILQ_INIT(&cwt->rx_head); + rc = kproc_kthread_add(cwt_main, cwt, &cxgbei_proc, NULL, 0, 0, + "cxgbei", "%d", i); + if (rc != 0) { + printf("cxgbei: failed to start thread #%d/%d (%d)\n", + i + 1, worker_thread_count, rc); + mtx_destroy(&cwt->cwt_lock); + cv_destroy(&cwt->cwt_cv); + bzero(&cwt, sizeof(*cwt)); + if (i == 0) { + free(cwt_softc, M_CXGBE); + worker_thread_count = 0; + + return (rc); + } + + /* Not fatal, carry on with fewer threads. */ + worker_thread_count = i; + rc = 0; + break; + } + + /* Wait for thread to start before moving on to the next one. */ + mtx_lock(&cwt->cwt_lock); + while (cwt->cwt_state == 0) + cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); + mtx_unlock(&cwt->cwt_lock); + } + + MPASS(cwt_softc != NULL); + MPASS(worker_thread_count > 0); + return (0); +} + +static void +stop_worker_threads(void) +{ + int i; + struct cxgbei_worker_thread_softc *cwt = &cwt_softc[0]; + + MPASS(worker_thread_count >= 0); + + for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) { + mtx_lock(&cwt->cwt_lock); + MPASS(cwt->cwt_state == CWT_RUNNING || + cwt->cwt_state == CWT_SLEEPING); + cwt->cwt_state = CWT_STOP; + cv_signal(&cwt->cwt_cv); + do { + cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); + } while (cwt->cwt_state != CWT_STOPPED); + mtx_unlock(&cwt->cwt_lock); + } + free(cwt_softc, M_CXGBE); +} + +/* Select a worker thread for a connection. */ +u_int +cxgbei_select_worker_thread(struct icl_cxgbei_conn *icc) +{ + struct adapter *sc = icc->sc; + struct toepcb *toep = icc->toep; + u_int i, n; + + n = worker_thread_count / sc->sge.nofldrxq; + if (n > 0) + i = toep->vi->pi->port_id * n + arc4random() % n; + else + i = arc4random() % worker_thread_count; + + CTR3(KTR_CXGBE, "%s: tid %u, cwt %u", __func__, toep->tid, i); + + return (i); +} + +static int +cxgbei_mod_load(void) +{ + int rc; + + rc = start_worker_threads(); + if (rc != 0) + return (rc); + + rc = t4_register_uld(&cxgbei_uld_info); + if (rc != 0) { + stop_worker_threads(); + return (rc); + } + + t4_iterate(cxgbei_activate_all, NULL); + + return (rc); +} + +static int +cxgbei_mod_unload(void) +{ + + t4_iterate(cxgbei_deactivate_all, NULL); + + if (t4_unregister_uld(&cxgbei_uld_info) == EBUSY) + return (EBUSY); + + stop_worker_threads(); + + return (0); +} +#endif + +static int +cxgbei_modevent(module_t mod, int cmd, void *arg) +{ + int rc = 0; + +#ifdef TCP_OFFLOAD + switch (cmd) { + case MOD_LOAD: + rc = cxgbei_mod_load(); + break; + + case MOD_UNLOAD: + rc = cxgbei_mod_unload(); + break; + + default: + rc = EINVAL; + } +#else + printf("cxgbei: compiled without TCP_OFFLOAD support.\n"); + rc = EOPNOTSUPP; +#endif + + return (rc); +} + +static moduledata_t cxgbei_mod = { + "cxgbei", + cxgbei_modevent, + NULL, +}; + +MODULE_VERSION(cxgbei, 1); +DECLARE_MODULE(cxgbei, cxgbei_mod, SI_SUB_EXEC, SI_ORDER_ANY); +MODULE_DEPEND(cxgbei, t4_tom, 1, 1, 1); +MODULE_DEPEND(cxgbei, cxgbe, 1, 1, 1); +MODULE_DEPEND(cxgbei, icl, 1, 1, 1); Property changes on: head/sys/dev/cxgbe/cxgbei/cxgbei.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/cxgbe/cxgbei/cxgbei.h =================================================================== --- head/sys/dev/cxgbe/cxgbei/cxgbei.h (nonexistent) +++ head/sys/dev/cxgbe/cxgbei/cxgbei.h (revision 292740) @@ -0,0 +1,167 @@ +/*- + * Copyright (c) 2012, 2015 Chelsio Communications, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef __CXGBEI_OFLD_H__ +#define __CXGBEI_OFLD_H__ + +#include + +enum { + CWT_SLEEPING = 1, + CWT_RUNNING = 2, + CWT_STOP = 3, + CWT_STOPPED = 4, +}; + +struct cxgbei_worker_thread_softc { + struct mtx cwt_lock; + struct cv cwt_cv; + volatile int cwt_state; + + TAILQ_HEAD(, icl_cxgbei_conn) rx_head; +} __aligned(CACHE_LINE_SIZE); + +#define CXGBEI_CONN_SIGNATURE 0x56788765 + +enum { + RXF_ACTIVE = 1 << 0, /* In the worker thread's queue */ +}; + +struct icl_cxgbei_conn { + struct icl_conn ic; + + /* cxgbei specific stuff goes here. */ + uint32_t icc_signature; + int ulp_submode; + struct adapter *sc; + struct toepcb *toep; + + /* Receive related. */ + u_int rx_flags; /* protected by so_rcv lock */ + u_int cwt; + STAILQ_HEAD(, icl_pdu) rcvd_pdus; /* protected by so_rcv lock */ + TAILQ_ENTRY(icl_cxgbei_conn) rx_link; /* protected by cwt lock */ +}; + +static inline struct icl_cxgbei_conn * +ic_to_icc(struct icl_conn *ic) +{ + + return (__containerof(ic, struct icl_cxgbei_conn, ic)); +} + +#define CXGBEI_PDU_SIGNATURE 0x12344321 + +struct icl_cxgbei_pdu { + struct icl_pdu ip; + + /* cxgbei specific stuff goes here. */ + uint32_t icp_signature; + uint32_t pdu_seq; /* For debug only */ + u_int pdu_flags; +}; + +static inline struct icl_cxgbei_pdu * +ip_to_icp(struct icl_pdu *ip) +{ + + return (__containerof(ip, struct icl_cxgbei_pdu, ip)); +} + +struct cxgbei_sgl { + int sg_flag; + void *sg_addr; + void *sg_dma_addr; + size_t sg_offset; + size_t sg_length; +}; + +#define cxgbei_scsi_for_each_sg(_sgl, _sgel, _n, _i) \ + for (_i = 0, _sgel = (cxgbei_sgl*) (_sgl); _i < _n; _i++, \ + _sgel++) +#define sg_dma_addr(_sgel) _sgel->sg_dma_addr +#define sg_virt(_sgel) _sgel->sg_addr +#define sg_len(_sgel) _sgel->sg_length +#define sg_off(_sgel) _sgel->sg_offset +#define sg_next(_sgel) _sgel + 1 + +#define SBUF_ULP_FLAG_HDR_RCVD 0x1 +#define SBUF_ULP_FLAG_DATA_RCVD 0x2 +#define SBUF_ULP_FLAG_STATUS_RCVD 0x4 +#define SBUF_ULP_FLAG_HCRC_ERROR 0x10 +#define SBUF_ULP_FLAG_DCRC_ERROR 0x20 +#define SBUF_ULP_FLAG_PAD_ERROR 0x40 +#define SBUF_ULP_FLAG_DATA_DDPED 0x80 + +/* private data for each scsi task */ +struct cxgbei_task_data { + struct cxgbei_sgl sgl[256]; + u_int nsge; + u_int sc_ddp_tag; +}; + +struct cxgbei_ulp2_tag_format { + u_char sw_bits; + u_char rsvd_bits; + u_char rsvd_shift; + u_char filler[1]; + uint32_t rsvd_mask; +}; + +struct cxgbei_data { + u_int max_txsz; + u_int max_rxsz; + u_int llimit; + u_int ulimit; + u_int nppods; + u_int idx_last; + u_char idx_bits; + uint32_t idx_mask; + uint32_t rsvd_tag_mask; + + struct mtx map_lock; + bus_dma_tag_t ulp_ddp_tag; + unsigned char *colors; + struct cxgbei_ulp2_gather_list **gl_map; + + struct cxgbei_ulp2_tag_format tag_format; +}; + +void cxgbei_conn_task_reserve_itt(void *, void **, void *, unsigned int *); +void cxgbei_conn_transfer_reserve_ttt(void *, void **, void *, unsigned int *); +void cxgbei_cleanup_task(void *, void *); +u_int cxgbei_select_worker_thread(struct icl_cxgbei_conn *); + +struct cxgbei_ulp2_pagepod_hdr; +int t4_ddp_set_map(struct cxgbei_data *, void *, + struct cxgbei_ulp2_pagepod_hdr *, u_int, u_int, + struct cxgbei_ulp2_gather_list *, int); +void t4_ddp_clear_map(struct cxgbei_data *, struct cxgbei_ulp2_gather_list *, + u_int, u_int, u_int, struct icl_cxgbei_conn *); +#endif Property changes on: head/sys/dev/cxgbe/cxgbei/cxgbei.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c =================================================================== --- head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c (nonexistent) +++ head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c (revision 292740) @@ -0,0 +1,417 @@ +/*- + * Copyright (c) 2012 Chelsio Communications, Inc. + * All rights reserved. + * + * Chelsio T5xx iSCSI driver + * cxgbei_ulp2_ddp.c: Chelsio iSCSI DDP Manager. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#ifdef TCP_OFFLOAD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "common/common.h" +#include "common/t4_msg.h" +#include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */ +#include "tom/t4_tom.h" +#include "cxgbei.h" +#include "cxgbei_ulp2_ddp.h" + +/* + * Map a single buffer address. + */ +static void +ulp2_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + bus_addr_t *ba = arg; + if (error) + return; + + KASSERT(nseg == 1, ("%s: %d segments returned!", __func__, nseg)); + + *ba = segs->ds_addr; +} + +/* + * iSCSI Direct Data Placement + * + * T4/5 ulp2 h/w can directly place the iSCSI Data-In or Data-Out PDU's + * payload into pre-posted final destination host-memory buffers based on the + * Initiator Task Tag (ITT) in Data-In or Target Task Tag (TTT) in Data-Out + * PDUs. + * + * The host memory address is programmed into h/w in the format of pagepod + * entries. + * The location of the pagepod entry is encoded into ddp tag which is used or + * is the base for ITT/TTT. + */ + + +static inline int +ddp_find_unused_entries(struct cxgbei_data *ci, u_int start, u_int max, + u_int count, u_int *idx, struct cxgbei_ulp2_gather_list *gl) +{ + unsigned int i, j, k; + + /* not enough entries */ + if (max - start < count) + return (EBUSY); + + max -= count; + mtx_lock(&ci->map_lock); + for (i = start; i < max;) { + for (j = 0, k = i; j < count; j++, k++) { + if (ci->gl_map[k]) + break; + } + if (j == count) { + for (j = 0, k = i; j < count; j++, k++) + ci->gl_map[k] = gl; + mtx_unlock(&ci->map_lock); + *idx = i; + return (0); + } + i += j + 1; + } + mtx_unlock(&ci->map_lock); + return (EBUSY); +} + +static inline void +ddp_unmark_entries(struct cxgbei_data *ci, u_int start, u_int count) +{ + + mtx_lock(&ci->map_lock); + memset(&ci->gl_map[start], 0, + count * sizeof(struct cxgbei_ulp2_gather_list *)); + mtx_unlock(&ci->map_lock); +} + +static inline void +ddp_gl_unmap(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl) +{ + int i; + + if (!gl->pages[0]) + return; + + for (i = 0; i < gl->nelem; i++) { + bus_dmamap_unload(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map); + bus_dmamap_destroy(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map); + } +} + +static inline int +ddp_gl_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl) +{ + int i, rc; + bus_addr_t pa; + + MPASS(ci != NULL); + + mtx_lock(&ci->map_lock); + for (i = 0; i < gl->nelem; i++) { + rc = bus_dmamap_create(ci->ulp_ddp_tag, 0, + &gl->dma_sg[i].bus_map); + if (rc != 0) + goto unmap; + rc = bus_dmamap_load(ci->ulp_ddp_tag, gl->dma_sg[i].bus_map, + gl->pages[i], PAGE_SIZE, ulp2_dma_map_addr, + &pa, BUS_DMA_NOWAIT); + if (rc != 0) + goto unmap; + gl->dma_sg[i].phys_addr = pa; + } + mtx_unlock(&ci->map_lock); + + return (0); + +unmap: + if (i) { + u_int nelem = gl->nelem; + + gl->nelem = i; + ddp_gl_unmap(ci, gl); + gl->nelem = nelem; + } + return (ENOMEM); +} + +/** + * cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec - build ddp page buffer list + * @xferlen: total buffer length + * @sgl: page buffer scatter-gather list (struct cxgbei_sgl) + * @sgcnt: # of page buffers + * @gfp: allocation mode + * + * construct a ddp page buffer list from the scsi scattergather list. + * coalesce buffers as much as possible, and obtain dma addresses for + * each page. + * + * Return the cxgbei_ulp2_gather_list constructed from the page buffers if the + * memory can be used for ddp. Return NULL otherwise. + */ +struct cxgbei_ulp2_gather_list * +cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int xferlen, struct cxgbei_sgl *sgl, + u_int sgcnt, struct cxgbei_data *ci, int gfp) +{ + struct cxgbei_ulp2_gather_list *gl; + struct cxgbei_sgl *sg = sgl; + void *sgpage = (void *)((u64)sg->sg_addr & (~PAGE_MASK)); + unsigned int sglen = sg->sg_length; + unsigned int sgoffset = (u64)sg->sg_addr & PAGE_MASK; + unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >> + PAGE_SHIFT; + int i = 1, j = 0; + + if (xferlen <= DDP_THRESHOLD) { + CTR2(KTR_CXGBE, "xfer %u < threshold %u, no ddp.", + xferlen, DDP_THRESHOLD); + return NULL; + } + + gl = malloc(sizeof(struct cxgbei_ulp2_gather_list) + + npages * (sizeof(struct dma_segments) + sizeof(void *)), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (gl == NULL) + return (NULL); + + gl->pages = (void **)&gl->dma_sg[npages]; + gl->length = xferlen; + gl->offset = sgoffset; + gl->pages[0] = sgpage; + CTR6(KTR_CXGBE, + "%s: xferlen:0x%x len:0x%x off:0x%x sg_addr:%p npages:%d", + __func__, xferlen, gl->length, gl->offset, sg->sg_addr, npages); + + for (i = 1, sg = sg_next(sg); i < sgcnt; i++, sg = sg_next(sg)) { + void *page = sg->sg_addr; + + if (sgpage == page && sg->sg_offset == sgoffset + sglen) + sglen += sg->sg_length; + else { + /* make sure the sgl is fit for ddp: + * each has the same page size, and + * all of the middle pages are used completely + */ + if ((j && sgoffset) || + ((i != sgcnt - 1) && + ((sglen + sgoffset) & ~CXGBEI_PAGE_MASK))){ + goto error_out; + } + + j++; + if (j == gl->nelem || sg->sg_offset) { + goto error_out; + } + gl->pages[j] = page; + sglen = sg->sg_length; + sgoffset = sg->sg_offset; + sgpage = page; + } + } + gl->nelem = ++j; + + if (ddp_gl_map(ci, gl) < 0) + goto error_out; + + return gl; + +error_out: + free(gl, M_DEVBUF); + return NULL; +} + +/** + * cxgbei_ulp2_ddp_release_gl - release a page buffer list + * @gl: a ddp page buffer list + * @pdev: pci_dev used for pci_unmap + * free a ddp page buffer list resulted from cxgbei_ulp2_ddp_make_gl(). + */ +void +cxgbei_ulp2_ddp_release_gl(struct cxgbei_data *ci, + struct cxgbei_ulp2_gather_list *gl) +{ + + ddp_gl_unmap(ci, gl); + free(gl, M_DEVBUF); +} + +/** + * cxgbei_ulp2_ddp_tag_reserve - set up ddp for a data transfer + * @ci: adapter's ddp info + * @tid: connection id + * @tformat: tag format + * @tagp: contains s/w tag initially, will be updated with ddp/hw tag + * @gl: the page momory list + * @gfp: allocation mode + * + * ddp setup for a given page buffer list and construct the ddp tag. + * return 0 if success, < 0 otherwise. + */ +int +cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *ci, void *icc, u_int tid, + struct cxgbei_ulp2_tag_format *tformat, u32 *tagp, + struct cxgbei_ulp2_gather_list *gl, int gfp, int reply) +{ + struct cxgbei_ulp2_pagepod_hdr hdr; + u_int npods, idx; + int rc; + u32 sw_tag = *tagp; + u32 tag; + + MPASS(ci != NULL); + + if (!gl || !gl->nelem || gl->length < DDP_THRESHOLD) + return (EINVAL); + + npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT; + + if (ci->idx_last == ci->nppods) + rc = ddp_find_unused_entries(ci, 0, ci->nppods, npods, &idx, + gl); + else { + rc = ddp_find_unused_entries(ci, ci->idx_last + 1, + ci->nppods, npods, &idx, gl); + if (rc && ci->idx_last >= npods) { + rc = ddp_find_unused_entries(ci, 0, + min(ci->idx_last + npods, ci->nppods), + npods, &idx, gl); + } + } + if (rc) { + CTR3(KTR_CXGBE, "xferlen %u, gl %u, npods %u NO DDP.", + gl->length, gl->nelem, npods); + return (rc); + } + + tag = cxgbei_ulp2_ddp_tag_base(idx, ci->colors, tformat, sw_tag); + CTR4(KTR_CXGBE, "%s: sw_tag:0x%x idx:0x%x tag:0x%x", + __func__, sw_tag, idx, tag); + + hdr.rsvd = 0; + hdr.vld_tid = htonl(F_IPPOD_VALID | V_IPPOD_TID(tid)); + hdr.pgsz_tag_clr = htonl(tag & ci->rsvd_tag_mask); + hdr.maxoffset = htonl(gl->length); + hdr.pgoffset = htonl(gl->offset); + + rc = t4_ddp_set_map(ci, icc, &hdr, idx, npods, gl, reply); + if (rc < 0) + goto unmark_entries; + + ci->idx_last = idx; + *tagp = tag; + return (0); + +unmark_entries: + ddp_unmark_entries(ci, idx, npods); + return (rc); +} + +/** + * cxgbei_ulp2_ddp_tag_release - release a ddp tag + * @ci: adapter's ddp info + * @tag: ddp tag + * ddp cleanup for a given ddp tag and release all the resources held + */ +void +cxgbei_ulp2_ddp_tag_release(struct cxgbei_data *ci, uint32_t tag, + struct icl_cxgbei_conn *icc) +{ + uint32_t idx; + + MPASS(ci != NULL); + MPASS(icc != NULL); + + idx = (tag >> IPPOD_IDX_SHIFT) & ci->idx_mask; + CTR3(KTR_CXGBE, "tag:0x%x idx:0x%x nppods:0x%x", + tag, idx, ci->nppods); + if (idx < ci->nppods) { + struct cxgbei_ulp2_gather_list *gl = ci->gl_map[idx]; + unsigned int npods; + + if (!gl || !gl->nelem) { + CTR4(KTR_CXGBE, + "release 0x%x, idx 0x%x, gl 0x%p, %u.", + tag, idx, gl, gl ? gl->nelem : 0); + return; + } + npods = (gl->nelem + IPPOD_PAGES_MAX - 1) >> IPPOD_PAGES_SHIFT; + CTR3(KTR_CXGBE, "ddp tag 0x%x, release idx 0x%x, npods %u.", + tag, idx, npods); + t4_ddp_clear_map(ci, gl, tag, idx, npods, icc); + ddp_unmark_entries(ci, idx, npods); + cxgbei_ulp2_ddp_release_gl(ci, gl); + } else + CTR3(KTR_CXGBE, "ddp tag 0x%x, idx 0x%x > max 0x%x.", + tag, idx, ci->nppods); +} + +/** + * cxgbei_ddp_cleanup - release the adapter's ddp resources + */ +void +cxgbei_ddp_cleanup(struct cxgbei_data *ci) +{ + int i = 0; + + while (i < ci->nppods) { + struct cxgbei_ulp2_gather_list *gl = ci->gl_map[i]; + if (gl) { + int npods = (gl->nelem + IPPOD_PAGES_MAX - 1) + >> IPPOD_PAGES_SHIFT; + free(gl, M_DEVBUF); + i += npods; + } else + i++; + } + free(ci->colors, M_CXGBE); + free(ci->gl_map, M_CXGBE); +} +#endif Property changes on: head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h =================================================================== --- head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h (nonexistent) +++ head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h (revision 292740) @@ -0,0 +1,217 @@ +/*- + * Copyright (c) 2012 Chelsio Communications, Inc. + * All rights reserved. + * + * Chelsio T5xx iSCSI driver + * cxgbei_ulp2_ddp.c: Chelsio iSCSI DDP Manager. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef __CXGBEI_ULP2_DDP_H__ +#define __CXGBEI_ULP2_DDP_H__ + +#define CXGBEI_PAGE_MASK (~(PAGE_SIZE-1)) +#define DDP_THRESHOLD 2048 + +/* + * cxgbei ddp tag are 32 bits, it consists of reserved bits used by h/w and + * non-reserved bits that can be used by the iscsi s/w. + * The reserved bits are identified by the rsvd_bits and rsvd_shift fields + * in struct cxgbei_ulp2_tag_format. + * + * The upper most reserved bit can be used to check if a tag is ddp tag or not: + * if the bit is 0, the tag is a valid ddp tag + */ + +/* + * cxgbei_ulp2_is_ddp_tag - check if a given tag is a hw/ddp tag + * @tformat: tag format information + * @tag: tag to be checked + * + * return true if the tag is a ddp tag, false otherwise. + */ +static inline int +cxgbei_ulp2_is_ddp_tag(struct cxgbei_ulp2_tag_format *tformat, uint32_t tag) +{ + + return (!(tag & (1 << (tformat->rsvd_bits + tformat->rsvd_shift - 1)))); +} + +/* + * cxgbei_ulp2_sw_tag_usable - check if s/w tag has enough bits left for hw bits + * @tformat: tag format information + * @sw_tag: s/w tag to be checked + * + * return true if the tag can be used for hw ddp tag, false otherwise. + */ +static inline int +cxgbei_ulp2_sw_tag_usable(struct cxgbei_ulp2_tag_format *tformat, + uint32_t sw_tag) +{ + + return (1); /* XXXNP: huh? */ + + sw_tag >>= (32 - tformat->rsvd_bits + tformat->rsvd_shift); + return !sw_tag; +} + +/* + * cxgbei_ulp2_set_non_ddp_tag - mark a given s/w tag as an invalid ddp tag + * @tformat: tag format information + * @sw_tag: s/w tag to be checked + * + * insert 1 at the upper most reserved bit to mark it as an invalid ddp tag. + */ +static inline uint32_t +cxgbei_ulp2_set_non_ddp_tag(struct cxgbei_ulp2_tag_format *tformat, + uint32_t sw_tag) +{ + uint32_t rsvd_bits = tformat->rsvd_bits + tformat->rsvd_shift; + if (sw_tag) { + u32 v1 = sw_tag & ((1 << (rsvd_bits - 1)) - 1); + u32 v2 = (sw_tag >> (rsvd_bits - 1)) << rsvd_bits; + return v2 | (1 << (rsvd_bits - 1)) | v1; + } + + return sw_tag | (1 << (rsvd_bits - 1)) ; +} + +struct dma_segments { + bus_dmamap_t bus_map; + bus_addr_t phys_addr; +}; +/* + * struct cxgbei_ulp2_gather_list - cxgbei direct data placement memory + * + * @tag: ddp tag + * @length: total data buffer length + * @offset: initial offset to the 1st page + * @nelem: # of pages + * @pages: page pointers + * @phys_addr: physical address + */ +struct cxgbei_ulp2_gather_list { + uint32_t tag; + uint32_t tid; + uint32_t port_id; + void *egress_dev; + unsigned int length; + unsigned int offset; + unsigned int nelem; + bus_size_t mapsize; + bus_dmamap_t bus_map; + bus_dma_segment_t *segments; + void **pages; + struct dma_segments dma_sg[0]; +}; + +#define IPPOD_SIZE sizeof(struct cxgbei_ulp2_pagepod) /* 64 */ +#define IPPOD_SIZE_SHIFT 6 + +#define IPPOD_COLOR_SHIFT 0 +#define IPPOD_COLOR_SIZE 6 +#define IPPOD_COLOR_MASK ((1 << IPPOD_COLOR_SIZE) - 1) + +#define IPPOD_IDX_SHIFT IPPOD_COLOR_SIZE +#define IPPOD_IDX_MAX_SIZE 24 + +#define S_IPPOD_TID 0 +#define M_IPPOD_TID 0xFFFFFF +#define V_IPPOD_TID(x) ((x) << S_IPPOD_TID) + +#define S_IPPOD_VALID 24 +#define V_IPPOD_VALID(x) ((x) << S_IPPOD_VALID) +#define F_IPPOD_VALID V_IPPOD_VALID(1U) + +#define S_IPPOD_COLOR 0 +#define M_IPPOD_COLOR 0x3F +#define V_IPPOD_COLOR(x) ((x) << S_IPPOD_COLOR) + +#define S_IPPOD_TAG 6 +#define M_IPPOD_TAG 0xFFFFFF +#define V_IPPOD_TAG(x) ((x) << S_IPPOD_TAG) + +#define S_IPPOD_PGSZ 30 +#define M_IPPOD_PGSZ 0x3 +#define V_IPPOD_PGSZ(x) ((x) << S_IPPOD_PGSZ) + +static inline uint32_t +cxgbei_ulp2_ddp_tag_base(u_int idx, u_char *colors, + struct cxgbei_ulp2_tag_format *tformat, uint32_t sw_tag) +{ + if (__predict_false(++colors[idx] == 1 << IPPOD_IDX_SHIFT)) + colors[idx] = 0; + + sw_tag <<= tformat->rsvd_bits + tformat->rsvd_shift; + + return (sw_tag | idx << IPPOD_IDX_SHIFT | colors[idx]); +} + +#define ISCSI_PDU_NONPAYLOAD_LEN 312 /* bhs(48) + ahs(256) + digest(8) */ + +/* + * align pdu size to multiple of 512 for better performance + */ +#define cxgbei_align_pdu_size(n) do { n = (n) & (~511); } while (0) + +#define ULP2_MAX_PKT_SIZE 16224 +#define ULP2_MAX_PDU_PAYLOAD (ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_LEN) +#define IPPOD_PAGES_MAX 4 +#define IPPOD_PAGES_SHIFT 2 /* 4 pages per pod */ + +/* + * struct pagepod_hdr, pagepod - pagepod format + */ +struct cxgbei_ulp2_pagepod_hdr { + uint32_t vld_tid; + uint32_t pgsz_tag_clr; + uint32_t maxoffset; + uint32_t pgoffset; + uint64_t rsvd; +}; + +struct cxgbei_ulp2_pagepod { + struct cxgbei_ulp2_pagepod_hdr hdr; + uint64_t addr[IPPOD_PAGES_MAX + 1]; +}; + +int cxgbei_ulp2_ddp_tag_reserve(struct cxgbei_data *, void *, unsigned int, + struct cxgbei_ulp2_tag_format *, uint32_t *, + struct cxgbei_ulp2_gather_list *, int , int ); +void cxgbei_ulp2_ddp_tag_release(struct cxgbei_data *, uint32_t, + struct icl_cxgbei_conn *); + +struct cxgbei_ulp2_gather_list *cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(u_int, + struct cxgbei_sgl *, u_int, struct cxgbei_data *, int); +void cxgbei_ulp2_ddp_release_gl(struct cxgbei_data *, + struct cxgbei_ulp2_gather_list *); + +int cxgbei_ulp2_ddp_find_page_index(u_long); +int cxgbei_ulp2_adapter_ddp_info(struct cxgbei_data *, + struct cxgbei_ulp2_tag_format *); + +void cxgbei_ddp_cleanup(struct cxgbei_data *); +#endif Property changes on: head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c =================================================================== --- head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c (nonexistent) +++ head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c (revision 292740) @@ -0,0 +1,896 @@ +/*- + * Copyright (c) 2012 The FreeBSD Foundation + * Copyright (c) 2015 Chelsio Communications, Inc. + * All rights reserved. + * + * This software was developed by Edward Tomasz Napierala under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/* + * cxgbei implementation of iSCSI Common Layer kobj(9) interface. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#ifdef TCP_OFFLOAD +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "common/common.h" +#include "tom/t4_tom.h" +#include "cxgbei.h" + +SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD, 0, "Chelsio iSCSI offload"); +static int coalesce = 1; +SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, coalesce, CTLFLAG_RWTUN, + &coalesce, 0, "Try to coalesce PDUs before sending"); +static int partial_receive_len = 128 * 1024; +SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, + &partial_receive_len, 0, "Minimum read size for partially received " + "data segment"); +static int sendspace = 1048576; +SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN, + &sendspace, 0, "Default send socket buffer size"); +static int recvspace = 1048576; +SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN, + &recvspace, 0, "Default receive socket buffer size"); + +static uma_zone_t icl_transfer_zone; + +static volatile u_int icl_cxgbei_ncons; + +#define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) +#define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) +#define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) +#define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) + +struct icl_pdu *icl_cxgbei_new_pdu(int); +void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *); + +static icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu; +icl_conn_pdu_free_t icl_cxgbei_conn_pdu_free; +static icl_conn_pdu_data_segment_length_t + icl_cxgbei_conn_pdu_data_segment_length; +static icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data; +static icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data; +static icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue; +static icl_conn_handoff_t icl_cxgbei_conn_handoff; +static icl_conn_free_t icl_cxgbei_conn_free; +static icl_conn_close_t icl_cxgbei_conn_close; +static icl_conn_task_setup_t icl_cxgbei_conn_task_setup; +static icl_conn_task_done_t icl_cxgbei_conn_task_done; +static icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup; +static icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done; + +static kobj_method_t icl_cxgbei_methods[] = { + KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu), + KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free), + KOBJMETHOD(icl_conn_pdu_data_segment_length, + icl_cxgbei_conn_pdu_data_segment_length), + KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data), + KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data), + KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue), + KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff), + KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free), + KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close), + KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup), + KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done), + KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup), + KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done), + { 0, 0 } +}; + +DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn)); + +#if 0 +/* + * Subtract another 256 for AHS from MAX_DSL if AHS could be used. + */ +#define CXGBEI_MAX_PDU 16224 +#define CXGBEI_MAX_DSL (CXGBEI_MAX_PDU - sizeof(struct iscsi_bhs) - 8) +#endif +#define CXGBEI_MAX_DSL 8192 +#define CXGBEI_MAX_PDU (CXGBEI_MAX_DSL + sizeof(struct iscsi_bhs) + 8) + +void +icl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) +{ +#ifdef INVARIANTS + struct icl_cxgbei_pdu *icp = ip_to_icp(ip); +#endif + + MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); + MPASS(ic == ip->ip_conn); + MPASS(ip->ip_bhs_mbuf != NULL); + + m_freem(ip->ip_ahs_mbuf); + m_freem(ip->ip_data_mbuf); + m_freem(ip->ip_bhs_mbuf); /* storage for icl_cxgbei_pdu itself */ + +#ifdef DIAGNOSTIC + if (__predict_true(ic != NULL)) + refcount_release(&ic->ic_outstanding_pdus); +#endif +} + +struct icl_pdu * +icl_cxgbei_new_pdu(int flags) +{ + struct icl_cxgbei_pdu *icp; + struct icl_pdu *ip; + struct mbuf *m; + uintptr_t a; + + m = m_gethdr(flags, MT_DATA); + if (__predict_false(m == NULL)) + return (NULL); + + a = roundup2(mtod(m, uintptr_t), _Alignof(struct icl_cxgbei_pdu)); + icp = (struct icl_cxgbei_pdu *)a; + bzero(icp, sizeof(*icp)); + + icp->icp_signature = CXGBEI_PDU_SIGNATURE; + ip = &icp->ip; + ip->ip_bhs_mbuf = m; + + a = roundup2((uintptr_t)(icp + 1), _Alignof(struct iscsi_bhs *)); + ip->ip_bhs = (struct iscsi_bhs *)a; +#ifdef INVARIANTS + /* Everything must fit entirely in the mbuf. */ + a = (uintptr_t)(ip->ip_bhs + 1); + MPASS(a <= (uintptr_t)m + MSIZE); +#endif + bzero(ip->ip_bhs, sizeof(*ip->ip_bhs)); + + m->m_data = (void *)ip->ip_bhs; + m->m_len = sizeof(struct iscsi_bhs); + m->m_pkthdr.len = m->m_len; + + return (ip); +} + +void +icl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic) +{ + + ip->ip_conn = ic; +#ifdef DIAGNOSTIC + refcount_acquire(&ic->ic_outstanding_pdus); +#endif +} + +/* + * Allocate icl_pdu with empty BHS to fill up by the caller. + */ +static struct icl_pdu * +icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags) +{ + struct icl_pdu *ip; + + ip = icl_cxgbei_new_pdu(flags); + if (__predict_false(ip == NULL)) + return (NULL); + icl_cxgbei_new_pdu_set_conn(ip, ic); + + return (ip); +} + +static size_t +icl_pdu_data_segment_length(const struct icl_pdu *request) +{ + uint32_t len = 0; + + len += request->ip_bhs->bhs_data_segment_len[0]; + len <<= 8; + len += request->ip_bhs->bhs_data_segment_len[1]; + len <<= 8; + len += request->ip_bhs->bhs_data_segment_len[2]; + + return (len); +} + +size_t +icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic, + const struct icl_pdu *request) +{ + + return (icl_pdu_data_segment_length(request)); +} + +static uint32_t +icl_conn_build_tasktag(struct icl_conn *ic, uint32_t tag) +{ + return tag; +} + +static struct mbuf * +finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp) +{ + struct icl_pdu *ip = &icp->ip; + uint8_t ulp_submode, padding; + struct mbuf *m, *last; + struct iscsi_bhs *bhs; + + /* + * Fix up the data segment mbuf first. + */ + m = ip->ip_data_mbuf; + ulp_submode = icc->ulp_submode; + if (m) { + last = m_last(m); + + /* + * Round up the data segment to a 4B boundary. Pad with 0 if + * necessary. There will definitely be room in the mbuf. + */ + padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len; + if (padding) { + bzero(mtod(last, uint8_t *) + last->m_len, padding); + last->m_len += padding; + } + } else { + MPASS(ip->ip_data_len == 0); + ulp_submode &= ~ULP_CRC_DATA; + padding = 0; + } + + /* + * Now the header mbuf that has the BHS. + */ + m = ip->ip_bhs_mbuf; + MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs)); + MPASS(m->m_len == sizeof(struct iscsi_bhs)); + + bhs = ip->ip_bhs; + bhs->bhs_data_segment_len[2] = ip->ip_data_len; + bhs->bhs_data_segment_len[1] = ip->ip_data_len >> 8; + bhs->bhs_data_segment_len[0] = ip->ip_data_len >> 16; + + /* "Convert" PDU to mbuf chain. Do not use icp/ip after this. */ + m->m_pkthdr.len = sizeof(struct iscsi_bhs) + ip->ip_data_len + padding; + m->m_next = ip->ip_data_mbuf; + set_mbuf_ulp_submode(m, ulp_submode); +#ifdef INVARIANTS + bzero(icp, sizeof(*icp)); +#endif +#ifdef DIAGNOSTIC + refcount_release(&icc->ic.ic_outstanding_pdus); +#endif + + return (m); +} + +int +icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip, + const void *addr, size_t len, int flags) +{ + struct mbuf *m; +#ifdef INVARIANTS + struct icl_cxgbei_pdu *icp = ip_to_icp(ip); +#endif + + MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); + MPASS(ic == ip->ip_conn); + KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len)); + + m = ip->ip_data_mbuf; + if (m == NULL) { + m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES); + if (__predict_false(m == NULL)) + return (ENOMEM); + + ip->ip_data_mbuf = m; + } + + if (__predict_true(m_append(m, len, addr) != 0)) { + ip->ip_data_len += len; + MPASS(ip->ip_data_len <= CXGBEI_MAX_DSL); + return (0); + } else { + if (flags & M_WAITOK) { + CXGBE_UNIMPLEMENTED("fail safe append"); + } + ip->ip_data_len = m_length(m, NULL); + return (1); + } +} + +void +icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, + size_t off, void *addr, size_t len) +{ + struct icl_cxgbei_pdu *icp = ip_to_icp(ip); + + if (icp->pdu_flags & SBUF_ULP_FLAG_DATA_DDPED) + return; /* data is DDP'ed, no need to copy */ + m_copydata(ip->ip_data_mbuf, off, len, addr); +} + +void +icl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) +{ + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + struct icl_cxgbei_pdu *icp = ip_to_icp(ip); + struct socket *so = ic->ic_socket; + struct toepcb *toep = icc->toep; + struct inpcb *inp; + struct mbuf *m; + + MPASS(ic == ip->ip_conn); + MPASS(ip->ip_bhs_mbuf != NULL); + /* The kernel doesn't generate PDUs with AHS. */ + MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0); + + ICL_CONN_LOCK_ASSERT(ic); + /* NOTE: sowriteable without so_snd lock is a mostly harmless race. */ + if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) { + icl_cxgbei_conn_pdu_free(ic, ip); + return; + } + + m = finalize_pdu(icc, icp); + M_ASSERTPKTHDR(m); + MPASS((m->m_pkthdr.len & 3) == 0); + MPASS(m->m_pkthdr.len + 8 <= CXGBEI_MAX_PDU); + + /* + * Do not get inp from toep->inp as the toepcb might have detached + * already. + */ + inp = sotoinpcb(so); + INP_WLOCK(inp); + if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) || + __predict_false((toep->flags & TPF_ATTACHED) == 0)) + m_freem(m); + else { + mbufq_enqueue(&toep->ulp_pduq, m); + t4_push_pdus(icc->sc, toep, 0); + } + INP_WUNLOCK(inp); +} + +static struct icl_conn * +icl_cxgbei_new_conn(const char *name, struct mtx *lock) +{ + struct icl_cxgbei_conn *icc; + struct icl_conn *ic; + + refcount_acquire(&icl_cxgbei_ncons); + + icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE, + M_WAITOK | M_ZERO); + icc->icc_signature = CXGBEI_CONN_SIGNATURE; + STAILQ_INIT(&icc->rcvd_pdus); + + ic = &icc->ic; + ic->ic_lock = lock; + + /* XXXNP: review. Most of these icl_conn fields aren't really used */ + STAILQ_INIT(&ic->ic_to_send); + cv_init(&ic->ic_send_cv, "icl_cxgbei_tx"); + cv_init(&ic->ic_receive_cv, "icl_cxgbei_rx"); +#ifdef DIAGNOSTIC + refcount_init(&ic->ic_outstanding_pdus, 0); +#endif + ic->ic_max_data_segment_length = CXGBEI_MAX_DSL; + ic->ic_name = name; + ic->ic_offload = "cxgbei"; + + CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); + + return (ic); +} + +void +icl_cxgbei_conn_free(struct icl_conn *ic) +{ + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + + MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); + + CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); + + cv_destroy(&ic->ic_send_cv); + cv_destroy(&ic->ic_receive_cv); + + kobj_delete((struct kobj *)icc, M_CXGBE); + refcount_release(&icl_cxgbei_ncons); +} + +static int +icl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so) +{ + size_t minspace; + struct sockopt opt; + int error, one = 1; + + /* + * For sendspace, this is required because the current code cannot + * send a PDU in pieces; thus, the minimum buffer size is equal + * to the maximum PDU size. "+4" is to account for possible padding. + * + * What we should actually do here is to use autoscaling, but set + * some minimal buffer size to "minspace". I don't know a way to do + * that, though. + */ + minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + + ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; + if (sendspace < minspace) + sendspace = minspace; + if (recvspace < minspace) + recvspace = minspace; + + error = soreserve(so, sendspace, recvspace); + if (error != 0) { + icl_cxgbei_conn_close(ic); + return (error); + } + SOCKBUF_LOCK(&so->so_snd); + so->so_snd.sb_flags |= SB_AUTOSIZE; + SOCKBUF_UNLOCK(&so->so_snd); + SOCKBUF_LOCK(&so->so_rcv); + so->so_rcv.sb_flags |= SB_AUTOSIZE; + SOCKBUF_UNLOCK(&so->so_rcv); + + /* + * Disable Nagle. + */ + bzero(&opt, sizeof(opt)); + opt.sopt_dir = SOPT_SET; + opt.sopt_level = IPPROTO_TCP; + opt.sopt_name = TCP_NODELAY; + opt.sopt_val = &one; + opt.sopt_valsize = sizeof(one); + error = sosetopt(so, &opt); + if (error != 0) { + icl_cxgbei_conn_close(ic); + return (error); + } + + return (0); +} + +/* + * Request/response structure used to find out the adapter offloading a socket. + */ +struct find_ofld_adapter_rr { + struct socket *so; + struct adapter *sc; /* result */ +}; + +static void +find_offload_adapter(struct adapter *sc, void *arg) +{ + struct find_ofld_adapter_rr *fa = arg; + struct socket *so = fa->so; + struct tom_data *td = sc->tom_softc; + struct tcpcb *tp; + struct inpcb *inp; + + /* Non-TCP were filtered out earlier. */ + MPASS(so->so_proto->pr_protocol == IPPROTO_TCP); + + if (fa->sc != NULL) + return; /* Found already. */ + + if (td == NULL) + return; /* TOE not enabled on this adapter. */ + + inp = sotoinpcb(so); + INP_WLOCK(inp); + if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { + tp = intotcpcb(inp); + if (tp->t_flags & TF_TOE && tp->tod == &td->tod) + fa->sc = sc; /* Found. */ + } + INP_WUNLOCK(inp); +} + +/* XXXNP: move this to t4_tom. */ +static void +send_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen) +{ + struct wrqe *wr; + struct fw_flowc_wr *flowc; + const u_int nparams = 1; + u_int flowclen; + struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + + flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); + + wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); + if (wr == NULL) { + /* XXX */ + panic("%s: allocation failure.", __func__); + } + flowc = wrtod(wr); + memset(flowc, 0, wr->wr_len); + + flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | + V_FW_FLOWC_WR_NPARAMS(nparams)); + flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | + V_FW_WR_FLOWID(toep->tid)); + + flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; + flowc->mnemval[0].val = htobe32(maxlen); + + txsd->tx_credits = howmany(flowclen, 16); + txsd->plen = 0; + KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, + ("%s: not enough credits (%d)", __func__, toep->tx_credits)); + toep->tx_credits -= txsd->tx_credits; + if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) + toep->txsd_pidx = 0; + toep->txsd_avail--; + + t4_wrq_tx(sc, wr); +} + +static void +set_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, int hcrc, int dcrc) +{ + uint64_t val = 0; + + if (hcrc) + val |= ULP_CRC_HEADER; + if (dcrc) + val |= ULP_CRC_DATA; + val <<= 4; + val |= ULP_MODE_ISCSI; + + CTR4(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, CRC hdr=%d data=%d", + __func__, toep->tid, hcrc, dcrc); + + t4_set_tcb_field(sc, toep, 1, 0, 0xfff, val); +} + +/* + * XXXNP: Who is responsible for cleaning up the socket if this returns with an + * error? Review all error paths. + * + * XXXNP: What happens to the socket's fd reference if the operation is + * successful, and how does that affect the socket's life cycle? + */ +int +icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd) +{ + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + struct find_ofld_adapter_rr fa; + struct file *fp; + struct socket *so; + struct inpcb *inp; + struct tcpcb *tp; + struct toepcb *toep; + cap_rights_t rights; + int error; + + MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); + ICL_CONN_LOCK_ASSERT_NOT(ic); + + /* + * Steal the socket from userland. + */ + error = fget(curthread, fd, + cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); + if (error != 0) + return (error); + if (fp->f_type != DTYPE_SOCKET) { + fdrop(fp, curthread); + return (EINVAL); + } + so = fp->f_data; + if (so->so_type != SOCK_STREAM || + so->so_proto->pr_protocol != IPPROTO_TCP) { + fdrop(fp, curthread); + return (EINVAL); + } + + ICL_CONN_LOCK(ic); + if (ic->ic_socket != NULL) { + ICL_CONN_UNLOCK(ic); + fdrop(fp, curthread); + return (EBUSY); + } + ic->ic_disconnecting = false; + ic->ic_socket = so; + fp->f_ops = &badfileops; + fp->f_data = NULL; + fdrop(fp, curthread); + ICL_CONN_UNLOCK(ic); + + /* Find the adapter offloading this socket. */ + fa.sc = NULL; + fa.so = so; + t4_iterate(find_offload_adapter, &fa); + if (fa.sc == NULL) + return (EINVAL); + icc->sc = fa.sc; + + error = icl_cxgbei_setsockopt(ic, so); + if (error) + return (error); + + inp = sotoinpcb(so); + INP_WLOCK(inp); + tp = intotcpcb(inp); + if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) + error = EBUSY; + else { + /* + * socket could not have been "unoffloaded" if here. + */ + MPASS(tp->t_flags & TF_TOE); + MPASS(tp->tod != NULL); + MPASS(tp->t_toe != NULL); + toep = tp->t_toe; + MPASS(toep->vi->pi->adapter == icc->sc); + icc->toep = toep; + icc->cwt = cxgbei_select_worker_thread(icc); + icc->ulp_submode = 0; + if (ic->ic_header_crc32c) + icc->ulp_submode |= ULP_CRC_HEADER; + if (ic->ic_data_crc32c) + icc->ulp_submode |= ULP_CRC_DATA; + so->so_options |= SO_NO_DDP; + toep->ulp_mode = ULP_MODE_ISCSI; + toep->ulpcb = icc; + + send_iscsi_flowc_wr(icc->sc, toep, CXGBEI_MAX_PDU); + set_ulp_mode_iscsi(icc->sc, toep, ic->ic_header_crc32c, + ic->ic_data_crc32c); + error = 0; + } + INP_WUNLOCK(inp); + + return (error); +} + +void +icl_cxgbei_conn_close(struct icl_conn *ic) +{ + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + struct icl_pdu *ip; + struct socket *so; + struct sockbuf *sb; + struct inpcb *inp; + struct toepcb *toep = icc->toep; + + MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); + ICL_CONN_LOCK_ASSERT_NOT(ic); + + ICL_CONN_LOCK(ic); + so = ic->ic_socket; + if (ic->ic_disconnecting || so == NULL) { + CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p", + __func__, icc, ic->ic_disconnecting, so); + ICL_CONN_UNLOCK(ic); + return; + } + ic->ic_disconnecting = true; + + /* These are unused in this driver right now. */ + MPASS(STAILQ_EMPTY(&ic->ic_to_send)); + MPASS(ic->ic_receive_pdu == NULL); + +#ifdef DIAGNOSTIC + KASSERT(ic->ic_outstanding_pdus == 0, + ("destroying session with %d outstanding PDUs", + ic->ic_outstanding_pdus)); +#endif + ICL_CONN_UNLOCK(ic); + + CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1, + icc); + inp = sotoinpcb(so); + sb = &so->so_rcv; + INP_WLOCK(inp); + if (toep != NULL) { /* NULL if connection was never offloaded. */ + toep->ulpcb = NULL; + mbufq_drain(&toep->ulp_pduq); + SOCKBUF_LOCK(sb); + if (icc->rx_flags & RXF_ACTIVE) { + volatile u_int *p = &icc->rx_flags; + + SOCKBUF_UNLOCK(sb); + INP_WUNLOCK(inp); + + while (*p & RXF_ACTIVE) + pause("conclo", 1); + + INP_WLOCK(inp); + SOCKBUF_LOCK(sb); + } + + while (!STAILQ_EMPTY(&icc->rcvd_pdus)) { + ip = STAILQ_FIRST(&icc->rcvd_pdus); + STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next); + icl_cxgbei_conn_pdu_free(ic, ip); + } + SOCKBUF_UNLOCK(sb); + } + INP_WUNLOCK(inp); + + ICL_CONN_LOCK(ic); + ic->ic_socket = NULL; + ICL_CONN_UNLOCK(ic); + + /* + * XXXNP: we should send RST instead of FIN when PDUs held in various + * queues were purged instead of delivered reliably but soabort isn't + * really general purpose and wouldn't do the right thing here. + */ + soclose(so); +} + +int +icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct ccb_scsiio *csio, + uint32_t *task_tagp, void **prvp) +{ + void *prv; + + *task_tagp = icl_conn_build_tasktag(ic, *task_tagp); + + prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO); + if (prv == NULL) + return (ENOMEM); + + *prvp = prv; + + cxgbei_conn_task_reserve_itt(ic, prvp, csio, task_tagp); + + return (0); +} + +void +icl_cxgbei_conn_task_done(struct icl_conn *ic, void *prv) +{ + + cxgbei_cleanup_task(ic, prv); + uma_zfree(icl_transfer_zone, prv); +} + +int +icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, + uint32_t *transfer_tag, void **prvp) +{ + void *prv; + + *transfer_tag = icl_conn_build_tasktag(ic, *transfer_tag); + + prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO); + if (prv == NULL) + return (ENOMEM); + + *prvp = prv; + + cxgbei_conn_transfer_reserve_ttt(ic, prvp, io, transfer_tag); + + return (0); +} + +void +icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *prv) +{ + cxgbei_cleanup_task(ic, prv); + uma_zfree(icl_transfer_zone, prv); +} + +static int +icl_cxgbei_limits(size_t *limitp) +{ + + *limitp = CXGBEI_MAX_DSL; + + return (0); +} + +static int +icl_cxgbei_load(void) +{ + int error; + + icl_transfer_zone = uma_zcreate("icl_transfer", + 16 * 1024, NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + + refcount_init(&icl_cxgbei_ncons, 0); + + error = icl_register("cxgbei", 100, icl_cxgbei_limits, + icl_cxgbei_new_conn); + KASSERT(error == 0, ("failed to register")); + + return (error); +} + +static int +icl_cxgbei_unload(void) +{ + + if (icl_cxgbei_ncons != 0) + return (EBUSY); + + icl_unregister("cxgbei"); + + uma_zdestroy(icl_transfer_zone); + + return (0); +} + +static int +icl_cxgbei_modevent(module_t mod, int what, void *arg) +{ + + switch (what) { + case MOD_LOAD: + return (icl_cxgbei_load()); + case MOD_UNLOAD: + return (icl_cxgbei_unload()); + default: + return (EINVAL); + } +} + +moduledata_t icl_cxgbei_data = { + "icl_cxgbei", + icl_cxgbei_modevent, + 0 +}; + +DECLARE_MODULE(icl_cxgbei, icl_cxgbei_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); +MODULE_DEPEND(icl_cxgbei, icl, 1, 1, 1); +MODULE_VERSION(icl_cxgbei, 1); +#endif Property changes on: head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/modules/cxgbe/Makefile =================================================================== --- head/sys/modules/cxgbe/Makefile (revision 292739) +++ head/sys/modules/cxgbe/Makefile (revision 292740) @@ -1,26 +1,28 @@ # # $FreeBSD$ # SYSDIR?=${.CURDIR}/../.. .include "${SYSDIR}/conf/kern.opts.mk" SUBDIR= if_cxgbe SUBDIR+= if_cxl SUBDIR+= t4_firmware SUBDIR+= t5_firmware SUBDIR+= ${_tom} SUBDIR+= ${_iw_cxgbe} +SUBDIR+= ${_cxgbei} .if ${MACHINE_CPUARCH} == "amd64" _tom= tom +_cxgbei= cxgbei .if ${MK_OFED} != "no" || defined(ALL_MODULES) _iw_cxgbe= iw_cxgbe .endif .endif .if ${MACHINE_CPUARCH} == "i386" _tom= tom .endif .include Index: head/sys/modules/cxgbe/cxgbei/Makefile =================================================================== --- head/sys/modules/cxgbe/cxgbei/Makefile (nonexistent) +++ head/sys/modules/cxgbe/cxgbei/Makefile (revision 292740) @@ -0,0 +1,23 @@ +# $FreeBSD$ + +CXGBE = ${.CURDIR}/../../../dev/cxgbe +.PATH: ${CXGBE}/cxgbei + +KMOD= cxgbei + +SRCS= cxgbei.c +SRCS+= cxgbei_ulp2_ddp.c +SRCS+= icl_cxgbei.c +SRCS+= bus_if.h +SRCS+= device_if.h +SRCS+= opt_inet.h +SRCS+= opt_inet6.h +SRCS+= pci_if.h +SRCS+= opt_cam.h +SRCS+= icl_conn_if.h + +CFLAGS+= -I${CXGBE} + +MFILES= kern/bus_if.m kern/device_if.m dev/iscsi/icl_conn_if.m dev/pci/pci_if.m + +.include Property changes on: head/sys/modules/cxgbe/cxgbei/Makefile ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property