Page MenuHomeFreeBSD
Differential D20839 Diff 59332 sys/dev/cxgbe/tom/t4_cpl_io.c

Changeset View

Standalone View

View Options

sys/dev/cxgbe/tom/t4_cpl_io.c

Show First 20 Lines • Show All 70 Lines • ▼ Show 20 Lines
#include "common/t4_regs.h" #include "common/t4_regs.h"
#include "common/t4_tcb.h" #include "common/t4_tcb.h"
#include "tom/t4_tom_l2t.h" #include "tom/t4_tom_l2t.h"
#include "tom/t4_tom.h" #include "tom/t4_tom.h"
static void t4_aiotx_cancel(struct kaiocb *job); static void t4_aiotx_cancel(struct kaiocb *job);
static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep); static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep);
static size_t
aiotx_mbuf_pgoff(struct mbuf *m)
{
struct aiotx_buffer *ab;
MPASS(IS_AIOTX_MBUF(m));
ab = m->m_ext.ext_arg1;
return ((ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) % PAGE_SIZE);
}
static vm_page_t *
aiotx_mbuf_pages(struct mbuf *m)
{
struct aiotx_buffer *ab;
int npages;
MPASS(IS_AIOTX_MBUF(m));
ab = m->m_ext.ext_arg1;
npages = (ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) / PAGE_SIZE;
return (ab->ps.pages + npages);
}
void void
send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
{ {
struct wrqe *wr; struct wrqe *wr;
struct fw_flowc_wr *flowc; struct fw_flowc_wr *flowc;
unsigned int nparams, flowclen, paramidx; unsigned int nparams, flowclen, paramidx;
struct vi_info *vi = toep->vi; struct vi_info *vi = toep->vi;
struct port_info *pi = vi->pi; struct port_info *pi = vi->pi;
▲ Show 20 Lines • Show All 533 Lines • ▼ Show 20 Lineswrite_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n)
KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
sglist_init(&sg, n, segs); sglist_init(&sg, n, segs);
usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
V_ULPTX_NSGE(nsegs)); V_ULPTX_NSGE(nsegs));
i = -1; i = -1;
for (m = start; m != stop; m = m->m_next) { for (m = start; m != stop; m = m->m_next) {
if (IS_AIOTX_MBUF(m)) if (m->m_flags & M_NOMAP)
rc = sglist_append_vmpages(&sg, aiotx_mbuf_pages(m),
aiotx_mbuf_pgoff(m), m->m_len);
else if (m->m_flags & M_NOMAP)
rc = sglist_append_mb_ext_pgs(&sg, m); rc = sglist_append_mb_ext_pgs(&sg, m);
else else
rc = sglist_append(&sg, mtod(m, void *), m->m_len); rc = sglist_append(&sg, mtod(m, void *), m->m_len);
if (__predict_false(rc != 0)) if (__predict_false(rc != 0))
panic("%s: sglist_append %d", __func__, rc); panic("%s: sglist_append %d", __func__, rc);
for (j = 0; j < sg.sg_nseg; i++, j++) { for (j = 0; j < sg.sg_nseg; i++, j++) {
if (i < 0) { if (i < 0) {
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Linest4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
struct wrqe *wr; struct wrqe *wr;
u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
struct inpcb *inp = toep->inp; struct inpcb *inp = toep->inp;
struct tcpcb *tp = intotcpcb(inp); struct tcpcb *tp = intotcpcb(inp);
struct socket *so = inp->inp_socket; struct socket *so = inp->inp_socket;
struct sockbuf *sb = &so->so_snd; struct sockbuf *sb = &so->so_snd;
int tx_credits, shove, compl, sowwakeup; int tx_credits, shove, compl, sowwakeup;
struct ofld_tx_sdesc *txsd; struct ofld_tx_sdesc *txsd;
bool aiotx_mbuf_seen; bool nomap_mbuf_seen;
INP_WLOCK_ASSERT(inp); INP_WLOCK_ASSERT(inp);
KASSERT(toep->flags & TPF_FLOWC_WR_SENT, KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
KASSERT(toep->ulp_mode == ULP_MODE_NONE || KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TCPDDP ||
toep->ulp_mode == ULP_MODE_TLS || toep->ulp_mode == ULP_MODE_TLS ||
Show All 36 Lines if (drop) {
sbdrop_locked(sb, drop); sbdrop_locked(sb, drop);
drop = 0; drop = 0;
} }
sb_sndptr = sb->sb_sndptr; sb_sndptr = sb->sb_sndptr;
sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb;
plen = 0; plen = 0;
nsegs = 0; nsegs = 0;
max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
aiotx_mbuf_seen = false; nomap_mbuf_seen = false;
for (m = sndptr; m != NULL; m = m->m_next) { for (m = sndptr; m != NULL; m = m->m_next) {
int n; int n;
if (IS_AIOTX_MBUF(m)) if (m->m_flags & M_NOMAP)
n = sglist_count_vmpages(aiotx_mbuf_pages(m),
aiotx_mbuf_pgoff(m), m->m_len);
else if (m->m_flags & M_NOMAP)
n = sglist_count_mb_ext_pgs(m); n = sglist_count_mb_ext_pgs(m);
else else
n = sglist_count(mtod(m, void *), m->m_len); n = sglist_count(mtod(m, void *), m->m_len);
nsegs += n; nsegs += n;
plen += m->m_len; plen += m->m_len;
/* This mbuf sent us _over_ the nsegs limit, back out */ /* This mbuf sent us _over_ the nsegs limit, back out */
Show All 12 Lines for (m = sndptr; m != NULL; m = m->m_next) {
} else } else
SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK(sb);
SOCKBUF_UNLOCK_ASSERT(sb); SOCKBUF_UNLOCK_ASSERT(sb);
return; return;
} }
break; break;
} }
if (IS_AIOTX_MBUF(m)) if (m->m_flags & M_NOMAP)
aiotx_mbuf_seen = true; nomap_mbuf_seen = true;
if (max_nsegs_1mbuf < n) if (max_nsegs_1mbuf < n)
max_nsegs_1mbuf = n; max_nsegs_1mbuf = n;
sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ sb_sndptr = m; /* new sb->sb_sndptr if all goes well */
/* This mbuf put us right at the max_nsegs limit */ /* This mbuf put us right at the max_nsegs limit */
if (plen > max_imm && nsegs == max_nsegs) { if (plen > max_imm && nsegs == max_nsegs) {
m = m->m_next; m = m->m_next;
break; break;
Show All 32 Lines if (plen == 0) {
("%s: nothing to send, but m != NULL", __func__)); ("%s: nothing to send, but m != NULL", __func__));
break; break;
} }
if (__predict_false(toep->flags & TPF_FIN_SENT)) if (__predict_false(toep->flags & TPF_FIN_SENT))
panic("%s: excess tx.", __func__); panic("%s: excess tx.", __func__);
shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
if (plen <= max_imm && !aiotx_mbuf_seen) { if (plen <= max_imm && !nomap_mbuf_seen) {
/* Immediate data tx */ /* Immediate data tx */
wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16),
toep->ofld_txq); toep->ofld_txq);
if (wr == NULL) { if (wr == NULL) {
/* XXX: how will we recover from this? */ /* XXX: how will we recover from this? */
toep->flags |= TPF_TX_SUSPENDED; toep->flags |= TPF_TX_SUSPENDED;
▲ Show 20 Lines • Show All 1,041 Lines • ▼ Show 20 Linest4_uninit_cpl_io_handlers(void)
t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL); t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL);
t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL); t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL);
t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM); t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM);
t4_register_cpl_handler(CPL_RX_DATA, NULL); t4_register_cpl_handler(CPL_RX_DATA, NULL);
t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM); t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM);
} }
/* /*
* Use the 'backend3' field in AIO jobs to store the amount of data * Use the 'backend1' field in AIO jobs to hold an error that should
* sent by the AIO job so far and the 'backend4' field to hold an * be reported when the job is completed, the 'backend3' field to
* error that should be reported when the job is completed. * store the amount of data sent by the AIO job so far, and the
* 'backend4' field to hold a reference count on the job.
*
* Each unmapped mbuf holds a reference on the job as does the queue
* so long as the job is queued.
*/ */
#define aio_error backend1
#define aio_sent backend3 #define aio_sent backend3
#define aio_error backend4 #define aio_refs backend4
#define jobtotid(job) \ #define jobtotid(job) \
(((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid) (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid)
static void static void
free_aiotx_buffer(struct aiotx_buffer *ab) aiotx_free_job(struct kaiocb *job)
{ {
struct kaiocb *job;
long status; long status;
int error; int error;
if (refcount_release(&ab->refcount) == 0) if (refcount_release(&job->aio_refs) == 0)
return; return;
job = ab->job; error = (intptr_t)job->aio_error;
error = job->aio_error;
status = job->aio_sent; status = job->aio_sent;
vm_page_unhold_pages(ab->ps.pages, ab->ps.npages);
free(ab, M_CXGBE);
#ifdef VERBOSE_TRACES #ifdef VERBOSE_TRACES
CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__, CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__,
jobtotid(job), job, status, error); jobtotid(job), job, status, error);
#endif #endif
if (error == ECANCELED && status != 0) if (error != 0 && status != 0)
error = 0; error = 0;
if (error == ECANCELED) if (error == ECANCELED)
aio_cancel(job); aio_cancel(job);
else if (error) else if (error)
aio_complete(job, -1, error); aio_complete(job, -1, error);
else else {
job->msgsnd = 1;
aio_complete(job, status, 0); aio_complete(job, status, 0);
} }
}
static void static void
t4_aiotx_mbuf_free(struct mbuf *m) aiotx_free_pgs(struct mbuf *m)
{ {
struct aiotx_buffer *ab = m->m_ext.ext_arg1; struct mbuf_ext_pgs *ext_pgs;
struct kaiocb *job;
struct mtx *mtx;
vm_page_t pg;
MBUF_EXT_PGS_ASSERT(m);
ext_pgs = m->m_ext.ext_pgs;
job = m->m_ext.ext_arg2;
#ifdef VERBOSE_TRACES #ifdef VERBOSE_TRACES
CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__,
m->m_len, jobtotid(ab->job)); m->m_len, jobtotid(job));
#endif #endif
free_aiotx_buffer(ab);
mtx = NULL;
for (int i = 0; i < ext_pgs->npgs; i++) {
jhbAuthorUnsubmitted
Done
Inline Actions

This is effectively vm_unhold_pages(), but it has to use PHYS_TO_VM_PAGE to get back to the page pointer.

jhb: This is effectively vm_unhold_pages(), but it has to use PHYS_TO_VM_PAGE to get back to the…
pg = PHYS_TO_VM_PAGE(ext_pgs->pa[i]);
vm_page_change_lock(pg, &mtx);
vm_page_unhold(pg);
} }
if (mtx != NULL)
mtx_unlock(mtx);
aiotx_free_job(job);
}
/* /*
* Hold the buffer backing an AIO request and return an AIO transmit * Allocate a chain of unmapped mbufs describing the next 'len' bytes
* buffer. * of an AIO job.
*/ */
static int static struct mbuf *
hold_aio(struct kaiocb *job) alloc_aiotx_mbuf(struct kaiocb *job, int len)
{ {
struct aiotx_buffer *ab;
struct vmspace *vm; struct vmspace *vm;
vm_page_t pgs[MBUF_PEXT_MAX_PGS];
struct mbuf *m, *top, *last;
struct mbuf_ext_pgs *ext_pgs;
vm_map_t map; vm_map_t map;
vm_offset_t start, end, pgoff; vm_offset_t start;
int n; int i, mlen, npages, pgoff;
MPASS(job->backend1 == NULL); KASSERT(job->aio_sent + len <= job->uaiocb.aio_nbytes,
("%s(%p, %d): request to send beyond end of buffer", __func__,
job, len));
/* /*
* The AIO subsystem will cancel and drain all requests before * The AIO subsystem will cancel and drain all requests before
* permitting a process to exit or exec, so p_vmspace should * permitting a process to exit or exec, so p_vmspace should
* be stable here. * be stable here.
*/ */
vm = job->userproc->p_vmspace; vm = job->userproc->p_vmspace;
map = &vm->vm_map; map = &vm->vm_map;
start = (uintptr_t)job->uaiocb.aio_buf; start = (uintptr_t)job->uaiocb.aio_buf + job->aio_sent;
pgoff = start & PAGE_MASK; pgoff = start & PAGE_MASK;
end = round_page(start + job->uaiocb.aio_nbytes);
start = trunc_page(start);
n = atop(end - start);
ab = malloc(sizeof(*ab) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK | top = NULL;
M_ZERO); last = NULL;
refcount_init(&ab->refcount, 1); while (len > 0) {
ab->ps.pages = (vm_page_t *)(ab + 1); mlen = imin(len, MBUF_PEXT_MAX_PGS * PAGE_SIZE - pgoff);
ab->ps.npages = vm_fault_quick_hold_pages(map, start, end - start, KASSERT(mlen == len || (start + mlen & PAGE_MASK) == 0,
VM_PROT_WRITE, ab->ps.pages, n); ("%s: next start (%#jx + %#x) is not page aligned",
if (ab->ps.npages < 0) { __func__, (uintmax_t)start, mlen));
free(ab, M_CXGBE);
return (EFAULT); npages = vm_fault_quick_hold_pages(map, start, mlen,
VM_PROT_WRITE, pgs, nitems(pgs));
if (npages < 0)
break;
m = mb_alloc_ext_pgs(M_WAITOK, false, aiotx_free_pgs);
if (m == NULL) {
vm_page_unhold_pages(pgs, npages);
break;
} }
KASSERT(ab->ps.npages == n, ext_pgs = m->m_ext.ext_pgs;
("hold_aio: page count mismatch: %d vs %d", ab->ps.npages, n)); ext_pgs->first_pg_off = pgoff;
ext_pgs->npgs = npages;
if (npages == 1) {
KASSERT(mlen + pgoff <= PAGE_SIZE,
("%s: single page is too large (off %d len %d)",
__func__, pgoff, mlen));
ext_pgs->last_pg_len = mlen;
} else {
ext_pgs->last_pg_len = mlen - (PAGE_SIZE - pgoff) -
(npages - 2) * PAGE_SIZE;
}
for (i = 0; i < npages; i++)
ext_pgs->pa[i] = VM_PAGE_TO_PHYS(pgs[i]);
ab->ps.offset = pgoff; m->m_len = mlen;
ab->ps.len = job->uaiocb.aio_nbytes; m->m_ext.ext_size = npages * PAGE_SIZE;
ab->job = job; m->m_ext.ext_arg2 = job;
gallatinUnsubmitted
Not Done
Inline Actions

Why are you using arg2 rather than arg1? Is there a use of arg1 that I'm missing?

In general, you want to prefer arg1, as it falls inside the 2nd cacheline in the mbuf, but arg2 will cause you to touch the 3rd cache line and incur a miss.

gallatin: Why are you using arg2 rather than arg1? Is there a use of arg1 that I'm missing? In general…
jhbAuthorUnsubmitted
Done
Inline Actions

No good reason, too focused on sendfile use ext_arg2 and missed it also used arg1 (and that arg1 is the "primary" arg for sendfile). Will fix.

jhb: No good reason, too focused on sendfile use ext_arg2 and missed it also used arg1 (and that…
job->backend1 = ab; refcount_acquire(&job->aio_refs);
#ifdef VERBOSE_TRACES #ifdef VERBOSE_TRACES
CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d", CTR5(KTR_CXGBE, "%s: tid %d, new mbuf %p for job %p, npages %d",
__func__, jobtotid(job), &ab->ps, job, ab->ps.npages); __func__, jobtotid(job), m, job, npages);
#endif #endif
return (0);
if (top == NULL)
top = m;
else
last->m_next = m;
last = m;
len -= mlen;
start += mlen;
pgoff = 0;
} }
return (top);
}
static void static void
t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job) t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job)
{ {
struct adapter *sc; struct adapter *sc;
struct sockbuf *sb; struct sockbuf *sb;
struct file *fp; struct file *fp;
struct aiotx_buffer *ab;
struct inpcb *inp; struct inpcb *inp;
struct tcpcb *tp; struct tcpcb *tp;
struct mbuf *m; struct mbuf *m;
int error; int error, len;
bool moretocome, sendmore; bool moretocome, sendmore;
sc = td_adapter(toep->td); sc = td_adapter(toep->td);
sb = &so->so_snd; sb = &so->so_snd;
SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK(sb);
fp = job->fd_file; fp = job->fd_file;
ab = job->backend1;
m = NULL; m = NULL;
#ifdef MAC #ifdef MAC
error = mac_socket_check_send(fp->f_cred, so); error = mac_socket_check_send(fp->f_cred, so);
if (error != 0) if (error != 0)
goto out; goto out;
#endif #endif
if (ab == NULL) {
error = hold_aio(job);
if (error != 0)
goto out;
ab = job->backend1;
}
/* Inline sosend_generic(). */ /* Inline sosend_generic(). */
job->msgsnd = 1;
error = sblock(sb, SBL_WAIT); error = sblock(sb, SBL_WAIT);
MPASS(error == 0); MPASS(error == 0);
sendanother: sendanother:
m = m_get(M_WAITOK, MT_DATA);
SOCKBUF_LOCK(sb); SOCKBUF_LOCK(sb);
if (so->so_snd.sb_state & SBS_CANTSENDMORE) { if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK(sb);
sbunlock(sb); sbunlock(sb);
if ((so->so_options & SO_NOSIGPIPE) == 0) { if ((so->so_options & SO_NOSIGPIPE) == 0) {
PROC_LOCK(job->userproc); PROC_LOCK(job->userproc);
kern_psignal(job->userproc, SIGPIPE); kern_psignal(job->userproc, SIGPIPE);
PROC_UNLOCK(job->userproc); PROC_UNLOCK(job->userproc);
Show All 32 Lines if (sbspace(sb) < sb->sb_lowat) {
sbunlock(sb); sbunlock(sb);
goto out; goto out;
} }
/* /*
* Write as much data as the socket permits, but no more than a * Write as much data as the socket permits, but no more than a
* a single sndbuf at a time. * a single sndbuf at a time.
*/ */
m->m_len = sbspace(sb); len = sbspace(sb);
if (m->m_len > ab->ps.len - job->aio_sent) { if (len > job->uaiocb.aio_nbytes - job->aio_sent) {
m->m_len = ab->ps.len - job->aio_sent; len = job->uaiocb.aio_nbytes - job->aio_sent;
moretocome = false; moretocome = false;
} else } else
moretocome = true; moretocome = true;
if (m->m_len > sc->tt.sndbuf) { if (len > sc->tt.sndbuf) {
m->m_len = sc->tt.sndbuf; len = sc->tt.sndbuf;
sendmore = true; sendmore = true;
} else } else
sendmore = false; sendmore = false;
if (!TAILQ_EMPTY(&toep->aiotx_jobq)) if (!TAILQ_EMPTY(&toep->aiotx_jobq))
moretocome = true; moretocome = true;
SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK(sb);
MPASS(m->m_len != 0); MPASS(len != 0);
m = alloc_aiotx_mbuf(job, len);
if (m == NULL) {
sbunlock(sb);
error = EFAULT;
goto out;
}
/* Inlined tcp_usr_send(). */ /* Inlined tcp_usr_send(). */
inp = toep->inp; inp = toep->inp;
INP_WLOCK(inp); INP_WLOCK(inp);
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
INP_WUNLOCK(inp); INP_WUNLOCK(inp);
sbunlock(sb); sbunlock(sb);
error = ECONNRESET; error = ECONNRESET;
goto out; goto out;
} }
refcount_acquire(&ab->refcount); job->aio_sent += m_length(m, NULL);
m_extadd(m, NULL, ab->ps.len, t4_aiotx_mbuf_free, ab,
(void *)(uintptr_t)job->aio_sent, 0, EXT_NET_DRV);
m->m_ext.ext_flags |= EXT_FLAG_AIOTX;
job->aio_sent += m->m_len;
sbappendstream(sb, m, 0); sbappendstream(sb, m, 0);
m = NULL; m = NULL;
if (!(inp->inp_flags & INP_DROPPED)) { if (!(inp->inp_flags & INP_DROPPED)) {
tp = intotcpcb(inp); tp = intotcpcb(inp);
if (moretocome) if (moretocome)
tp->t_flags |= TF_MORETOCOME; tp->t_flags |= TF_MORETOCOME;
error = tp->t_fb->tfb_tcp_output(tp); error = tp->t_fb->tfb_tcp_output(tp);
if (moretocome) if (moretocome)
tp->t_flags &= ~TF_MORETOCOME; tp->t_flags &= ~TF_MORETOCOME;
} }
INP_WUNLOCK(inp); INP_WUNLOCK(inp);
if (sendmore) if (sendmore)
goto sendanother; goto sendanother;
sbunlock(sb); sbunlock(sb);
if (error) if (error)
goto out; goto out;
/* /*
* If this is a non-blocking socket and the request has not * If this is a blocking socket and the request has not been
* been fully completed, requeue it until the socket is ready * fully completed, requeue it until the socket is ready
* again. * again.
*/ */
if (job->aio_sent < job->uaiocb.aio_nbytes && if (job->aio_sent < job->uaiocb.aio_nbytes &&
!(so->so_state & SS_NBIO)) { !(so->so_state & SS_NBIO)) {
SOCKBUF_LOCK(sb); SOCKBUF_LOCK(sb);
if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { if (!aio_set_cancel_function(job, t4_aiotx_cancel)) {
SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK(sb);
error = ECANCELED; error = ECANCELED;
goto out; goto out;
} }
TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list);
return; return;
} }
/* /*
* If the request will not be requeued, drop a reference on * If the request will not be requeued, drop the queue's
* the aiotx buffer. Any mbufs in flight should still * reference to the job. Any mbufs in flight should still
* contain a reference, but this drops the reference that the * hold a reference, but this drops the reference that the
* job owns while it is waiting to queue mbufs to the socket. * queue owns while it is waiting to queue mbufs to the
* socket.
*/ */
free_aiotx_buffer(ab); aiotx_free_job(job);
out: out:
if (error) { if (error) {
if (ab != NULL) { job->aio_error = (void *)(intptr_t)error;
job->aio_error = error; aiotx_free_job(job);
free_aiotx_buffer(ab);
} else {
MPASS(job->aio_sent == 0);
aio_complete(job, -1, error);
} }
}
if (m != NULL) if (m != NULL)
m_free(m); m_free(m);
SOCKBUF_LOCK(sb); SOCKBUF_LOCK(sb);
} }
static void static void
t4_aiotx_task(void *context, int pending) t4_aiotx_task(void *context, int pending)
{ {
Show All 36 Lines#endif
toep->aiotx_so = so; toep->aiotx_so = so;
hold_toepcb(toep); hold_toepcb(toep);
soaio_enqueue(&toep->aiotx_task); soaio_enqueue(&toep->aiotx_task);
} }
static void static void
t4_aiotx_cancel(struct kaiocb *job) t4_aiotx_cancel(struct kaiocb *job)
{ {
struct aiotx_buffer *ab;
struct socket *so; struct socket *so;
struct sockbuf *sb; struct sockbuf *sb;
struct tcpcb *tp; struct tcpcb *tp;
struct toepcb *toep; struct toepcb *toep;
so = job->fd_file->f_data; so = job->fd_file->f_data;
tp = so_sototcpcb(so); tp = so_sototcpcb(so);
toep = tp->t_toe; toep = tp->t_toe;
MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE);
sb = &so->so_snd; sb = &so->so_snd;
SOCKBUF_LOCK(sb); SOCKBUF_LOCK(sb);
if (!aio_cancel_cleared(job)) if (!aio_cancel_cleared(job))
TAILQ_REMOVE(&toep->aiotx_jobq, job, list); TAILQ_REMOVE(&toep->aiotx_jobq, job, list);
SOCKBUF_UNLOCK(sb); SOCKBUF_UNLOCK(sb);
ab = job->backend1; job->aio_error = (void *)(intptr_t)ECANCELED;
if (ab != NULL) aiotx_free_job(job);
free_aiotx_buffer(ab);
else
aio_cancel(job);
} }
int int
t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job) t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job)
{ {
struct tcpcb *tp = so_sototcpcb(so); struct tcpcb *tp = so_sototcpcb(so);
struct toepcb *toep = tp->t_toe; struct toepcb *toep = tp->t_toe;
struct adapter *sc = td_adapter(toep->td); struct adapter *sc = td_adapter(toep->td);
Show All 9 Lines if (tls_tx_key(toep))
return (EOPNOTSUPP); return (EOPNOTSUPP);
SOCKBUF_LOCK(&so->so_snd); SOCKBUF_LOCK(&so->so_snd);
#ifdef VERBOSE_TRACES #ifdef VERBOSE_TRACES
CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid); CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);
#endif #endif
if (!aio_set_cancel_function(job, t4_aiotx_cancel)) if (!aio_set_cancel_function(job, t4_aiotx_cancel))
panic("new job was cancelled"); panic("new job was cancelled");
refcount_init(&job->aio_refs, 1);
TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list);
if (sowriteable(so)) if (sowriteable(so))
t4_aiotx_queue_toep(so, toep); t4_aiotx_queue_toep(so, toep);
SOCKBUF_UNLOCK(&so->so_snd); SOCKBUF_UNLOCK(&so->so_snd);
return (0); return (0);
} }
void void
aiotx_init_toep(struct toepcb *toep) aiotx_init_toep(struct toepcb *toep)
{ {
TAILQ_INIT(&toep->aiotx_jobq); TAILQ_INIT(&toep->aiotx_jobq);
TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep); TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep);
} }
#endif #endif