Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/cxgbe/tom/t4_cpl_io.c
Show First 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | |||||
#include "common/t4_regs.h" | #include "common/t4_regs.h" | ||||
#include "common/t4_tcb.h" | #include "common/t4_tcb.h" | ||||
#include "tom/t4_tom_l2t.h" | #include "tom/t4_tom_l2t.h" | ||||
#include "tom/t4_tom.h" | #include "tom/t4_tom.h" | ||||
static void t4_aiotx_cancel(struct kaiocb *job); | static void t4_aiotx_cancel(struct kaiocb *job); | ||||
static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep); | static void t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep); | ||||
static size_t | |||||
aiotx_mbuf_pgoff(struct mbuf *m) | |||||
{ | |||||
struct aiotx_buffer *ab; | |||||
MPASS(IS_AIOTX_MBUF(m)); | |||||
ab = m->m_ext.ext_arg1; | |||||
return ((ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) % PAGE_SIZE); | |||||
} | |||||
static vm_page_t * | |||||
aiotx_mbuf_pages(struct mbuf *m) | |||||
{ | |||||
struct aiotx_buffer *ab; | |||||
int npages; | |||||
MPASS(IS_AIOTX_MBUF(m)); | |||||
ab = m->m_ext.ext_arg1; | |||||
npages = (ab->ps.offset + (uintptr_t)m->m_ext.ext_arg2) / PAGE_SIZE; | |||||
return (ab->ps.pages + npages); | |||||
} | |||||
void | void | ||||
send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) | send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) | ||||
{ | { | ||||
struct wrqe *wr; | struct wrqe *wr; | ||||
struct fw_flowc_wr *flowc; | struct fw_flowc_wr *flowc; | ||||
unsigned int nparams, flowclen, paramidx; | unsigned int nparams, flowclen, paramidx; | ||||
struct vi_info *vi = toep->vi; | struct vi_info *vi = toep->vi; | ||||
struct port_info *pi = vi->pi; | struct port_info *pi = vi->pi; | ||||
▲ Show 20 Lines • Show All 533 Lines • ▼ Show 20 Lines | write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) | ||||
KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); | KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); | ||||
sglist_init(&sg, n, segs); | sglist_init(&sg, n, segs); | ||||
usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | | usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | | ||||
V_ULPTX_NSGE(nsegs)); | V_ULPTX_NSGE(nsegs)); | ||||
i = -1; | i = -1; | ||||
for (m = start; m != stop; m = m->m_next) { | for (m = start; m != stop; m = m->m_next) { | ||||
if (IS_AIOTX_MBUF(m)) | if (m->m_flags & M_NOMAP) | ||||
rc = sglist_append_vmpages(&sg, aiotx_mbuf_pages(m), | |||||
aiotx_mbuf_pgoff(m), m->m_len); | |||||
else if (m->m_flags & M_NOMAP) | |||||
rc = sglist_append_mb_ext_pgs(&sg, m); | rc = sglist_append_mb_ext_pgs(&sg, m); | ||||
else | else | ||||
rc = sglist_append(&sg, mtod(m, void *), m->m_len); | rc = sglist_append(&sg, mtod(m, void *), m->m_len); | ||||
if (__predict_false(rc != 0)) | if (__predict_false(rc != 0)) | ||||
panic("%s: sglist_append %d", __func__, rc); | panic("%s: sglist_append %d", __func__, rc); | ||||
for (j = 0; j < sg.sg_nseg; i++, j++) { | for (j = 0; j < sg.sg_nseg; i++, j++) { | ||||
if (i < 0) { | if (i < 0) { | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) | ||||
struct wrqe *wr; | struct wrqe *wr; | ||||
u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; | u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; | ||||
struct inpcb *inp = toep->inp; | struct inpcb *inp = toep->inp; | ||||
struct tcpcb *tp = intotcpcb(inp); | struct tcpcb *tp = intotcpcb(inp); | ||||
struct socket *so = inp->inp_socket; | struct socket *so = inp->inp_socket; | ||||
struct sockbuf *sb = &so->so_snd; | struct sockbuf *sb = &so->so_snd; | ||||
int tx_credits, shove, compl, sowwakeup; | int tx_credits, shove, compl, sowwakeup; | ||||
struct ofld_tx_sdesc *txsd; | struct ofld_tx_sdesc *txsd; | ||||
bool aiotx_mbuf_seen; | bool nomap_mbuf_seen; | ||||
INP_WLOCK_ASSERT(inp); | INP_WLOCK_ASSERT(inp); | ||||
KASSERT(toep->flags & TPF_FLOWC_WR_SENT, | KASSERT(toep->flags & TPF_FLOWC_WR_SENT, | ||||
("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); | ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); | ||||
KASSERT(toep->ulp_mode == ULP_MODE_NONE || | KASSERT(toep->ulp_mode == ULP_MODE_NONE || | ||||
toep->ulp_mode == ULP_MODE_TCPDDP || | toep->ulp_mode == ULP_MODE_TCPDDP || | ||||
toep->ulp_mode == ULP_MODE_TLS || | toep->ulp_mode == ULP_MODE_TLS || | ||||
Show All 36 Lines | if (drop) { | ||||
sbdrop_locked(sb, drop); | sbdrop_locked(sb, drop); | ||||
drop = 0; | drop = 0; | ||||
} | } | ||||
sb_sndptr = sb->sb_sndptr; | sb_sndptr = sb->sb_sndptr; | ||||
sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; | sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; | ||||
plen = 0; | plen = 0; | ||||
nsegs = 0; | nsegs = 0; | ||||
max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ | max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ | ||||
aiotx_mbuf_seen = false; | nomap_mbuf_seen = false; | ||||
for (m = sndptr; m != NULL; m = m->m_next) { | for (m = sndptr; m != NULL; m = m->m_next) { | ||||
int n; | int n; | ||||
if (IS_AIOTX_MBUF(m)) | if (m->m_flags & M_NOMAP) | ||||
n = sglist_count_vmpages(aiotx_mbuf_pages(m), | |||||
aiotx_mbuf_pgoff(m), m->m_len); | |||||
else if (m->m_flags & M_NOMAP) | |||||
n = sglist_count_mb_ext_pgs(m); | n = sglist_count_mb_ext_pgs(m); | ||||
else | else | ||||
n = sglist_count(mtod(m, void *), m->m_len); | n = sglist_count(mtod(m, void *), m->m_len); | ||||
nsegs += n; | nsegs += n; | ||||
plen += m->m_len; | plen += m->m_len; | ||||
/* This mbuf sent us _over_ the nsegs limit, back out */ | /* This mbuf sent us _over_ the nsegs limit, back out */ | ||||
Show All 12 Lines | for (m = sndptr; m != NULL; m = m->m_next) { | ||||
} else | } else | ||||
SOCKBUF_UNLOCK(sb); | SOCKBUF_UNLOCK(sb); | ||||
SOCKBUF_UNLOCK_ASSERT(sb); | SOCKBUF_UNLOCK_ASSERT(sb); | ||||
return; | return; | ||||
} | } | ||||
break; | break; | ||||
} | } | ||||
if (IS_AIOTX_MBUF(m)) | if (m->m_flags & M_NOMAP) | ||||
aiotx_mbuf_seen = true; | nomap_mbuf_seen = true; | ||||
if (max_nsegs_1mbuf < n) | if (max_nsegs_1mbuf < n) | ||||
max_nsegs_1mbuf = n; | max_nsegs_1mbuf = n; | ||||
sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ | sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ | ||||
/* This mbuf put us right at the max_nsegs limit */ | /* This mbuf put us right at the max_nsegs limit */ | ||||
if (plen > max_imm && nsegs == max_nsegs) { | if (plen > max_imm && nsegs == max_nsegs) { | ||||
m = m->m_next; | m = m->m_next; | ||||
break; | break; | ||||
Show All 32 Lines | if (plen == 0) { | ||||
("%s: nothing to send, but m != NULL", __func__)); | ("%s: nothing to send, but m != NULL", __func__)); | ||||
break; | break; | ||||
} | } | ||||
if (__predict_false(toep->flags & TPF_FIN_SENT)) | if (__predict_false(toep->flags & TPF_FIN_SENT)) | ||||
panic("%s: excess tx.", __func__); | panic("%s: excess tx.", __func__); | ||||
shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); | shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); | ||||
if (plen <= max_imm && !aiotx_mbuf_seen) { | if (plen <= max_imm && !nomap_mbuf_seen) { | ||||
/* Immediate data tx */ | /* Immediate data tx */ | ||||
wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), | wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), | ||||
toep->ofld_txq); | toep->ofld_txq); | ||||
if (wr == NULL) { | if (wr == NULL) { | ||||
/* XXX: how will we recover from this? */ | /* XXX: how will we recover from this? */ | ||||
toep->flags |= TPF_TX_SUSPENDED; | toep->flags |= TPF_TX_SUSPENDED; | ||||
▲ Show 20 Lines • Show All 1,041 Lines • ▼ Show 20 Lines | t4_uninit_cpl_io_handlers(void) | ||||
t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL); | t4_register_cpl_handler(CPL_CLOSE_CON_RPL, NULL); | ||||
t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL); | t4_register_cpl_handler(CPL_ABORT_REQ_RSS, NULL); | ||||
t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM); | t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS, NULL, CPL_COOKIE_TOM); | ||||
t4_register_cpl_handler(CPL_RX_DATA, NULL); | t4_register_cpl_handler(CPL_RX_DATA, NULL); | ||||
t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM); | t4_register_shared_cpl_handler(CPL_FW4_ACK, NULL, CPL_COOKIE_TOM); | ||||
} | } | ||||
/* | /* | ||||
* Use the 'backend3' field in AIO jobs to store the amount of data | * Use the 'backend1' field in AIO jobs to hold an error that should | ||||
* sent by the AIO job so far and the 'backend4' field to hold an | * be reported when the job is completed, the 'backend3' field to | ||||
* error that should be reported when the job is completed. | * store the amount of data sent by the AIO job so far, and the | ||||
* 'backend4' field to hold a reference count on the job. | |||||
* | |||||
* Each unmapped mbuf holds a reference on the job as does the queue | |||||
* so long as the job is queued. | |||||
*/ | */ | ||||
#define aio_error backend1 | |||||
#define aio_sent backend3 | #define aio_sent backend3 | ||||
#define aio_error backend4 | #define aio_refs backend4 | ||||
#define jobtotid(job) \ | #define jobtotid(job) \ | ||||
(((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid) | (((struct toepcb *)(so_sototcpcb((job)->fd_file->f_data)->t_toe))->tid) | ||||
static void | static void | ||||
free_aiotx_buffer(struct aiotx_buffer *ab) | aiotx_free_job(struct kaiocb *job) | ||||
{ | { | ||||
struct kaiocb *job; | |||||
long status; | long status; | ||||
int error; | int error; | ||||
if (refcount_release(&ab->refcount) == 0) | if (refcount_release(&job->aio_refs) == 0) | ||||
return; | return; | ||||
job = ab->job; | error = (intptr_t)job->aio_error; | ||||
error = job->aio_error; | |||||
status = job->aio_sent; | status = job->aio_sent; | ||||
vm_page_unhold_pages(ab->ps.pages, ab->ps.npages); | |||||
free(ab, M_CXGBE); | |||||
#ifdef VERBOSE_TRACES | #ifdef VERBOSE_TRACES | ||||
CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__, | CTR5(KTR_CXGBE, "%s: tid %d completed %p len %ld, error %d", __func__, | ||||
jobtotid(job), job, status, error); | jobtotid(job), job, status, error); | ||||
#endif | #endif | ||||
if (error == ECANCELED && status != 0) | if (error != 0 && status != 0) | ||||
error = 0; | error = 0; | ||||
if (error == ECANCELED) | if (error == ECANCELED) | ||||
aio_cancel(job); | aio_cancel(job); | ||||
else if (error) | else if (error) | ||||
aio_complete(job, -1, error); | aio_complete(job, -1, error); | ||||
else | else { | ||||
job->msgsnd = 1; | |||||
aio_complete(job, status, 0); | aio_complete(job, status, 0); | ||||
} | } | ||||
} | |||||
static void | static void | ||||
t4_aiotx_mbuf_free(struct mbuf *m) | aiotx_free_pgs(struct mbuf *m) | ||||
{ | { | ||||
struct aiotx_buffer *ab = m->m_ext.ext_arg1; | struct mbuf_ext_pgs *ext_pgs; | ||||
struct kaiocb *job; | |||||
struct mtx *mtx; | |||||
vm_page_t pg; | |||||
MBUF_EXT_PGS_ASSERT(m); | |||||
ext_pgs = m->m_ext.ext_pgs; | |||||
job = m->m_ext.ext_arg1; | |||||
#ifdef VERBOSE_TRACES | #ifdef VERBOSE_TRACES | ||||
CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, | CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__, | ||||
m->m_len, jobtotid(ab->job)); | m->m_len, jobtotid(job)); | ||||
#endif | #endif | ||||
free_aiotx_buffer(ab); | |||||
mtx = NULL; | |||||
for (int i = 0; i < ext_pgs->npgs; i++) { | |||||
jhb: This is effectively vm_unhold_pages(), but it has to use PHYS_TO_VM_PAGE to get back to the… | |||||
pg = PHYS_TO_VM_PAGE(ext_pgs->pa[i]); | |||||
vm_page_change_lock(pg, &mtx); | |||||
vm_page_unhold(pg); | |||||
} | } | ||||
if (mtx != NULL) | |||||
mtx_unlock(mtx); | |||||
aiotx_free_job(job); | |||||
} | |||||
/* | /* | ||||
* Hold the buffer backing an AIO request and return an AIO transmit | * Allocate a chain of unmapped mbufs describing the next 'len' bytes | ||||
* buffer. | * of an AIO job. | ||||
*/ | */ | ||||
static int | static struct mbuf * | ||||
hold_aio(struct kaiocb *job) | alloc_aiotx_mbuf(struct kaiocb *job, int len) | ||||
{ | { | ||||
struct aiotx_buffer *ab; | |||||
struct vmspace *vm; | struct vmspace *vm; | ||||
vm_page_t pgs[MBUF_PEXT_MAX_PGS]; | |||||
struct mbuf *m, *top, *last; | |||||
struct mbuf_ext_pgs *ext_pgs; | |||||
vm_map_t map; | vm_map_t map; | ||||
vm_offset_t start, end, pgoff; | vm_offset_t start; | ||||
int n; | int i, mlen, npages, pgoff; | ||||
MPASS(job->backend1 == NULL); | KASSERT(job->aio_sent + len <= job->uaiocb.aio_nbytes, | ||||
("%s(%p, %d): request to send beyond end of buffer", __func__, | |||||
job, len)); | |||||
/* | /* | ||||
* The AIO subsystem will cancel and drain all requests before | * The AIO subsystem will cancel and drain all requests before | ||||
* permitting a process to exit or exec, so p_vmspace should | * permitting a process to exit or exec, so p_vmspace should | ||||
* be stable here. | * be stable here. | ||||
*/ | */ | ||||
vm = job->userproc->p_vmspace; | vm = job->userproc->p_vmspace; | ||||
map = &vm->vm_map; | map = &vm->vm_map; | ||||
start = (uintptr_t)job->uaiocb.aio_buf; | start = (uintptr_t)job->uaiocb.aio_buf + job->aio_sent; | ||||
pgoff = start & PAGE_MASK; | pgoff = start & PAGE_MASK; | ||||
end = round_page(start + job->uaiocb.aio_nbytes); | |||||
start = trunc_page(start); | |||||
n = atop(end - start); | |||||
ab = malloc(sizeof(*ab) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK | | top = NULL; | ||||
M_ZERO); | last = NULL; | ||||
refcount_init(&ab->refcount, 1); | while (len > 0) { | ||||
ab->ps.pages = (vm_page_t *)(ab + 1); | mlen = imin(len, MBUF_PEXT_MAX_PGS * PAGE_SIZE - pgoff); | ||||
ab->ps.npages = vm_fault_quick_hold_pages(map, start, end - start, | KASSERT(mlen == len || (start + mlen & PAGE_MASK) == 0, | ||||
VM_PROT_WRITE, ab->ps.pages, n); | ("%s: next start (%#jx + %#x) is not page aligned", | ||||
if (ab->ps.npages < 0) { | __func__, (uintmax_t)start, mlen)); | ||||
free(ab, M_CXGBE); | |||||
return (EFAULT); | npages = vm_fault_quick_hold_pages(map, start, mlen, | ||||
VM_PROT_WRITE, pgs, nitems(pgs)); | |||||
if (npages < 0) | |||||
break; | |||||
m = mb_alloc_ext_pgs(M_WAITOK, false, aiotx_free_pgs); | |||||
if (m == NULL) { | |||||
vm_page_unhold_pages(pgs, npages); | |||||
break; | |||||
} | } | ||||
KASSERT(ab->ps.npages == n, | ext_pgs = m->m_ext.ext_pgs; | ||||
("hold_aio: page count mismatch: %d vs %d", ab->ps.npages, n)); | ext_pgs->first_pg_off = pgoff; | ||||
ext_pgs->npgs = npages; | |||||
if (npages == 1) { | |||||
KASSERT(mlen + pgoff <= PAGE_SIZE, | |||||
("%s: single page is too large (off %d len %d)", | |||||
__func__, pgoff, mlen)); | |||||
ext_pgs->last_pg_len = mlen; | |||||
} else { | |||||
ext_pgs->last_pg_len = mlen - (PAGE_SIZE - pgoff) - | |||||
(npages - 2) * PAGE_SIZE; | |||||
} | |||||
for (i = 0; i < npages; i++) | |||||
ext_pgs->pa[i] = VM_PAGE_TO_PHYS(pgs[i]); | |||||
ab->ps.offset = pgoff; | m->m_len = mlen; | ||||
ab->ps.len = job->uaiocb.aio_nbytes; | m->m_ext.ext_size = npages * PAGE_SIZE; | ||||
ab->job = job; | m->m_ext.ext_arg1 = job; | ||||
Not Done Inline ActionsWhy are you using arg2 rather than arg1? Is there a use of arg1 that I'm missing? In general, you want to prefer arg1, as it falls inside the 2nd cacheline in the mbuf, but arg2 will cause you to touch the 3rd cache line and incur a miss. gallatin: Why are you using arg2 rather than arg1? Is there a use of arg1 that I'm missing?
In general… | |||||
Done Inline ActionsNo good reason, too focused on sendfile use ext_arg2 and missed it also used arg1 (and that arg1 is the "primary" arg for sendfile). Will fix. jhb: No good reason, too focused on sendfile use ext_arg2 and missed it also used arg1 (and that… | |||||
job->backend1 = ab; | refcount_acquire(&job->aio_refs); | ||||
#ifdef VERBOSE_TRACES | #ifdef VERBOSE_TRACES | ||||
CTR5(KTR_CXGBE, "%s: tid %d, new pageset %p for job %p, npages %d", | CTR5(KTR_CXGBE, "%s: tid %d, new mbuf %p for job %p, npages %d", | ||||
__func__, jobtotid(job), &ab->ps, job, ab->ps.npages); | __func__, jobtotid(job), m, job, npages); | ||||
#endif | #endif | ||||
return (0); | |||||
if (top == NULL) | |||||
top = m; | |||||
else | |||||
last->m_next = m; | |||||
last = m; | |||||
len -= mlen; | |||||
start += mlen; | |||||
pgoff = 0; | |||||
} | } | ||||
return (top); | |||||
} | |||||
static void | static void | ||||
t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job) | t4_aiotx_process_job(struct toepcb *toep, struct socket *so, struct kaiocb *job) | ||||
{ | { | ||||
struct adapter *sc; | struct adapter *sc; | ||||
struct sockbuf *sb; | struct sockbuf *sb; | ||||
struct file *fp; | struct file *fp; | ||||
struct aiotx_buffer *ab; | |||||
struct inpcb *inp; | struct inpcb *inp; | ||||
struct tcpcb *tp; | struct tcpcb *tp; | ||||
struct mbuf *m; | struct mbuf *m; | ||||
int error; | int error, len; | ||||
bool moretocome, sendmore; | bool moretocome, sendmore; | ||||
sc = td_adapter(toep->td); | sc = td_adapter(toep->td); | ||||
sb = &so->so_snd; | sb = &so->so_snd; | ||||
SOCKBUF_UNLOCK(sb); | SOCKBUF_UNLOCK(sb); | ||||
fp = job->fd_file; | fp = job->fd_file; | ||||
ab = job->backend1; | |||||
m = NULL; | m = NULL; | ||||
#ifdef MAC | #ifdef MAC | ||||
error = mac_socket_check_send(fp->f_cred, so); | error = mac_socket_check_send(fp->f_cred, so); | ||||
if (error != 0) | if (error != 0) | ||||
goto out; | goto out; | ||||
#endif | #endif | ||||
if (ab == NULL) { | |||||
error = hold_aio(job); | |||||
if (error != 0) | |||||
goto out; | |||||
ab = job->backend1; | |||||
} | |||||
/* Inline sosend_generic(). */ | /* Inline sosend_generic(). */ | ||||
job->msgsnd = 1; | |||||
error = sblock(sb, SBL_WAIT); | error = sblock(sb, SBL_WAIT); | ||||
MPASS(error == 0); | MPASS(error == 0); | ||||
sendanother: | sendanother: | ||||
m = m_get(M_WAITOK, MT_DATA); | |||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
if (so->so_snd.sb_state & SBS_CANTSENDMORE) { | if (so->so_snd.sb_state & SBS_CANTSENDMORE) { | ||||
SOCKBUF_UNLOCK(sb); | SOCKBUF_UNLOCK(sb); | ||||
sbunlock(sb); | sbunlock(sb); | ||||
if ((so->so_options & SO_NOSIGPIPE) == 0) { | if ((so->so_options & SO_NOSIGPIPE) == 0) { | ||||
PROC_LOCK(job->userproc); | PROC_LOCK(job->userproc); | ||||
kern_psignal(job->userproc, SIGPIPE); | kern_psignal(job->userproc, SIGPIPE); | ||||
PROC_UNLOCK(job->userproc); | PROC_UNLOCK(job->userproc); | ||||
Show All 32 Lines | if (sbspace(sb) < sb->sb_lowat) { | ||||
sbunlock(sb); | sbunlock(sb); | ||||
goto out; | goto out; | ||||
} | } | ||||
/* | /* | ||||
* Write as much data as the socket permits, but no more than a | * Write as much data as the socket permits, but no more than a | ||||
* a single sndbuf at a time. | * a single sndbuf at a time. | ||||
*/ | */ | ||||
m->m_len = sbspace(sb); | len = sbspace(sb); | ||||
if (m->m_len > ab->ps.len - job->aio_sent) { | if (len > job->uaiocb.aio_nbytes - job->aio_sent) { | ||||
m->m_len = ab->ps.len - job->aio_sent; | len = job->uaiocb.aio_nbytes - job->aio_sent; | ||||
moretocome = false; | moretocome = false; | ||||
} else | } else | ||||
moretocome = true; | moretocome = true; | ||||
if (m->m_len > sc->tt.sndbuf) { | if (len > sc->tt.sndbuf) { | ||||
m->m_len = sc->tt.sndbuf; | len = sc->tt.sndbuf; | ||||
sendmore = true; | sendmore = true; | ||||
} else | } else | ||||
sendmore = false; | sendmore = false; | ||||
if (!TAILQ_EMPTY(&toep->aiotx_jobq)) | if (!TAILQ_EMPTY(&toep->aiotx_jobq)) | ||||
moretocome = true; | moretocome = true; | ||||
SOCKBUF_UNLOCK(sb); | SOCKBUF_UNLOCK(sb); | ||||
MPASS(m->m_len != 0); | MPASS(len != 0); | ||||
m = alloc_aiotx_mbuf(job, len); | |||||
if (m == NULL) { | |||||
sbunlock(sb); | |||||
error = EFAULT; | |||||
goto out; | |||||
} | |||||
/* Inlined tcp_usr_send(). */ | /* Inlined tcp_usr_send(). */ | ||||
inp = toep->inp; | inp = toep->inp; | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { | if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
sbunlock(sb); | sbunlock(sb); | ||||
error = ECONNRESET; | error = ECONNRESET; | ||||
goto out; | goto out; | ||||
} | } | ||||
refcount_acquire(&ab->refcount); | job->aio_sent += m_length(m, NULL); | ||||
m_extadd(m, NULL, ab->ps.len, t4_aiotx_mbuf_free, ab, | |||||
(void *)(uintptr_t)job->aio_sent, 0, EXT_NET_DRV); | |||||
m->m_ext.ext_flags |= EXT_FLAG_AIOTX; | |||||
job->aio_sent += m->m_len; | |||||
sbappendstream(sb, m, 0); | sbappendstream(sb, m, 0); | ||||
m = NULL; | m = NULL; | ||||
if (!(inp->inp_flags & INP_DROPPED)) { | if (!(inp->inp_flags & INP_DROPPED)) { | ||||
tp = intotcpcb(inp); | tp = intotcpcb(inp); | ||||
if (moretocome) | if (moretocome) | ||||
tp->t_flags |= TF_MORETOCOME; | tp->t_flags |= TF_MORETOCOME; | ||||
error = tp->t_fb->tfb_tcp_output(tp); | error = tp->t_fb->tfb_tcp_output(tp); | ||||
if (moretocome) | if (moretocome) | ||||
tp->t_flags &= ~TF_MORETOCOME; | tp->t_flags &= ~TF_MORETOCOME; | ||||
} | } | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
if (sendmore) | if (sendmore) | ||||
goto sendanother; | goto sendanother; | ||||
sbunlock(sb); | sbunlock(sb); | ||||
if (error) | if (error) | ||||
goto out; | goto out; | ||||
/* | /* | ||||
* If this is a non-blocking socket and the request has not | * If this is a blocking socket and the request has not been | ||||
* been fully completed, requeue it until the socket is ready | * fully completed, requeue it until the socket is ready | ||||
* again. | * again. | ||||
*/ | */ | ||||
if (job->aio_sent < job->uaiocb.aio_nbytes && | if (job->aio_sent < job->uaiocb.aio_nbytes && | ||||
!(so->so_state & SS_NBIO)) { | !(so->so_state & SS_NBIO)) { | ||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { | if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { | ||||
SOCKBUF_UNLOCK(sb); | SOCKBUF_UNLOCK(sb); | ||||
error = ECANCELED; | error = ECANCELED; | ||||
goto out; | goto out; | ||||
} | } | ||||
TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); | TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); | ||||
return; | return; | ||||
} | } | ||||
/* | /* | ||||
* If the request will not be requeued, drop a reference on | * If the request will not be requeued, drop the queue's | ||||
* the aiotx buffer. Any mbufs in flight should still | * reference to the job. Any mbufs in flight should still | ||||
* contain a reference, but this drops the reference that the | * hold a reference, but this drops the reference that the | ||||
* job owns while it is waiting to queue mbufs to the socket. | * queue owns while it is waiting to queue mbufs to the | ||||
* socket. | |||||
*/ | */ | ||||
free_aiotx_buffer(ab); | aiotx_free_job(job); | ||||
out: | out: | ||||
if (error) { | if (error) { | ||||
if (ab != NULL) { | job->aio_error = (void *)(intptr_t)error; | ||||
job->aio_error = error; | aiotx_free_job(job); | ||||
free_aiotx_buffer(ab); | |||||
} else { | |||||
MPASS(job->aio_sent == 0); | |||||
aio_complete(job, -1, error); | |||||
} | } | ||||
} | |||||
if (m != NULL) | if (m != NULL) | ||||
m_free(m); | m_free(m); | ||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
} | } | ||||
static void | static void | ||||
t4_aiotx_task(void *context, int pending) | t4_aiotx_task(void *context, int pending) | ||||
{ | { | ||||
Show All 36 Lines | #endif | ||||
toep->aiotx_so = so; | toep->aiotx_so = so; | ||||
hold_toepcb(toep); | hold_toepcb(toep); | ||||
soaio_enqueue(&toep->aiotx_task); | soaio_enqueue(&toep->aiotx_task); | ||||
} | } | ||||
static void | static void | ||||
t4_aiotx_cancel(struct kaiocb *job) | t4_aiotx_cancel(struct kaiocb *job) | ||||
{ | { | ||||
struct aiotx_buffer *ab; | |||||
struct socket *so; | struct socket *so; | ||||
struct sockbuf *sb; | struct sockbuf *sb; | ||||
struct tcpcb *tp; | struct tcpcb *tp; | ||||
struct toepcb *toep; | struct toepcb *toep; | ||||
so = job->fd_file->f_data; | so = job->fd_file->f_data; | ||||
tp = so_sototcpcb(so); | tp = so_sototcpcb(so); | ||||
toep = tp->t_toe; | toep = tp->t_toe; | ||||
MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); | MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); | ||||
sb = &so->so_snd; | sb = &so->so_snd; | ||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
if (!aio_cancel_cleared(job)) | if (!aio_cancel_cleared(job)) | ||||
TAILQ_REMOVE(&toep->aiotx_jobq, job, list); | TAILQ_REMOVE(&toep->aiotx_jobq, job, list); | ||||
SOCKBUF_UNLOCK(sb); | SOCKBUF_UNLOCK(sb); | ||||
ab = job->backend1; | job->aio_error = (void *)(intptr_t)ECANCELED; | ||||
if (ab != NULL) | aiotx_free_job(job); | ||||
free_aiotx_buffer(ab); | |||||
else | |||||
aio_cancel(job); | |||||
} | } | ||||
int | int | ||||
t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job) | t4_aio_queue_aiotx(struct socket *so, struct kaiocb *job) | ||||
{ | { | ||||
struct tcpcb *tp = so_sototcpcb(so); | struct tcpcb *tp = so_sototcpcb(so); | ||||
struct toepcb *toep = tp->t_toe; | struct toepcb *toep = tp->t_toe; | ||||
struct adapter *sc = td_adapter(toep->td); | struct adapter *sc = td_adapter(toep->td); | ||||
Show All 9 Lines | if (tls_tx_key(toep)) | ||||
return (EOPNOTSUPP); | return (EOPNOTSUPP); | ||||
SOCKBUF_LOCK(&so->so_snd); | SOCKBUF_LOCK(&so->so_snd); | ||||
#ifdef VERBOSE_TRACES | #ifdef VERBOSE_TRACES | ||||
CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid); | CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid); | ||||
#endif | #endif | ||||
if (!aio_set_cancel_function(job, t4_aiotx_cancel)) | if (!aio_set_cancel_function(job, t4_aiotx_cancel)) | ||||
panic("new job was cancelled"); | panic("new job was cancelled"); | ||||
refcount_init(&job->aio_refs, 1); | |||||
TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); | TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); | ||||
if (sowriteable(so)) | if (sowriteable(so)) | ||||
t4_aiotx_queue_toep(so, toep); | t4_aiotx_queue_toep(so, toep); | ||||
SOCKBUF_UNLOCK(&so->so_snd); | SOCKBUF_UNLOCK(&so->so_snd); | ||||
return (0); | return (0); | ||||
} | } | ||||
void | void | ||||
aiotx_init_toep(struct toepcb *toep) | aiotx_init_toep(struct toepcb *toep) | ||||
{ | { | ||||
TAILQ_INIT(&toep->aiotx_jobq); | TAILQ_INIT(&toep->aiotx_jobq); | ||||
TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep); | TASK_INIT(&toep->aiotx_task, 0, t4_aiotx_task, toep); | ||||
} | } | ||||
#endif | #endif |
This is effectively vm_unhold_pages(), but it has to use PHYS_TO_VM_PAGE to get back to the page pointer.