diff --git a/sys/dev/nvmf/controller/ctl_frontend_nvmf.c b/sys/dev/nvmf/controller/ctl_frontend_nvmf.c --- a/sys/dev/nvmf/controller/ctl_frontend_nvmf.c +++ b/sys/dev/nvmf/controller/ctl_frontend_nvmf.c @@ -19,7 +19,9 @@ #include #include #include +#include #include +#include #include #include @@ -31,8 +33,10 @@ #include #include +#include #include #include +#include /* * Store pointers to the capsule and qpair in the two pointer members @@ -47,6 +51,9 @@ int flag, struct thread *td); static int nvmft_shutdown(void); +extern struct ctl_softc *control_softc; + +static struct taskqueue *nvmft_taskq; static TAILQ_HEAD(, nvmft_port) nvmft_ports; static struct sx nvmft_ports_lock; @@ -458,8 +465,8 @@ ctl_datamove_done((union ctl_io *)ctnio, true); } -static void -nvmft_datamove(union ctl_io *io) +void +nvmft_handle_datamove(union ctl_io *io) { struct nvmf_capsule *nc; struct nvmft_qpair *qp; @@ -478,6 +485,35 @@ nvmft_datamove_out(&io->nvmeio, qp, nc); } +void +nvmft_abort_datamove(union ctl_io *io) +{ + io->io_hdr.port_status = 1; + io->io_hdr.flags |= CTL_FLAG_ABORT; + ctl_datamove_done(io, true); +} + +static void +nvmft_datamove(union ctl_io *io) +{ + struct nvmft_qpair *qp; + + qp = NVMFT_QP(io); + nvmft_qpair_datamove(qp, io); +} + +void +nvmft_enqueue_task(struct task *task) +{ + taskqueue_enqueue(nvmft_taskq, task); +} + +void +nvmft_drain_task(struct task *task) +{ + taskqueue_drain(nvmft_taskq, task); +} + static void hip_add(uint64_t pair[2], uint64_t addend) { @@ -561,6 +597,17 @@ static int nvmft_init(void) { + int error; + + nvmft_taskq = taskqueue_create("nvmft", M_WAITOK, + taskqueue_thread_enqueue, &nvmft_taskq); + error = taskqueue_start_threads_in_proc(&nvmft_taskq, mp_ncpus, PWAIT, + control_softc->ctl_proc, "nvmft"); + if (error != 0) { + taskqueue_free(nvmft_taskq); + return (error); + } + TAILQ_INIT(&nvmft_ports); sx_init(&nvmft_ports_lock, "nvmft ports"); return (0); @@ -1115,6 +1162,7 @@ if (!TAILQ_EMPTY(&nvmft_ports)) return (EBUSY); + taskqueue_free(nvmft_taskq); sx_destroy(&nvmft_ports_lock); return (0); } diff --git a/sys/dev/nvmf/controller/nvmft_qpair.c b/sys/dev/nvmf/controller/nvmft_qpair.c --- a/sys/dev/nvmf/controller/nvmft_qpair.c +++ b/sys/dev/nvmf/controller/nvmft_qpair.c @@ -34,6 +34,9 @@ uint16_t sqtail; volatile u_int qp_refs; /* Internal references on 'qp'. */ + struct task datamove_task; + STAILQ_HEAD(, ctl_io_hdr) datamove_queue; + struct mtx lock; char name[16]; @@ -41,6 +44,7 @@ static int _nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_status); +static void nvmft_datamove_task(void *context, int pending); static void nvmft_qpair_error(void *arg, int error) @@ -114,6 +118,8 @@ strlcpy(qp->name, name, sizeof(qp->name)); mtx_init(&qp->lock, "nvmft qp", NULL, MTX_DEF); qp->cids = BITSET_ALLOC(NUM_CIDS, M_NVMFT, M_WAITOK | M_ZERO); + STAILQ_INIT(&qp->datamove_queue); + TASK_INIT(&qp->datamove_task, 0, nvmft_datamove_task, qp); qp->qp = nvmf_allocate_qpair(trtype, true, handoff, nvmft_qpair_error, qp, nvmft_receive_capsule, qp); @@ -131,14 +137,25 @@ void nvmft_qpair_shutdown(struct nvmft_qpair *qp) { + STAILQ_HEAD(, ctl_io_hdr) datamove_queue; struct nvmf_qpair *nq; + union ctl_io *io; + STAILQ_INIT(&datamove_queue); mtx_lock(&qp->lock); nq = qp->qp; qp->qp = NULL; + STAILQ_CONCAT(&datamove_queue, &qp->datamove_queue); mtx_unlock(&qp->lock); if (nq != NULL && refcount_release(&qp->qp_refs)) nvmf_free_qpair(nq); + + while (!STAILQ_EMPTY(&datamove_queue)) { + io = (union ctl_io *)STAILQ_FIRST(&datamove_queue); + STAILQ_REMOVE_HEAD(&datamove_queue, links); + nvmft_abort_datamove(io); + } + nvmft_drain_task(&qp->datamove_task); } void @@ -359,3 +376,43 @@ rsp.status_code_specific.success.cntlid = htole16(ctrlr->cntlid); return (nvmft_send_connect_response(qp, &rsp)); } + +void +nvmft_qpair_datamove(struct nvmft_qpair *qp, union ctl_io *io) +{ + bool enqueue_task; + + mtx_lock(&qp->lock); + if (qp->qp == NULL) { + mtx_unlock(&qp->lock); + nvmft_abort_datamove(io); + return; + } + enqueue_task = STAILQ_EMPTY(&qp->datamove_queue); + STAILQ_INSERT_TAIL(&qp->datamove_queue, &io->io_hdr, links); + mtx_unlock(&qp->lock); + if (enqueue_task) + nvmft_enqueue_task(&qp->datamove_task); +} + +static void +nvmft_datamove_task(void *context, int pending __unused) +{ + struct nvmft_qpair *qp = context; + union ctl_io *io; + bool abort; + + mtx_lock(&qp->lock); + while (!STAILQ_EMPTY(&qp->datamove_queue)) { + io = (union ctl_io *)STAILQ_FIRST(&qp->datamove_queue); + STAILQ_REMOVE_HEAD(&qp->datamove_queue, links); + abort = (qp->qp == NULL); + mtx_unlock(&qp->lock); + if (abort) + nvmft_abort_datamove(io); + else + nvmft_handle_datamove(io); + mtx_lock(&qp->lock); + } + mtx_unlock(&qp->lock); +} diff --git a/sys/dev/nvmf/controller/nvmft_var.h b/sys/dev/nvmf/controller/nvmft_var.h --- a/sys/dev/nvmf/controller/nvmft_var.h +++ b/sys/dev/nvmf/controller/nvmft_var.h @@ -110,6 +110,10 @@ void nvmft_dispatch_command(struct nvmft_qpair *qp, struct nvmf_capsule *nc, bool admin); void nvmft_terminate_commands(struct nvmft_controller *ctrlr); +void nvmft_abort_datamove(union ctl_io *io); +void nvmft_handle_datamove(union ctl_io *io); +void nvmft_drain_task(struct task *task); +void nvmft_enqueue_task(struct task *task); /* nvmft_controller.c */ void nvmft_controller_error(struct nvmft_controller *ctrlr, @@ -138,6 +142,7 @@ void nvmft_qpair_shutdown(struct nvmft_qpair *qp); void nvmft_qpair_destroy(struct nvmft_qpair *qp); struct nvmft_controller *nvmft_qpair_ctrlr(struct nvmft_qpair *qp); +void nvmft_qpair_datamove(struct nvmft_qpair *qp, union ctl_io *io); uint16_t nvmft_qpair_id(struct nvmft_qpair *qp); const char *nvmft_qpair_name(struct nvmft_qpair *qp); void nvmft_command_completed(struct nvmft_qpair *qp,