diff --git a/sys/dev/nvmf/controller/ctl_frontend_nvmf.c b/sys/dev/nvmf/controller/ctl_frontend_nvmf.c index a203bb1c90a6..bc061947a9a0 100644 --- a/sys/dev/nvmf/controller/ctl_frontend_nvmf.c +++ b/sys/dev/nvmf/controller/ctl_frontend_nvmf.c @@ -1,1123 +1,1171 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023-2024 Chelsio Communications, Inc. * Written by: John Baldwin */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include +#include #include #include #include #include #include #include #include #include +#include #include #include +#include /* * Store pointers to the capsule and qpair in the two pointer members * of CTL_PRIV_FRONTEND. */ #define NVMFT_NC(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[0]) #define NVMFT_QP(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[1]) static void nvmft_done(union ctl_io *io); static int nvmft_init(void); static int nvmft_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int flag, struct thread *td); static int nvmft_shutdown(void); +extern struct ctl_softc *control_softc; + +static struct taskqueue *nvmft_taskq; static TAILQ_HEAD(, nvmft_port) nvmft_ports; static struct sx nvmft_ports_lock; MALLOC_DEFINE(M_NVMFT, "nvmft", "NVMe over Fabrics controller"); static struct ctl_frontend nvmft_frontend = { .name = "nvmf", .init = nvmft_init, .ioctl = nvmft_ioctl, .fe_dump = NULL, .shutdown = nvmft_shutdown, }; static void nvmft_online(void *arg) { struct nvmft_port *np = arg; sx_xlock(&np->lock); np->online = true; sx_xunlock(&np->lock); } static void nvmft_offline(void *arg) { struct nvmft_port *np = arg; struct nvmft_controller *ctrlr; sx_xlock(&np->lock); np->online = false; TAILQ_FOREACH(ctrlr, &np->controllers, link) { nvmft_printf(ctrlr, "shutting down due to port going offline\n"); nvmft_controller_error(ctrlr, NULL, ENODEV); } while (!TAILQ_EMPTY(&np->controllers)) sx_sleep(np, &np->lock, 0, "nvmfoff", 0); sx_xunlock(&np->lock); } static int nvmft_lun_enable(void *arg, int lun_id) { struct nvmft_port *np = arg; struct nvmft_controller *ctrlr; uint32_t *old_ns, *new_ns; uint32_t nsid; u_int i; if (lun_id >= le32toh(np->cdata.nn)) { printf("NVMFT: %s lun %d larger than maximum nsid %u\n", np->cdata.subnqn, lun_id, le32toh(np->cdata.nn)); return (EOPNOTSUPP); } nsid = lun_id + 1; sx_xlock(&np->lock); new_ns = mallocarray(np->num_ns + 1, sizeof(*new_ns), M_NVMFT, M_WAITOK); for (i = 0; i < np->num_ns; i++) { if (np->active_ns[i] < nsid) continue; if (np->active_ns[i] == nsid) { sx_xunlock(&np->lock); free(new_ns, M_NVMFT); printf("NVMFT: %s duplicate lun %d\n", np->cdata.subnqn, lun_id); return (EINVAL); } break; } /* Copy over IDs smaller than nsid. */ memcpy(new_ns, np->active_ns, i * sizeof(*np->active_ns)); /* Insert nsid. */ new_ns[i] = nsid; /* Copy over IDs greater than nsid. */ memcpy(new_ns + i + 1, np->active_ns + i, (np->num_ns - i) * sizeof(*np->active_ns)); np->num_ns++; old_ns = np->active_ns; np->active_ns = new_ns; TAILQ_FOREACH(ctrlr, &np->controllers, link) { nvmft_controller_lun_changed(ctrlr, lun_id); } sx_xunlock(&np->lock); free(old_ns, M_NVMFT); return (0); } static int nvmft_lun_disable(void *arg, int lun_id) { struct nvmft_port *np = arg; struct nvmft_controller *ctrlr; uint32_t nsid; u_int i; if (lun_id >= le32toh(np->cdata.nn)) return (0); nsid = lun_id + 1; sx_xlock(&np->lock); for (i = 0; i < np->num_ns; i++) { if (np->active_ns[i] == nsid) goto found; } sx_xunlock(&np->lock); printf("NVMFT: %s request to disable nonexistent lun %d\n", np->cdata.subnqn, lun_id); return (EINVAL); found: /* Move down IDs greater than nsid. */ memmove(np->active_ns + i, np->active_ns + i + 1, (np->num_ns - (i + 1)) * sizeof(*np->active_ns)); np->num_ns--; /* NB: Don't bother freeing the old active_ns array. */ TAILQ_FOREACH(ctrlr, &np->controllers, link) { nvmft_controller_lun_changed(ctrlr, lun_id); } sx_xunlock(&np->lock); return (0); } void nvmft_populate_active_nslist(struct nvmft_port *np, uint32_t nsid, struct nvme_ns_list *nslist) { u_int i, count; sx_slock(&np->lock); count = 0; for (i = 0; i < np->num_ns; i++) { if (np->active_ns[i] <= nsid) continue; nslist->ns[count] = htole32(np->active_ns[i]); count++; if (count == nitems(nslist->ns)) break; } sx_sunlock(&np->lock); } void nvmft_dispatch_command(struct nvmft_qpair *qp, struct nvmf_capsule *nc, bool admin) { struct nvmft_controller *ctrlr = nvmft_qpair_ctrlr(qp); const struct nvme_command *cmd = nvmf_capsule_sqe(nc); struct nvmft_port *np = ctrlr->np; union ctl_io *io; int error; if (cmd->nsid == htole32(0)) { nvmft_send_generic_error(qp, nc, NVME_SC_INVALID_NAMESPACE_OR_FORMAT); nvmf_free_capsule(nc); return; } mtx_lock(&ctrlr->lock); if (ctrlr->pending_commands == 0) ctrlr->start_busy = sbinuptime(); ctrlr->pending_commands++; mtx_unlock(&ctrlr->lock); io = ctl_alloc_io(np->port.ctl_pool_ref); ctl_zero_io(io); NVMFT_NC(io) = nc; NVMFT_QP(io) = qp; io->io_hdr.io_type = admin ? CTL_IO_NVME_ADMIN : CTL_IO_NVME; io->io_hdr.nexus.initid = ctrlr->cntlid; io->io_hdr.nexus.targ_port = np->port.targ_port; io->io_hdr.nexus.targ_lun = le32toh(cmd->nsid) - 1; io->nvmeio.cmd = *cmd; error = ctl_run(io); if (error != 0) { nvmft_printf(ctrlr, "ctl_run failed for command on %s: %d\n", nvmft_qpair_name(qp), error); ctl_nvme_set_generic_error(&io->nvmeio, NVME_SC_INTERNAL_DEVICE_ERROR); nvmft_done(io); nvmft_controller_error(ctrlr, qp, ENXIO); } } void nvmft_terminate_commands(struct nvmft_controller *ctrlr) { struct nvmft_port *np = ctrlr->np; union ctl_io *io; int error; mtx_lock(&ctrlr->lock); if (ctrlr->pending_commands == 0) ctrlr->start_busy = sbinuptime(); ctrlr->pending_commands++; mtx_unlock(&ctrlr->lock); io = ctl_alloc_io(np->port.ctl_pool_ref); ctl_zero_io(io); NVMFT_QP(io) = ctrlr->admin; io->io_hdr.io_type = CTL_IO_TASK; io->io_hdr.nexus.initid = ctrlr->cntlid; io->io_hdr.nexus.targ_port = np->port.targ_port; io->io_hdr.nexus.targ_lun = 0; io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX: unused? */ io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; error = ctl_run(io); if (error != CTL_RETVAL_COMPLETE) { nvmft_printf(ctrlr, "failed to terminate tasks: %d\n", error); #ifdef INVARIANTS io->io_hdr.status = CTL_SUCCESS; #endif nvmft_done(io); } } static void nvmft_datamove_out_cb(void *arg, size_t xfered, int error) { struct ctl_nvmeio *ctnio = arg; if (error != 0) { ctl_nvme_set_data_transfer_error(ctnio); } else { MPASS(xfered == ctnio->kern_data_len); ctnio->kern_data_resid -= xfered; } if (ctnio->kern_sg_entries) { free(ctnio->ext_data_ptr, M_NVMFT); ctnio->ext_data_ptr = NULL; } else MPASS(ctnio->ext_data_ptr == NULL); ctl_datamove_done((union ctl_io *)ctnio, false); } static void nvmft_datamove_out(struct ctl_nvmeio *ctnio, struct nvmft_qpair *qp, struct nvmf_capsule *nc) { struct memdesc mem; int error; MPASS(ctnio->ext_data_ptr == NULL); if (ctnio->kern_sg_entries > 0) { struct ctl_sg_entry *sgl; struct bus_dma_segment *vlist; vlist = mallocarray(ctnio->kern_sg_entries, sizeof(*vlist), M_NVMFT, M_WAITOK); ctnio->ext_data_ptr = (void *)vlist; sgl = (struct ctl_sg_entry *)ctnio->kern_data_ptr; for (u_int i = 0; i < ctnio->kern_sg_entries; i++) { vlist[i].ds_addr = (uintptr_t)sgl[i].addr; vlist[i].ds_len = sgl[i].len; } mem = memdesc_vlist(vlist, ctnio->kern_sg_entries); } else mem = memdesc_vaddr(ctnio->kern_data_ptr, ctnio->kern_data_len); error = nvmf_receive_controller_data(nc, ctnio->kern_rel_offset, &mem, ctnio->kern_data_len, nvmft_datamove_out_cb, ctnio); if (error == 0) return; nvmft_printf(nvmft_qpair_ctrlr(qp), "Failed to request capsule data: %d\n", error); ctl_nvme_set_data_transfer_error(ctnio); if (ctnio->kern_sg_entries) { free(ctnio->ext_data_ptr, M_NVMFT); ctnio->ext_data_ptr = NULL; } else MPASS(ctnio->ext_data_ptr == NULL); ctl_datamove_done((union ctl_io *)ctnio, true); } static struct mbuf * nvmft_copy_data(struct ctl_nvmeio *ctnio) { struct ctl_sg_entry *sgl; struct mbuf *m0, *m; uint32_t resid, off, todo; int mlen; MPASS(ctnio->kern_data_len != 0); m0 = m_getm2(NULL, ctnio->kern_data_len, M_WAITOK, MT_DATA, 0); if (ctnio->kern_sg_entries == 0) { m_copyback(m0, 0, ctnio->kern_data_len, ctnio->kern_data_ptr); return (m0); } resid = ctnio->kern_data_len; sgl = (struct ctl_sg_entry *)ctnio->kern_data_ptr; off = 0; m = m0; mlen = M_TRAILINGSPACE(m); for (;;) { todo = MIN(mlen, sgl->len - off); memcpy(mtod(m, char *) + m->m_len, (char *)sgl->addr + off, todo); m->m_len += todo; resid -= todo; if (resid == 0) { MPASS(m->m_next == NULL); break; } off += todo; if (off == sgl->len) { sgl++; off = 0; } mlen -= todo; if (mlen == 0) { m = m->m_next; mlen = M_TRAILINGSPACE(m); } } return (m0); } static void m_free_ref_data(struct mbuf *m) { ctl_ref kern_data_ref = m->m_ext.ext_arg1; kern_data_ref(m->m_ext.ext_arg2, -1); } static struct mbuf * m_get_ref_data(struct ctl_nvmeio *ctnio, void *buf, u_int size) { struct mbuf *m; m = m_get(M_WAITOK, MT_DATA); m_extadd(m, buf, size, m_free_ref_data, ctnio->kern_data_ref, ctnio->kern_data_arg, M_RDONLY, EXT_CTL); m->m_len = size; ctnio->kern_data_ref(ctnio->kern_data_arg, 1); return (m); } static struct mbuf * nvmft_ref_data(struct ctl_nvmeio *ctnio) { struct ctl_sg_entry *sgl; struct mbuf *m0, *m; MPASS(ctnio->kern_data_len != 0); if (ctnio->kern_sg_entries == 0) return (m_get_ref_data(ctnio, ctnio->kern_data_ptr, ctnio->kern_data_len)); sgl = (struct ctl_sg_entry *)ctnio->kern_data_ptr; m0 = m_get_ref_data(ctnio, sgl[0].addr, sgl[0].len); m = m0; for (u_int i = 1; i < ctnio->kern_sg_entries; i++) { m->m_next = m_get_ref_data(ctnio, sgl[i].addr, sgl[i].len); m = m->m_next; } return (m0); } static void nvmft_datamove_in(struct ctl_nvmeio *ctnio, struct nvmft_qpair *qp, struct nvmf_capsule *nc) { struct mbuf *m; u_int status; if (ctnio->kern_data_ref != NULL) m = nvmft_ref_data(ctnio); else m = nvmft_copy_data(ctnio); status = nvmf_send_controller_data(nc, ctnio->kern_rel_offset, m, ctnio->kern_data_len); switch (status) { case NVMF_SUCCESS_SENT: ctnio->success_sent = true; nvmft_command_completed(qp, nc); /* FALLTHROUGH */ case NVMF_MORE: case NVME_SC_SUCCESS: break; default: ctl_nvme_set_generic_error(ctnio, status); break; } ctl_datamove_done((union ctl_io *)ctnio, true); } -static void -nvmft_datamove(union ctl_io *io) +void +nvmft_handle_datamove(union ctl_io *io) { struct nvmf_capsule *nc; struct nvmft_qpair *qp; /* Some CTL commands preemptively set a success status. */ MPASS(io->io_hdr.status == CTL_STATUS_NONE || io->io_hdr.status == CTL_SUCCESS); MPASS(!io->nvmeio.success_sent); nc = NVMFT_NC(io); qp = NVMFT_QP(io); if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) nvmft_datamove_in(&io->nvmeio, qp, nc); else nvmft_datamove_out(&io->nvmeio, qp, nc); } +void +nvmft_abort_datamove(union ctl_io *io) +{ + io->io_hdr.port_status = 1; + io->io_hdr.flags |= CTL_FLAG_ABORT; + ctl_datamove_done(io, true); +} + +static void +nvmft_datamove(union ctl_io *io) +{ + struct nvmft_qpair *qp; + + qp = NVMFT_QP(io); + nvmft_qpair_datamove(qp, io); +} + +void +nvmft_enqueue_task(struct task *task) +{ + taskqueue_enqueue(nvmft_taskq, task); +} + +void +nvmft_drain_task(struct task *task) +{ + taskqueue_drain(nvmft_taskq, task); +} + static void hip_add(uint64_t pair[2], uint64_t addend) { uint64_t old, new; old = le64toh(pair[0]); new = old + addend; pair[0] = htole64(new); if (new < old) pair[1] += htole64(1); } static void nvmft_done(union ctl_io *io) { struct nvmft_controller *ctrlr; const struct nvme_command *cmd; struct nvmft_qpair *qp; struct nvmf_capsule *nc; size_t len; KASSERT(io->io_hdr.status == CTL_SUCCESS || io->io_hdr.status == CTL_NVME_ERROR, ("%s: bad status %u", __func__, io->io_hdr.status)); nc = NVMFT_NC(io); qp = NVMFT_QP(io); ctrlr = nvmft_qpair_ctrlr(qp); if (nc == NULL) { /* Completion of nvmft_terminate_commands. */ goto end; } cmd = nvmf_capsule_sqe(nc); if (io->io_hdr.status == CTL_SUCCESS) len = nvmf_capsule_data_len(nc) / 512; else len = 0; switch (cmd->opc) { case NVME_OPC_WRITE: mtx_lock(&ctrlr->lock); hip_add(ctrlr->hip.host_write_commands, 1); len += ctrlr->partial_duw; if (len > 1000) hip_add(ctrlr->hip.data_units_written, len / 1000); ctrlr->partial_duw = len % 1000; mtx_unlock(&ctrlr->lock); break; case NVME_OPC_READ: case NVME_OPC_COMPARE: case NVME_OPC_VERIFY: mtx_lock(&ctrlr->lock); if (cmd->opc != NVME_OPC_VERIFY) hip_add(ctrlr->hip.host_read_commands, 1); len += ctrlr->partial_dur; if (len > 1000) hip_add(ctrlr->hip.data_units_read, len / 1000); ctrlr->partial_dur = len % 1000; mtx_unlock(&ctrlr->lock); break; } if (io->nvmeio.success_sent) { MPASS(io->io_hdr.status == CTL_SUCCESS); } else { io->nvmeio.cpl.cid = cmd->cid; nvmft_send_response(qp, &io->nvmeio.cpl); } nvmf_free_capsule(nc); end: ctl_free_io(io); mtx_lock(&ctrlr->lock); ctrlr->pending_commands--; if (ctrlr->pending_commands == 0) ctrlr->busy_total += sbinuptime() - ctrlr->start_busy; mtx_unlock(&ctrlr->lock); } static int nvmft_init(void) { + int error; + + nvmft_taskq = taskqueue_create("nvmft", M_WAITOK, + taskqueue_thread_enqueue, &nvmft_taskq); + error = taskqueue_start_threads_in_proc(&nvmft_taskq, mp_ncpus, PWAIT, + control_softc->ctl_proc, "nvmft"); + if (error != 0) { + taskqueue_free(nvmft_taskq); + return (error); + } + TAILQ_INIT(&nvmft_ports); sx_init(&nvmft_ports_lock, "nvmft ports"); return (0); } void nvmft_port_free(struct nvmft_port *np) { KASSERT(TAILQ_EMPTY(&np->controllers), ("%s(%p): active controllers", __func__, np)); if (np->port.targ_port != -1) { if (ctl_port_deregister(&np->port) != 0) printf("%s: ctl_port_deregister() failed\n", __func__); } free(np->active_ns, M_NVMFT); clean_unrhdr(np->ids); delete_unrhdr(np->ids); sx_destroy(&np->lock); free(np, M_NVMFT); } static struct nvmft_port * nvmft_port_find(const char *subnqn) { struct nvmft_port *np; KASSERT(nvmf_nqn_valid(subnqn), ("%s: invalid nqn", __func__)); sx_assert(&nvmft_ports_lock, SA_LOCKED); TAILQ_FOREACH(np, &nvmft_ports, link) { if (strcmp(np->cdata.subnqn, subnqn) == 0) break; } return (np); } static struct nvmft_port * nvmft_port_find_by_id(int port_id) { struct nvmft_port *np; sx_assert(&nvmft_ports_lock, SA_LOCKED); TAILQ_FOREACH(np, &nvmft_ports, link) { if (np->port.targ_port == port_id) break; } return (np); } /* * Helper function to fetch a number stored as a string in an nv_list. * Returns false if the string was not a valid number. */ static bool dnvlist_get_strnum(nvlist_t *nvl, const char *name, u_long default_value, u_long *value) { const char *str; char *cp; str = dnvlist_get_string(nvl, name, NULL); if (str == NULL) { *value = default_value; return (true); } if (*str == '\0') return (false); *value = strtoul(str, &cp, 0); if (*cp != '\0') return (false); return (true); } /* * NVMeoF ports support the following parameters: * * Mandatory: * * subnqn: subsystem NVMe Qualified Name * portid: integer port ID from Discovery Log Page entry * * Optional: * serial: Serial Number string * max_io_qsize: Maximum number of I/O queue entries * enable_timeout: Timeout for controller enable in milliseconds * ioccsz: Maximum command capsule size * iorcsz: Maximum response capsule size * nn: Number of namespaces */ static void nvmft_port_create(struct ctl_req *req) { struct nvmft_port *np; struct ctl_port *port; const char *serial, *subnqn; char serial_buf[NVME_SERIAL_NUMBER_LENGTH]; u_long enable_timeout, hostid, ioccsz, iorcsz, max_io_qsize, nn, portid; int error; /* Required parameters. */ subnqn = dnvlist_get_string(req->args_nvl, "subnqn", NULL); if (subnqn == NULL || !nvlist_exists_string(req->args_nvl, "portid")) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Missing required argument"); return; } if (!nvmf_nqn_valid(subnqn)) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid SubNQN"); return; } if (!dnvlist_get_strnum(req->args_nvl, "portid", UINT16_MAX, &portid) || portid > UINT16_MAX) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid port ID"); return; } /* Optional parameters. */ if (!dnvlist_get_strnum(req->args_nvl, "max_io_qsize", NVMF_MAX_IO_ENTRIES, &max_io_qsize) || max_io_qsize < NVME_MIN_IO_ENTRIES || max_io_qsize > NVME_MAX_IO_ENTRIES) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid maximum I/O queue size"); return; } if (!dnvlist_get_strnum(req->args_nvl, "enable_timeout", NVMF_CC_EN_TIMEOUT * 500, &enable_timeout) || (enable_timeout % 500) != 0 || (enable_timeout / 500) > 255) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid enable timeout"); return; } if (!dnvlist_get_strnum(req->args_nvl, "ioccsz", NVMF_IOCCSZ, &ioccsz) || ioccsz < sizeof(struct nvme_command) || (ioccsz % 16) != 0) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid Command Capsule size"); return; } if (!dnvlist_get_strnum(req->args_nvl, "iorcsz", NVMF_IORCSZ, &iorcsz) || iorcsz < sizeof(struct nvme_completion) || (iorcsz % 16) != 0) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid Response Capsule size"); return; } if (!dnvlist_get_strnum(req->args_nvl, "nn", NVMF_NN, &nn) || nn < 1 || nn > UINT32_MAX) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid number of namespaces"); return; } serial = dnvlist_get_string(req->args_nvl, "serial", NULL); if (serial == NULL) { getcredhostid(curthread->td_ucred, &hostid); nvmf_controller_serial(serial_buf, sizeof(serial_buf), hostid); serial = serial_buf; } sx_xlock(&nvmft_ports_lock); np = nvmft_port_find(subnqn); if (np != NULL) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "SubNQN \"%s\" already exists", subnqn); sx_xunlock(&nvmft_ports_lock); return; } np = malloc(sizeof(*np), M_NVMFT, M_WAITOK | M_ZERO); refcount_init(&np->refs, 1); np->max_io_qsize = max_io_qsize; np->cap = _nvmf_controller_cap(max_io_qsize, enable_timeout / 500); sx_init(&np->lock, "nvmft port"); np->ids = new_unrhdr(0, MIN(CTL_MAX_INIT_PER_PORT - 1, NVMF_CNTLID_STATIC_MAX), UNR_NO_MTX); TAILQ_INIT(&np->controllers); /* The controller ID is set later for individual controllers. */ _nvmf_init_io_controller_data(0, max_io_qsize, serial, ostype, osrelease, subnqn, nn, ioccsz, iorcsz, &np->cdata); np->cdata.aerl = NVMFT_NUM_AER - 1; np->cdata.oaes = htole32(NVME_ASYNC_EVENT_NS_ATTRIBUTE); np->cdata.oncs = htole16(NVMEF(NVME_CTRLR_DATA_ONCS_VERIFY, 1) | NVMEF(NVME_CTRLR_DATA_ONCS_WRZERO, 1) | NVMEF(NVME_CTRLR_DATA_ONCS_DSM, 1) | NVMEF(NVME_CTRLR_DATA_ONCS_COMPARE, 1)); np->cdata.fuses = NVMEF(NVME_CTRLR_DATA_FUSES_CNW, 1); np->fp.afi = NVMEF(NVME_FIRMWARE_PAGE_AFI_SLOT, 1); memcpy(np->fp.revision[0], np->cdata.fr, sizeof(np->cdata.fr)); port = &np->port; port->frontend = &nvmft_frontend; port->port_type = CTL_PORT_NVMF; port->num_requested_ctl_io = max_io_qsize; port->port_name = "nvmf"; port->physical_port = portid; port->virtual_port = 0; port->port_online = nvmft_online; port->port_offline = nvmft_offline; port->onoff_arg = np; port->lun_enable = nvmft_lun_enable; port->lun_disable = nvmft_lun_disable; port->targ_lun_arg = np; port->fe_datamove = nvmft_datamove; port->fe_done = nvmft_done; port->targ_port = -1; port->options = nvlist_clone(req->args_nvl); error = ctl_port_register(port); if (error != 0) { sx_xunlock(&nvmft_ports_lock); nvlist_destroy(port->options); nvmft_port_rele(np); req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Failed to register CTL port with error %d", error); return; } TAILQ_INSERT_TAIL(&nvmft_ports, np, link); sx_xunlock(&nvmft_ports_lock); req->status = CTL_LUN_OK; req->result_nvl = nvlist_create(0); nvlist_add_number(req->result_nvl, "port_id", port->targ_port); } static void nvmft_port_remove(struct ctl_req *req) { struct nvmft_port *np; const char *subnqn; u_long port_id; /* * ctladm port -r just provides the port_id, so permit looking * up a port either by "subnqn" or "port_id". */ port_id = ULONG_MAX; subnqn = dnvlist_get_string(req->args_nvl, "subnqn", NULL); if (subnqn == NULL) { if (!nvlist_exists_string(req->args_nvl, "port_id")) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Missing required argument"); return; } if (!dnvlist_get_strnum(req->args_nvl, "port_id", ULONG_MAX, &port_id)) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Invalid CTL port ID"); return; } } else { if (nvlist_exists_string(req->args_nvl, "port_id")) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Ambiguous port removal request"); return; } } sx_xlock(&nvmft_ports_lock); if (subnqn != NULL) { np = nvmft_port_find(subnqn); if (np == NULL) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "SubNQN \"%s\" does not exist", subnqn); sx_xunlock(&nvmft_ports_lock); return; } } else { np = nvmft_port_find_by_id(port_id); if (np == NULL) { req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "CTL port %lu is not a NVMF port", port_id); sx_xunlock(&nvmft_ports_lock); return; } } TAILQ_REMOVE(&nvmft_ports, np, link); sx_xunlock(&nvmft_ports_lock); ctl_port_offline(&np->port); nvmft_port_rele(np); req->status = CTL_LUN_OK; } static void nvmft_handoff(struct ctl_nvmf *cn) { struct nvmf_fabric_connect_cmd cmd; struct nvmf_handoff_controller_qpair *handoff; struct nvmf_fabric_connect_data *data; struct nvmft_port *np; int error; np = NULL; data = NULL; handoff = &cn->data.handoff; error = copyin(handoff->cmd, &cmd, sizeof(cmd)); if (error != 0) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Failed to copyin CONNECT SQE"); return; } data = malloc(sizeof(*data), M_NVMFT, M_WAITOK); error = copyin(handoff->data, data, sizeof(*data)); if (error != 0) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Failed to copyin CONNECT data"); goto out; } if (!nvmf_nqn_valid(data->subnqn)) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Invalid SubNQN"); goto out; } sx_slock(&nvmft_ports_lock); np = nvmft_port_find(data->subnqn); if (np == NULL) { sx_sunlock(&nvmft_ports_lock); cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Unknown SubNQN"); goto out; } if (!np->online) { sx_sunlock(&nvmft_ports_lock); cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "CTL port offline"); np = NULL; goto out; } nvmft_port_ref(np); sx_sunlock(&nvmft_ports_lock); if (handoff->params.admin) { error = nvmft_handoff_admin_queue(np, handoff, &cmd, data); if (error != 0) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Failed to handoff admin queue: %d", error); goto out; } } else { error = nvmft_handoff_io_queue(np, handoff, &cmd, data); if (error != 0) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Failed to handoff admin queue: %d", error); goto out; } } cn->status = CTL_NVMF_OK; out: if (np != NULL) nvmft_port_rele(np); free(data, M_NVMFT); } static void nvmft_list(struct ctl_nvmf *cn) { struct ctl_nvmf_list_params *lp; struct nvmft_controller *ctrlr; struct nvmft_port *np; struct sbuf *sb; int error; lp = &cn->data.list; sb = sbuf_new(NULL, NULL, lp->alloc_len, SBUF_FIXEDLEN | SBUF_INCLUDENUL); if (sb == NULL) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Failed to allocate NVMeoF session list"); return; } sbuf_printf(sb, "\n"); sx_slock(&nvmft_ports_lock); TAILQ_FOREACH(np, &nvmft_ports, link) { sx_slock(&np->lock); TAILQ_FOREACH(ctrlr, &np->controllers, link) { sbuf_printf(sb, "" "%s" "%s" "%u" "\n", ctrlr->cntlid, ctrlr->hostnqn, np->cdata.subnqn, ctrlr->trtype); } sx_sunlock(&np->lock); } sx_sunlock(&nvmft_ports_lock); sbuf_printf(sb, "\n"); if (sbuf_finish(sb) != 0) { sbuf_delete(sb); cn->status = CTL_NVMF_LIST_NEED_MORE_SPACE; snprintf(cn->error_str, sizeof(cn->error_str), "Out of space, %d bytes is too small", lp->alloc_len); return; } error = copyout(sbuf_data(sb), lp->conn_xml, sbuf_len(sb)); if (error != 0) { sbuf_delete(sb); cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Failed to copyout session list: %d", error); return; } lp->fill_len = sbuf_len(sb); cn->status = CTL_NVMF_OK; sbuf_delete(sb); } static void nvmft_terminate(struct ctl_nvmf *cn) { struct ctl_nvmf_terminate_params *tp; struct nvmft_controller *ctrlr; struct nvmft_port *np; bool found, match; tp = &cn->data.terminate; found = false; sx_slock(&nvmft_ports_lock); TAILQ_FOREACH(np, &nvmft_ports, link) { sx_slock(&np->lock); TAILQ_FOREACH(ctrlr, &np->controllers, link) { if (tp->all != 0) match = true; else if (tp->cntlid != -1) match = tp->cntlid == ctrlr->cntlid; else if (tp->hostnqn[0] != '\0') match = strncmp(tp->hostnqn, ctrlr->hostnqn, sizeof(tp->hostnqn)) == 0; else match = false; if (!match) continue; nvmft_printf(ctrlr, "disconnecting due to administrative request\n"); nvmft_controller_error(ctrlr, NULL, ECONNABORTED); found = true; } sx_sunlock(&np->lock); } sx_sunlock(&nvmft_ports_lock); if (!found) { cn->status = CTL_NVMF_ASSOCIATION_NOT_FOUND; snprintf(cn->error_str, sizeof(cn->error_str), "No matching associations found"); return; } cn->status = CTL_NVMF_OK; } static int nvmft_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int flag, struct thread *td) { struct ctl_nvmf *cn; struct ctl_req *req; switch (cmd) { case CTL_PORT_REQ: req = (struct ctl_req *)data; switch (req->reqtype) { case CTL_REQ_CREATE: nvmft_port_create(req); break; case CTL_REQ_REMOVE: nvmft_port_remove(req); break; default: req->status = CTL_LUN_ERROR; snprintf(req->error_str, sizeof(req->error_str), "Unsupported request type %d", req->reqtype); break; } return (0); case CTL_NVMF: cn = (struct ctl_nvmf *)data; switch (cn->type) { case CTL_NVMF_HANDOFF: nvmft_handoff(cn); break; case CTL_NVMF_LIST: nvmft_list(cn); break; case CTL_NVMF_TERMINATE: nvmft_terminate(cn); break; default: cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), "Invalid NVMeoF request type %d", cn->type); break; } return (0); default: return (ENOTTY); } } static int nvmft_shutdown(void) { /* TODO: Need to check for active controllers. */ if (!TAILQ_EMPTY(&nvmft_ports)) return (EBUSY); + taskqueue_free(nvmft_taskq); sx_destroy(&nvmft_ports_lock); return (0); } CTL_FRONTEND_DECLARE(nvmft, nvmft_frontend); MODULE_DEPEND(nvmft, nvmf_transport, 1, 1, 1); diff --git a/sys/dev/nvmf/controller/nvmft_qpair.c b/sys/dev/nvmf/controller/nvmft_qpair.c index 6cb3ebd76884..e66d98f38225 100644 --- a/sys/dev/nvmf/controller/nvmft_qpair.c +++ b/sys/dev/nvmf/controller/nvmft_qpair.c @@ -1,361 +1,418 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023-2024 Chelsio Communications, Inc. * Written by: John Baldwin */ #include #include #include #include #include #include #include /* * A bitmask of command ID values. This is used to detect duplicate * commands with the same ID. */ #define NUM_CIDS (UINT16_MAX + 1) BITSET_DEFINE(cidset, NUM_CIDS); struct nvmft_qpair { struct nvmft_controller *ctrlr; struct nvmf_qpair *qp; struct cidset *cids; bool admin; bool sq_flow_control; uint16_t qid; u_int qsize; uint16_t sqhd; uint16_t sqtail; volatile u_int qp_refs; /* Internal references on 'qp'. */ + struct task datamove_task; + STAILQ_HEAD(, ctl_io_hdr) datamove_queue; + struct mtx lock; char name[16]; }; static int _nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_status); +static void nvmft_datamove_task(void *context, int pending); static void nvmft_qpair_error(void *arg, int error) { struct nvmft_qpair *qp = arg; struct nvmft_controller *ctrlr = qp->ctrlr; /* * XXX: The Linux TCP initiator sends a RST immediately after * the FIN, so treat ECONNRESET as plain EOF to avoid spurious * errors on shutdown. */ if (error == ECONNRESET) error = 0; if (error != 0) nvmft_printf(ctrlr, "error %d on %s\n", error, qp->name); nvmft_controller_error(ctrlr, qp, error); } static void nvmft_receive_capsule(void *arg, struct nvmf_capsule *nc) { struct nvmft_qpair *qp = arg; struct nvmft_controller *ctrlr = qp->ctrlr; const struct nvme_command *cmd; uint8_t sc_status; cmd = nvmf_capsule_sqe(nc); if (ctrlr == NULL) { printf("NVMFT: %s received CID %u opcode %u on newborn queue\n", qp->name, le16toh(cmd->cid), cmd->opc); nvmf_free_capsule(nc); return; } sc_status = nvmf_validate_command_capsule(nc); if (sc_status != NVME_SC_SUCCESS) { _nvmft_send_generic_error(qp, nc, sc_status); nvmf_free_capsule(nc); return; } /* Don't bother byte-swapping CID. */ if (BIT_TEST_SET_ATOMIC(NUM_CIDS, cmd->cid, qp->cids)) { _nvmft_send_generic_error(qp, nc, NVME_SC_COMMAND_ID_CONFLICT); nvmf_free_capsule(nc); return; } if (qp->admin) nvmft_handle_admin_command(ctrlr, nc); else nvmft_handle_io_command(qp, qp->qid, nc); } struct nvmft_qpair * nvmft_qpair_init(enum nvmf_trtype trtype, const struct nvmf_handoff_qpair_params *handoff, uint16_t qid, const char *name) { struct nvmft_qpair *qp; qp = malloc(sizeof(*qp), M_NVMFT, M_WAITOK | M_ZERO); qp->admin = handoff->admin; qp->sq_flow_control = handoff->sq_flow_control; qp->qsize = handoff->qsize; qp->qid = qid; qp->sqhd = handoff->sqhd; qp->sqtail = handoff->sqtail; strlcpy(qp->name, name, sizeof(qp->name)); mtx_init(&qp->lock, "nvmft qp", NULL, MTX_DEF); qp->cids = BITSET_ALLOC(NUM_CIDS, M_NVMFT, M_WAITOK | M_ZERO); + STAILQ_INIT(&qp->datamove_queue); + TASK_INIT(&qp->datamove_task, 0, nvmft_datamove_task, qp); qp->qp = nvmf_allocate_qpair(trtype, true, handoff, nvmft_qpair_error, qp, nvmft_receive_capsule, qp); if (qp->qp == NULL) { mtx_destroy(&qp->lock); free(qp->cids, M_NVMFT); free(qp, M_NVMFT); return (NULL); } refcount_init(&qp->qp_refs, 1); return (qp); } void nvmft_qpair_shutdown(struct nvmft_qpair *qp) { + STAILQ_HEAD(, ctl_io_hdr) datamove_queue; struct nvmf_qpair *nq; + union ctl_io *io; + STAILQ_INIT(&datamove_queue); mtx_lock(&qp->lock); nq = qp->qp; qp->qp = NULL; + STAILQ_CONCAT(&datamove_queue, &qp->datamove_queue); mtx_unlock(&qp->lock); if (nq != NULL && refcount_release(&qp->qp_refs)) nvmf_free_qpair(nq); + + while (!STAILQ_EMPTY(&datamove_queue)) { + io = (union ctl_io *)STAILQ_FIRST(&datamove_queue); + STAILQ_REMOVE_HEAD(&datamove_queue, links); + nvmft_abort_datamove(io); + } + nvmft_drain_task(&qp->datamove_task); } void nvmft_qpair_destroy(struct nvmft_qpair *qp) { nvmft_qpair_shutdown(qp); mtx_destroy(&qp->lock); free(qp->cids, M_NVMFT); free(qp, M_NVMFT); } struct nvmft_controller * nvmft_qpair_ctrlr(struct nvmft_qpair *qp) { return (qp->ctrlr); } uint16_t nvmft_qpair_id(struct nvmft_qpair *qp) { return (qp->qid); } const char * nvmft_qpair_name(struct nvmft_qpair *qp) { return (qp->name); } static int _nvmft_send_response(struct nvmft_qpair *qp, const void *cqe) { struct nvme_completion cpl; struct nvmf_qpair *nq; struct nvmf_capsule *rc; int error; memcpy(&cpl, cqe, sizeof(cpl)); mtx_lock(&qp->lock); nq = qp->qp; if (nq == NULL) { mtx_unlock(&qp->lock); return (ENOTCONN); } refcount_acquire(&qp->qp_refs); /* Set SQHD. */ if (qp->sq_flow_control) { qp->sqhd = (qp->sqhd + 1) % qp->qsize; cpl.sqhd = htole16(qp->sqhd); } else cpl.sqhd = 0; mtx_unlock(&qp->lock); rc = nvmf_allocate_response(nq, &cpl, M_WAITOK); error = nvmf_transmit_capsule(rc); nvmf_free_capsule(rc); if (refcount_release(&qp->qp_refs)) nvmf_free_qpair(nq); return (error); } void nvmft_command_completed(struct nvmft_qpair *qp, struct nvmf_capsule *nc) { const struct nvme_command *cmd = nvmf_capsule_sqe(nc); /* Don't bother byte-swapping CID. */ KASSERT(BIT_ISSET(NUM_CIDS, cmd->cid, qp->cids), ("%s: CID %u not busy", __func__, cmd->cid)); BIT_CLR_ATOMIC(NUM_CIDS, cmd->cid, qp->cids); } int nvmft_send_response(struct nvmft_qpair *qp, const void *cqe) { const struct nvme_completion *cpl = cqe; /* Don't bother byte-swapping CID. */ KASSERT(BIT_ISSET(NUM_CIDS, cpl->cid, qp->cids), ("%s: CID %u not busy", __func__, cpl->cid)); BIT_CLR_ATOMIC(NUM_CIDS, cpl->cid, qp->cids); return (_nvmft_send_response(qp, cqe)); } void nvmft_init_cqe(void *cqe, struct nvmf_capsule *nc, uint16_t status) { struct nvme_completion *cpl = cqe; const struct nvme_command *cmd = nvmf_capsule_sqe(nc); memset(cpl, 0, sizeof(*cpl)); cpl->cid = cmd->cid; cpl->status = htole16(status); } int nvmft_send_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_type, uint8_t sc_status) { struct nvme_completion cpl; uint16_t status; status = NVMEF(NVME_STATUS_SCT, sc_type) | NVMEF(NVME_STATUS_SC, sc_status); nvmft_init_cqe(&cpl, nc, status); return (nvmft_send_response(qp, &cpl)); } int nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_status) { return (nvmft_send_error(qp, nc, NVME_SCT_GENERIC, sc_status)); } /* * This version doesn't clear CID in qp->cids and is used for errors * before the CID is validated. */ static int _nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_status) { struct nvme_completion cpl; uint16_t status; status = NVMEF(NVME_STATUS_SCT, NVME_SCT_GENERIC) | NVMEF(NVME_STATUS_SC, sc_status); nvmft_init_cqe(&cpl, nc, status); return (_nvmft_send_response(qp, &cpl)); } int nvmft_send_success(struct nvmft_qpair *qp, struct nvmf_capsule *nc) { return (nvmft_send_generic_error(qp, nc, NVME_SC_SUCCESS)); } static void nvmft_init_connect_rsp(struct nvmf_fabric_connect_rsp *rsp, const struct nvmf_fabric_connect_cmd *cmd, uint16_t status) { memset(rsp, 0, sizeof(*rsp)); rsp->cid = cmd->cid; rsp->status = htole16(status); } static int nvmft_send_connect_response(struct nvmft_qpair *qp, const struct nvmf_fabric_connect_rsp *rsp) { struct nvmf_capsule *rc; struct nvmf_qpair *nq; int error; mtx_lock(&qp->lock); nq = qp->qp; if (nq == NULL) { mtx_unlock(&qp->lock); return (ENOTCONN); } refcount_acquire(&qp->qp_refs); mtx_unlock(&qp->lock); rc = nvmf_allocate_response(qp->qp, rsp, M_WAITOK); error = nvmf_transmit_capsule(rc); nvmf_free_capsule(rc); if (refcount_release(&qp->qp_refs)) nvmf_free_qpair(nq); return (error); } void nvmft_connect_error(struct nvmft_qpair *qp, const struct nvmf_fabric_connect_cmd *cmd, uint8_t sc_type, uint8_t sc_status) { struct nvmf_fabric_connect_rsp rsp; uint16_t status; status = NVMEF(NVME_STATUS_SCT, sc_type) | NVMEF(NVME_STATUS_SC, sc_status); nvmft_init_connect_rsp(&rsp, cmd, status); nvmft_send_connect_response(qp, &rsp); } void nvmft_connect_invalid_parameters(struct nvmft_qpair *qp, const struct nvmf_fabric_connect_cmd *cmd, bool data, uint16_t offset) { struct nvmf_fabric_connect_rsp rsp; nvmft_init_connect_rsp(&rsp, cmd, NVMEF(NVME_STATUS_SCT, NVME_SCT_COMMAND_SPECIFIC) | NVMEF(NVME_STATUS_SC, NVMF_FABRIC_SC_INVALID_PARAM)); rsp.status_code_specific.invalid.ipo = htole16(offset); rsp.status_code_specific.invalid.iattr = data ? 1 : 0; nvmft_send_connect_response(qp, &rsp); } int nvmft_finish_accept(struct nvmft_qpair *qp, const struct nvmf_fabric_connect_cmd *cmd, struct nvmft_controller *ctrlr) { struct nvmf_fabric_connect_rsp rsp; qp->ctrlr = ctrlr; nvmft_init_connect_rsp(&rsp, cmd, 0); if (qp->sq_flow_control) rsp.sqhd = htole16(qp->sqhd); else rsp.sqhd = htole16(0xffff); rsp.status_code_specific.success.cntlid = htole16(ctrlr->cntlid); return (nvmft_send_connect_response(qp, &rsp)); } + +void +nvmft_qpair_datamove(struct nvmft_qpair *qp, union ctl_io *io) +{ + bool enqueue_task; + + mtx_lock(&qp->lock); + if (qp->qp == NULL) { + mtx_unlock(&qp->lock); + nvmft_abort_datamove(io); + return; + } + enqueue_task = STAILQ_EMPTY(&qp->datamove_queue); + STAILQ_INSERT_TAIL(&qp->datamove_queue, &io->io_hdr, links); + mtx_unlock(&qp->lock); + if (enqueue_task) + nvmft_enqueue_task(&qp->datamove_task); +} + +static void +nvmft_datamove_task(void *context, int pending __unused) +{ + struct nvmft_qpair *qp = context; + union ctl_io *io; + bool abort; + + mtx_lock(&qp->lock); + while (!STAILQ_EMPTY(&qp->datamove_queue)) { + io = (union ctl_io *)STAILQ_FIRST(&qp->datamove_queue); + STAILQ_REMOVE_HEAD(&qp->datamove_queue, links); + abort = (qp->qp == NULL); + mtx_unlock(&qp->lock); + if (abort) + nvmft_abort_datamove(io); + else + nvmft_handle_datamove(io); + mtx_lock(&qp->lock); + } + mtx_unlock(&qp->lock); +} diff --git a/sys/dev/nvmf/controller/nvmft_var.h b/sys/dev/nvmf/controller/nvmft_var.h index fc1f86754382..4fda297c8a85 100644 --- a/sys/dev/nvmf/controller/nvmft_var.h +++ b/sys/dev/nvmf/controller/nvmft_var.h @@ -1,174 +1,179 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023-2024 Chelsio Communications, Inc. * Written by: John Baldwin */ #ifndef __NVMFT_VAR_H__ #define __NVMFT_VAR_H__ #include #include #include #include #include #include #include struct nvmf_capsule; struct nvmft_controller; struct nvmft_qpair; #define NVMFT_NUM_AER 16 struct nvmft_port { TAILQ_ENTRY(nvmft_port) link; u_int refs; struct ctl_port port; struct nvme_controller_data cdata; struct nvme_firmware_page fp; uint64_t cap; uint32_t max_io_qsize; bool online; struct sx lock; struct unrhdr *ids; TAILQ_HEAD(, nvmft_controller) controllers; uint32_t *active_ns; u_int num_ns; }; struct nvmft_io_qpair { struct nvmft_qpair *qp; bool shutdown; }; struct nvmft_controller { struct nvmft_qpair *admin; struct nvmft_io_qpair *io_qpairs; u_int num_io_queues; bool shutdown; bool admin_closed; uint16_t cntlid; uint32_t cc; uint32_t csts; struct nvmft_port *np; struct mtx lock; struct nvme_controller_data cdata; struct nvme_health_information_page hip; sbintime_t create_time; sbintime_t start_busy; sbintime_t busy_total; uint16_t partial_dur; uint16_t partial_duw; uint8_t hostid[16]; uint8_t hostnqn[NVME_NQN_FIELD_SIZE]; u_int trtype; TAILQ_ENTRY(nvmft_controller) link; /* * Each queue can have at most UINT16_MAX commands, so the total * across all queues will fit in a uint32_t. */ uint32_t pending_commands; volatile int ka_active_traffic; struct callout ka_timer; sbintime_t ka_sbt; /* AER fields. */ uint32_t aer_mask; uint16_t aer_cids[NVMFT_NUM_AER]; uint8_t aer_pending; uint8_t aer_cidx; uint8_t aer_pidx; /* Changed namespace IDs. */ struct nvme_ns_list *changed_ns; bool changed_ns_reported; struct task shutdown_task; struct timeout_task terminate_task; }; MALLOC_DECLARE(M_NVMFT); /* ctl_frontend_nvmf.c */ void nvmft_port_free(struct nvmft_port *np); void nvmft_populate_active_nslist(struct nvmft_port *np, uint32_t nsid, struct nvme_ns_list *nslist); void nvmft_dispatch_command(struct nvmft_qpair *qp, struct nvmf_capsule *nc, bool admin); void nvmft_terminate_commands(struct nvmft_controller *ctrlr); +void nvmft_abort_datamove(union ctl_io *io); +void nvmft_handle_datamove(union ctl_io *io); +void nvmft_drain_task(struct task *task); +void nvmft_enqueue_task(struct task *task); /* nvmft_controller.c */ void nvmft_controller_error(struct nvmft_controller *ctrlr, struct nvmft_qpair *qp, int error); void nvmft_controller_lun_changed(struct nvmft_controller *ctrlr, int lun_id); void nvmft_handle_admin_command(struct nvmft_controller *ctrlr, struct nvmf_capsule *nc); void nvmft_handle_io_command(struct nvmft_qpair *qp, uint16_t qid, struct nvmf_capsule *nc); int nvmft_handoff_admin_queue(struct nvmft_port *np, const struct nvmf_handoff_controller_qpair *handoff, const struct nvmf_fabric_connect_cmd *cmd, const struct nvmf_fabric_connect_data *data); int nvmft_handoff_io_queue(struct nvmft_port *np, const struct nvmf_handoff_controller_qpair *handoff, const struct nvmf_fabric_connect_cmd *cmd, const struct nvmf_fabric_connect_data *data); int nvmft_printf(struct nvmft_controller *ctrlr, const char *fmt, ...) __printflike(2, 3); /* nvmft_qpair.c */ struct nvmft_qpair *nvmft_qpair_init(enum nvmf_trtype trtype, const struct nvmf_handoff_qpair_params *handoff, uint16_t qid, const char *name); void nvmft_qpair_shutdown(struct nvmft_qpair *qp); void nvmft_qpair_destroy(struct nvmft_qpair *qp); struct nvmft_controller *nvmft_qpair_ctrlr(struct nvmft_qpair *qp); +void nvmft_qpair_datamove(struct nvmft_qpair *qp, union ctl_io *io); uint16_t nvmft_qpair_id(struct nvmft_qpair *qp); const char *nvmft_qpair_name(struct nvmft_qpair *qp); void nvmft_command_completed(struct nvmft_qpair *qp, struct nvmf_capsule *nc); int nvmft_send_response(struct nvmft_qpair *qp, const void *cqe); void nvmft_init_cqe(void *cqe, struct nvmf_capsule *nc, uint16_t status); int nvmft_send_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_type, uint8_t sc_status); int nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_status); int nvmft_send_success(struct nvmft_qpair *qp, struct nvmf_capsule *nc); void nvmft_connect_error(struct nvmft_qpair *qp, const struct nvmf_fabric_connect_cmd *cmd, uint8_t sc_type, uint8_t sc_status); void nvmft_connect_invalid_parameters(struct nvmft_qpair *qp, const struct nvmf_fabric_connect_cmd *cmd, bool data, uint16_t offset); int nvmft_finish_accept(struct nvmft_qpair *qp, const struct nvmf_fabric_connect_cmd *cmd, struct nvmft_controller *ctrlr); static __inline void nvmft_port_ref(struct nvmft_port *np) { refcount_acquire(&np->refs); } static __inline void nvmft_port_rele(struct nvmft_port *np) { if (refcount_release(&np->refs)) nvmft_port_free(np); } #endif /* !__NVMFT_VAR_H__ */