Page MenuHomeFreeBSD
Files F152038008

D44715.id136811.diff
No OneTemporary
Actions

D44715.id136811.diff
View Options

diff --git a/sbin/nvmecontrol/Makefile b/sbin/nvmecontrol/Makefile
--- a/sbin/nvmecontrol/Makefile
+++ b/sbin/nvmecontrol/Makefile
@@ -4,14 +4,16 @@
PACKAGE=nvme-tools
PROG= nvmecontrol
SRCS= comnd.c nvmecontrol.c
-SRCS+= devlist.c firmware.c format.c identify.c logpage.c ns.c nsid.c
-SRCS+= perftest.c power.c reset.c resv.c sanitize.c
+SRCS+= connect.c devlist.c disconnect.c discover.c fabrics.c firmware.c
+SRCS+= format.c identify.c logpage.c ns.c nsid.c
+SRCS+= perftest.c power.c reconnect.c reset.c resv.c sanitize.c
SRCS+= passthru.c
SRCS+= identify_ext.c nvme_util.c nc_util.c
SRCS+= selftest.c
+CFLAGS+= -I${SRCTOP}/lib/libnvmf
MAN= nvmecontrol.8
LDFLAGS+= -rdynamic
-LIBADD+= util
+LIBADD+= devctl nvmf util
SUBDIR= modules
HAS_TESTS=
SUBDIR.${MK_TESTS}+= tests
diff --git a/sbin/nvmecontrol/connect.c b/sbin/nvmecontrol/connect.c
new file mode 100644
--- /dev/null
+++ b/sbin/nvmecontrol/connect.c
@@ -0,0 +1,268 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2024 Chelsio Communications, Inc.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <err.h>
+#include <libnvmf.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sysexits.h>
+#include <unistd.h>
+
+#include "comnd.h"
+#include "fabrics.h"
+
+/*
+ * Settings that are currently hardcoded but could be exposed to the
+ * user via additional command line options:
+ *
+ * - ADMIN queue entries
+ * - Number of I/O queues
+ * - I/O queue entries
+ * - MaxR2T
+ * - KATO
+ */
+
+static struct options {
+ const char *transport;
+ const char *address;
+ const char *cntlid;
+ bool discover;
+ bool data_digests;
+ bool flow_control;
+ bool header_digests;
+} opt = {
+ .transport = "tcp",
+ .address = NULL,
+ .cntlid = "dynamic",
+ .discover = false,
+ .data_digests = false,
+ .flow_control = false,
+ .header_digests = false,
+};
+
+static void
+tcp_association_params(struct nvmf_association_params *params)
+{
+ params->tcp.pda = 0;
+ params->tcp.header_digests = opt.header_digests;
+ params->tcp.data_digests = opt.data_digests;
+ /* XXX */
+ params->tcp.maxr2t = 1;
+}
+
+static int
+connect_nvm_controller(enum nvmf_trtype trtype, int adrfam, const char *address,
+ const char *port, uint16_t cntlid, const char *subnqn)
+{
+ struct nvme_controller_data cdata;
+ struct nvmf_association_params aparams;
+ struct nvmf_qpair *admin, *io[1];
+ int error;
+
+ memset(&aparams, 0, sizeof(aparams));
+ aparams.sq_flow_control = opt.flow_control;
+ switch (trtype) {
+ case NVMF_TRTYPE_TCP:
+ tcp_association_params(&aparams);
+ break;
+ default:
+ warnx("Unsupported transport %s", nvmf_transport_type(trtype));
+ return (EX_UNAVAILABLE);
+ }
+
+ error = connect_nvm_queues(&aparams, trtype, adrfam, address, port,
+ cntlid, subnqn, &admin, io, nitems(io), &cdata);
+ if (error != 0)
+ return (error);
+
+ error = nvmf_handoff_host(admin, 1, io, &cdata);
+ if (error != 0) {
+ warnc(error, "Failed to handoff queues to kernel");
+ return (EX_IOERR);
+ }
+ return (0);
+}
+
+static void
+connect_discovery_entry(struct nvme_discovery_log_entry *entry)
+{
+ int adrfam;
+
+ switch (entry->trtype) {
+ case NVMF_TRTYPE_TCP:
+ switch (entry->adrfam) {
+ case NVMF_ADRFAM_IPV4:
+ adrfam = AF_INET;
+ break;
+ case NVMF_ADRFAM_IPV6:
+ adrfam = AF_INET6;
+ break;
+ default:
+ warnx("Skipping unsupported address family for %s",
+ entry->subnqn);
+ return;
+ }
+ switch (entry->tsas.tcp.sectype) {
+ case NVME_TCP_SECURITY_NONE:
+ break;
+ default:
+ warnx("Skipping unsupported TCP security type for %s",
+ entry->subnqn);
+ return;
+ }
+ break;
+ default:
+ warnx("Skipping unsupported transport %s for %s",
+ nvmf_transport_type(entry->trtype), entry->subnqn);
+ return;
+ }
+
+ /*
+ * XXX: Track portids and avoid duplicate connections for a
+ * given (subnqn,portid)?
+ */
+
+ /* XXX: Should this make use of entry->aqsz in some way? */
+ connect_nvm_controller(entry->trtype, adrfam, entry->traddr,
+ entry->trsvcid, entry->cntlid, entry->subnqn);
+}
+
+static void
+connect_discovery_log_page(struct nvmf_qpair *qp)
+{
+ struct nvme_discovery_log *log;
+ int error;
+
+ error = nvmf_host_fetch_discovery_log_page(qp, &log);
+ if (error != 0)
+ errc(EX_IOERR, error, "Failed to fetch discovery log page");
+
+ for (u_int i = 0; i < log->numrec; i++)
+ connect_discovery_entry(&log->entries[i]);
+ free(log);
+}
+
+static void
+discover_controllers(enum nvmf_trtype trtype, const char *address,
+ const char *port)
+{
+ struct nvmf_qpair *qp;
+
+ qp = connect_discovery_adminq(trtype, address, port);
+
+ connect_discovery_log_page(qp);
+
+ nvmf_free_qpair(qp);
+}
+
+static void
+connect_static(enum nvmf_trtype trtype, const char *address, const char *port,
+ const char *subnqn)
+{
+ u_long cntlid;
+ int error;
+
+ if (port == NULL)
+ errx(EX_USAGE, "Explicit port required");
+
+ cntlid = nvmf_parse_cntlid(opt.cntlid);
+
+ error = connect_nvm_controller(trtype, AF_UNSPEC, address, port, cntlid,
+ subnqn);
+ if (error != 0)
+ exit(error);
+}
+
+static void
+connect_fn(const struct cmd *f, int argc, char *argv[])
+{
+ enum nvmf_trtype trtype;
+ const char *address, *port, *subnqn;
+ char *tofree;
+
+ if (arg_parse(argc, argv, f))
+ return;
+
+ if (strcasecmp(opt.transport, "tcp") == 0) {
+ trtype = NVMF_TRTYPE_TCP;
+ } else
+ errx(EX_USAGE, "Unsupported or invalid transport");
+
+ nvmf_parse_address(opt.address, &address, &port, &tofree);
+ if (opt.discover)
+ discover_controllers(trtype, address, port);
+ else {
+ /*
+ * XXX: Using argv[optind] directly isn't super clean,
+ * but struct cmd doesn't support optional argments.
+ */
+ subnqn = argv[optind];
+ if (subnqn == NULL)
+ errx(EX_USAGE,
+ "Static connections require explicit NQN");
+ optind++;
+ connect_static(trtype, address, port, subnqn);
+ }
+
+ free(tofree);
+}
+
+static const struct opts connect_opts[] = {
+#define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc }
+ OPT("transport", 't', arg_string, opt, transport,
+ "Transport type"),
+ OPT("cntlid", 'c', arg_string, opt, cntlid,
+ "Controller ID"),
+ OPT("discover", 'd', arg_none, opt, discover,
+ "Connect to all controllers enumerated via discovery"),
+ OPT("header_digests", 'H', arg_none, opt, header_digests,
+ "Enable TCP PDU header digests"),
+ OPT("data_digests", 'D', arg_none, opt, data_digests,
+ "Enable TCP PDU data digests"),
+ OPT("flow_control", 'F', arg_none, opt, flow_control,
+ "Request SQ flow control"),
+ { NULL, 0, arg_none, NULL, NULL }
+};
+#undef OPT
+
+static const struct args connect_args[] = {
+ { arg_string, &opt.address, "address" },
+ { arg_none, NULL, NULL },
+};
+
+static struct cmd connect_cmd = {
+ .name = "connect",
+ .fn = connect_fn,
+ .descr = "Connect to a fabrics controller",
+ .ctx_size = sizeof(opt),
+ .opts = connect_opts,
+ .args = connect_args,
+};
+
+CMD_COMMAND(connect_cmd);
diff --git a/sbin/nvmecontrol/disconnect.c b/sbin/nvmecontrol/disconnect.c
new file mode 100644
--- /dev/null
+++ b/sbin/nvmecontrol/disconnect.c
@@ -0,0 +1,75 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2024 Chelsio Communications, Inc.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <err.h>
+#include <stdlib.h>
+#include <sysexits.h>
+#include <unistd.h>
+#include <devctl.h>
+
+#include "nvmecontrol.h"
+
+static struct options {
+ const char *dev;
+} opt = {
+ .dev = NULL
+};
+
+static const struct args args[] = {
+ { arg_string, &opt.dev, "controller-id|namespace-id" },
+ { arg_none, NULL, NULL },
+};
+
+static void
+disconnect(const struct cmd *f, int argc, char *argv[])
+{
+ int fd;
+ char *path;
+
+ if (arg_parse(argc, argv, f))
+ return;
+ open_dev(opt.dev, &fd, 1, 1);
+ get_nsid(fd, &path, NULL);
+ close(fd);
+
+ if (devctl_detach(path, false) == -1)
+ err(EX_IOERR, "detach of %s failed", path);
+ if (devctl_delete(path, true) == -1)
+ err(EX_IOERR, "delete of %s failed", path);
+
+ exit(0);
+}
+
+static struct cmd disconnect_cmd = {
+ .name = "disconnect",
+ .fn = disconnect,
+ .descr = "Disconnect from a fabrics controller",
+ .args = args,
+};
+
+CMD_COMMAND(disconnect_cmd);
diff --git a/sbin/nvmecontrol/discover.c b/sbin/nvmecontrol/discover.c
new file mode 100644
--- /dev/null
+++ b/sbin/nvmecontrol/discover.c
@@ -0,0 +1,317 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2024 Chelsio Communications, Inc.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <err.h>
+#include <libnvmf.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sysexits.h>
+
+#include "comnd.h"
+#include "fabrics.h"
+#include "nvmecontrol_ext.h"
+
+static struct options {
+ const char *transport;
+ const char *address;
+ bool verbose;
+} opt = {
+ .transport = "tcp",
+ .address = NULL,
+ .verbose = false,
+};
+
+static void
+identify_controller(struct nvmf_qpair *qp)
+{
+ struct nvme_controller_data cdata;
+ int error;
+
+ error = nvmf_host_identify_controller(qp, &cdata);
+ if (error != 0)
+ errc(EX_IOERR, error, "Failed to fetch controller data");
+ nvme_print_controller(&cdata);
+}
+
+static const char *
+nvmf_address_family(uint8_t adrfam)
+{
+ static char buf[8];
+
+ switch (adrfam) {
+ case NVMF_ADRFAM_IPV4:
+ return ("AF_INET");
+ case NVMF_ADRFAM_IPV6:
+ return ("AF_INET6");
+ case NVMF_ADRFAM_IB:
+ return ("InfiniBand");
+ case NVMF_ADRFAM_FC:
+ return ("Fibre Channel");
+ case NVMF_ADRFAM_INTRA_HOST:
+ return ("Intra-host");
+ default:
+ snprintf(buf, sizeof(buf), "0x%02x\n", adrfam);
+ return (buf);
+ }
+}
+
+static const char *
+nvmf_subsystem_type(uint8_t subtype)
+{
+ static char buf[8];
+
+ switch (subtype) {
+ case NVMF_SUBTYPE_DISCOVERY:
+ return ("Discovery");
+ case NVMF_SUBTYPE_NVME:
+ return ("NVMe");
+ default:
+ snprintf(buf, sizeof(buf), "0x%02x\n", subtype);
+ return (buf);
+ }
+}
+
+static const char *
+nvmf_secure_channel(uint8_t treq)
+{
+ switch (treq & 0x03) {
+ case NVMF_TREQ_SECURE_CHANNEL_NOT_SPECIFIED:
+ return ("Not specified");
+ case NVMF_TREQ_SECURE_CHANNEL_REQUIRED:
+ return ("Required");
+ case NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED:
+ return ("Not required");
+ default:
+ return ("0x03");
+ }
+}
+
+static const char *
+nvmf_controller_id(uint16_t cntlid)
+{
+ static char buf[8];
+
+ switch (cntlid) {
+ case NVMF_CNTLID_DYNAMIC:
+ return ("Dynamic");
+ case NVMF_CNTLID_STATIC_ANY:
+ return ("Static");
+ default:
+ snprintf(buf, sizeof(buf), "%u", cntlid);
+ return (buf);
+ }
+}
+
+static const char *
+nvmf_rdma_service_type(uint8_t qptype)
+{
+ static char buf[8];
+
+ switch (qptype) {
+ case NVMF_RDMA_QPTYPE_RELIABLE_CONNECTED:
+ return ("Reliable connected");
+ case NVMF_RDMA_QPTYPE_RELIABLE_DATAGRAM:
+ return ("Reliable datagram");
+ default:
+ snprintf(buf, sizeof(buf), "0x%02x\n", qptype);
+ return (buf);
+ }
+}
+
+static const char *
+nvmf_rdma_provider_type(uint8_t prtype)
+{
+ static char buf[8];
+
+ switch (prtype) {
+ case NVMF_RDMA_PRTYPE_NONE:
+ return ("None");
+ case NVMF_RDMA_PRTYPE_IB:
+ return ("InfiniBand");
+ case NVMF_RDMA_PRTYPE_ROCE:
+ return ("RoCE (v1)");
+ case NVMF_RDMA_PRTYPE_ROCE2:
+ return ("RoCE (v2)");
+ case NVMF_RDMA_PRTYPE_IWARP:
+ return ("iWARP");
+ default:
+ snprintf(buf, sizeof(buf), "0x%02x\n", prtype);
+ return (buf);
+ }
+}
+
+static const char *
+nvmf_rdma_cms(uint8_t cms)
+{
+ static char buf[8];
+
+ switch (cms) {
+ case NVMF_RDMA_CMS_RDMA_CM:
+ return ("RDMA_IP_CM");
+ default:
+ snprintf(buf, sizeof(buf), "0x%02x\n", cms);
+ return (buf);
+ }
+}
+
+static const char *
+nvmf_tcp_security_type(uint8_t sectype)
+{
+ static char buf[8];
+
+ switch (sectype) {
+ case NVME_TCP_SECURITY_NONE:
+ return ("None");
+ case NVME_TCP_SECURITY_TLS_1_2:
+ return ("TLS 1.2");
+ case NVME_TCP_SECURITY_TLS_1_3:
+ return ("TLS 1.3");
+ default:
+ snprintf(buf, sizeof(buf), "0x%02x\n", sectype);
+ return (buf);
+ }
+}
+
+static void
+print_discovery_entry(u_int i, struct nvme_discovery_log_entry *entry)
+{
+ printf("Entry %02d\n", i + 1);
+ printf("========\n");
+ printf(" Transport type: %s\n",
+ nvmf_transport_type(entry->trtype));
+ printf(" Address family: %s\n",
+ nvmf_address_family(entry->adrfam));
+ printf(" Subsystem type: %s\n",
+ nvmf_subsystem_type(entry->subtype));
+ printf(" SQ flow control: %s\n",
+ (entry->treq & (1 << 2)) == 0 ? "required" : "optional");
+ printf(" Secure Channel: %s\n", nvmf_secure_channel(entry->treq));
+ printf(" Port ID: %u\n", entry->portid);
+ printf(" Controller ID: %s\n",
+ nvmf_controller_id(entry->cntlid));
+ printf(" Max Admin SQ Size: %u\n", entry->aqsz);
+ printf(" Sub NQN: %s\n", entry->subnqn);
+ printf(" Transport address: %s\n", entry->traddr);
+ printf(" Service identifier: %s\n", entry->trsvcid);
+ switch (entry->trtype) {
+ case NVMF_TRTYPE_RDMA:
+ printf(" RDMA Service Type: %s\n",
+ nvmf_rdma_service_type(entry->tsas.rdma.rdma_qptype));
+ printf(" RDMA Provider Type: %s\n",
+ nvmf_rdma_provider_type(entry->tsas.rdma.rdma_prtype));
+ printf(" RDMA CMS: %s\n",
+ nvmf_rdma_cms(entry->tsas.rdma.rdma_cms));
+ printf(" Partition key: %u\n",
+ entry->tsas.rdma.rdma_pkey);
+ break;
+ case NVMF_TRTYPE_TCP:
+ printf(" Security Type: %s\n",
+ nvmf_tcp_security_type(entry->tsas.tcp.sectype));
+ break;
+ }
+}
+
+static void
+dump_discovery_log_page(struct nvmf_qpair *qp)
+{
+ struct nvme_discovery_log *log;
+ int error;
+
+ error = nvmf_host_fetch_discovery_log_page(qp, &log);
+ if (error != 0)
+ errc(EX_IOERR, error, "Failed to fetch discovery log page");
+
+ printf("Discovery\n");
+ printf("=========\n");
+ if (log->numrec == 0) {
+ printf("No entries found\n");
+ } else {
+ for (u_int i = 0; i < log->numrec; i++)
+ print_discovery_entry(i, &log->entries[i]);
+ }
+ free(log);
+}
+
+static void
+discover(const struct cmd *f, int argc, char *argv[])
+{
+ enum nvmf_trtype trtype;
+ struct nvmf_qpair *qp;
+ const char *address, *port;
+ char *tofree;
+
+ if (arg_parse(argc, argv, f))
+ return;
+
+ if (strcasecmp(opt.transport, "tcp") == 0) {
+ trtype = NVMF_TRTYPE_TCP;
+ } else
+ errx(EX_USAGE, "Unsupported or invalid transport");
+
+ nvmf_parse_address(opt.address, &address, &port, &tofree);
+ qp = connect_discovery_adminq(trtype, address, port);
+ free(tofree);
+
+ /* Use Identify to fetch controller data */
+ if (opt.verbose) {
+ identify_controller(qp);
+ printf("\n");
+ }
+
+ /* Fetch Log pages */
+ dump_discovery_log_page(qp);
+
+ nvmf_free_qpair(qp);
+}
+
+static const struct opts discover_opts[] = {
+#define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc }
+ OPT("transport", 't', arg_string, opt, transport,
+ "Transport type"),
+ OPT("verbose", 'v', arg_none, opt, verbose,
+ "Display the discovery controller's controller data"),
+ { NULL, 0, arg_none, NULL, NULL }
+};
+#undef OPT
+
+static const struct args discover_args[] = {
+ { arg_string, &opt.address, "address" },
+ { arg_none, NULL, NULL },
+};
+
+static struct cmd discover_cmd = {
+ .name = "discover",
+ .fn = discover,
+ .descr = "List discovery log pages from a fabrics controller",
+ .ctx_size = sizeof(opt),
+ .opts = discover_opts,
+ .args = discover_args,
+};
+
+CMD_COMMAND(discover_cmd);
diff --git a/sbin/nvmecontrol/fabrics.h b/sbin/nvmecontrol/fabrics.h
new file mode 100644
--- /dev/null
+++ b/sbin/nvmecontrol/fabrics.h
@@ -0,0 +1,64 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2024 Chelsio Communications, Inc.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __FABRICS_H__
+#define __FABRICS_H__
+
+const char *nvmf_transport_type(uint8_t trtype);
+
+/*
+ * Splits 'in_address' into separate 'address' and 'port' strings. If
+ * a separate buffer for the address was allocated, 'tofree' is set to
+ * the allocated buffer, otherwise 'tofree' is set to NULL.
+ */
+void nvmf_parse_address(const char *in_address, const char **address,
+ const char **port, char **tofree);
+
+uint16_t nvmf_parse_cntlid(const char *cntlid);
+
+/* Returns true if able to open a connection. */
+bool tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam,
+ const char *address, const char *port);
+
+/* Connect to a discovery controller and return the Admin qpair. */
+struct nvmf_qpair *connect_discovery_adminq(enum nvmf_trtype trtype,
+ const char *address, const char *port);
+
+/*
+ * Connect to an NVM controller establishing an Admin qpair and one or
+ * more I/O qpairs. The controller's controller data is returned in
+ * *cdata on success. Returns a non-zero value from <sysexits.h> on
+ * failure.
+ */
+int connect_nvm_queues(const struct nvmf_association_params *aparams,
+ enum nvmf_trtype trtype, int adrfam, const char *address,
+ const char *port, uint16_t cntlid, const char *subnqn,
+ struct nvmf_qpair **admin, struct nvmf_qpair **io, u_int num_io_queues,
+ struct nvme_controller_data *cdata);
+
+#endif /* !__FABRICS_H__ */
diff --git a/sbin/nvmecontrol/fabrics.c b/sbin/nvmecontrol/fabrics.c
new file mode 100644
--- /dev/null
+++ b/sbin/nvmecontrol/fabrics.c
@@ -0,0 +1,533 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2024 Chelsio Communications, Inc.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <err.h>
+#include <libnvmf.h>
+#include <netdb.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sysexits.h>
+#include <unistd.h>
+
+#include "fabrics.h"
+
+/*
+ * Subroutines shared by several Fabrics commands.
+ */
+static char nqn[NVMF_NQN_MAX_LEN];
+static uint8_t hostid[16];
+static bool hostid_initted = false;
+
+static bool
+init_hostid(void)
+{
+ int error;
+
+ if (hostid_initted)
+ return (true);
+
+ error = nvmf_hostid_from_hostuuid(hostid);
+ if (error != 0) {
+ warnc(error, "Failed to generate hostid");
+ return (false);
+ }
+ error = nvmf_nqn_from_hostuuid(nqn);
+ if (error != 0) {
+ warnc(error, "Failed to generate host NQN");
+ return (false);
+ }
+
+ hostid_initted = true;
+ return (true);
+}
+
+const char *
+nvmf_transport_type(uint8_t trtype)
+{
+ static char buf[8];
+
+ switch (trtype) {
+ case NVMF_TRTYPE_RDMA:
+ return ("RDMA");
+ case NVMF_TRTYPE_FC:
+ return ("Fibre Channel");
+ case NVMF_TRTYPE_TCP:
+ return ("TCP");
+ case NVMF_TRTYPE_INTRA_HOST:
+ return ("Intra-host");
+ default:
+ snprintf(buf, sizeof(buf), "0x%02x\n", trtype);
+ return (buf);
+ }
+}
+
+void
+nvmf_parse_address(const char *in_address, const char **address,
+ const char **port, char **tofree)
+{
+ char *cp;
+
+ /*
+ * Accepts the following address formats:
+ *
+ * [IPv6 address]:port
+ * IPv4 address:port
+ * hostname:port
+ * [IPv6 address]
+ * IPv6 address
+ * IPv4 address
+ * hostname
+ */
+ if (in_address[0] == '[') {
+ /* IPv6 address in square brackets. */
+ cp = strchr(in_address + 1, ']');
+ if (cp == NULL || cp == in_address + 1)
+ errx(EX_USAGE, "Invalid address %s", in_address);
+ *tofree = strndup(in_address + 1, cp - (in_address + 1));
+ *address = *tofree;
+
+ /* Skip over ']' */
+ cp++;
+ switch (*cp) {
+ case '\0':
+ *port = NULL;
+ return;
+ case ':':
+ if (cp[1] != '\0') {
+ *port = cp + 1;
+ return;
+ }
+ /* FALLTHROUGH */
+ default:
+ errx(EX_USAGE, "Invalid address %s", in_address);
+ }
+ }
+
+ /* Look for the first colon. */
+ cp = strchr(in_address, ':');
+ if (cp == NULL) {
+ *address = in_address;
+ *port = NULL;
+ *tofree = NULL;
+ return;
+ }
+
+ /* If there is another colon, assume this is an IPv6 address. */
+ if (strchr(cp + 1, ':') != NULL) {
+ *address = in_address;
+ *port = NULL;
+ *tofree = NULL;
+ return;
+ }
+
+ /* Both strings on either side of the colon must be non-empty. */
+ if (cp == in_address || cp[1] == '\0')
+ errx(EX_USAGE, "Invalid address %s", in_address);
+
+ *tofree = strndup(in_address, cp - in_address);
+ *address = *tofree;
+
+ /* Skip over ':' */
+ *port = cp + 1;
+}
+
+uint16_t
+nvmf_parse_cntlid(const char *cntlid)
+{
+ u_long value;
+
+ if (strcasecmp(cntlid, "dynamic") == 0)
+ return (NVMF_CNTLID_DYNAMIC);
+ else if (strcasecmp(cntlid, "static") == 0)
+ return (NVMF_CNTLID_STATIC_ANY);
+ else {
+ value = strtoul(cntlid, NULL, 0);
+
+ if (value > NVMF_CNTLID_STATIC_MAX)
+ errx(EX_USAGE, "Invalid controller ID");
+
+ return (value);
+ }
+}
+
+bool
+tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam,
+ const char *address, const char *port)
+{
+ struct addrinfo hints, *ai, *list;
+ int error, s;
+
+ memset(&hints, 0, sizeof(hints));
+ hints.ai_family = adrfam;
+ hints.ai_protocol = IPPROTO_TCP;
+ error = getaddrinfo(address, port, &hints, &list);
+ if (error != 0) {
+ warnx("%s", gai_strerror(error));
+ return (false);
+ }
+
+ for (ai = list; ai != NULL; ai = ai->ai_next) {
+ s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
+ if (s == -1)
+ continue;
+
+ if (connect(s, ai->ai_addr, ai->ai_addrlen) != 0) {
+ close(s);
+ continue;
+ }
+
+ params->tcp.fd = s;
+ freeaddrinfo(list);
+ return (true);
+ }
+ warn("Failed to connect to controller at %s:%s", address, port);
+ return (false);
+}
+
+static void
+tcp_discovery_association_params(struct nvmf_association_params *params)
+{
+ params->tcp.pda = 0;
+ params->tcp.header_digests = false;
+ params->tcp.data_digests = false;
+ params->tcp.maxr2t = 1;
+}
+
+struct nvmf_qpair *
+connect_discovery_adminq(enum nvmf_trtype trtype, const char *address,
+ const char *port)
+{
+ struct nvmf_association_params aparams;
+ struct nvmf_qpair_params qparams;
+ struct nvmf_association *na;
+ struct nvmf_qpair *qp;
+ uint64_t cap, cc, csts;
+ int error, timo;
+
+ memset(&aparams, 0, sizeof(aparams));
+ aparams.sq_flow_control = false;
+ switch (trtype) {
+ case NVMF_TRTYPE_TCP:
+ /* 7.4.9.3 Default port for discovery */
+ if (port == NULL)
+ port = "8009";
+ tcp_discovery_association_params(&aparams);
+ break;
+ default:
+ errx(EX_UNAVAILABLE, "Unsupported transport %s",
+ nvmf_transport_type(trtype));
+ }
+
+ if (!init_hostid())
+ exit(EX_IOERR);
+
+ na = nvmf_allocate_association(trtype, false, &aparams);
+ if (na == NULL)
+ err(EX_IOERR, "Failed to create discovery association");
+ memset(&qparams, 0, sizeof(qparams));
+ qparams.admin = true;
+ if (!tcp_qpair_params(&qparams, AF_UNSPEC, address, port))
+ exit(EX_NOHOST);
+ qp = nvmf_connect(na, &qparams, 0, NVME_MIN_ADMIN_ENTRIES, hostid,
+ NVMF_CNTLID_DYNAMIC, NVMF_DISCOVERY_NQN, nqn, 0);
+ if (qp == NULL)
+ errx(EX_IOERR, "Failed to connect to discovery controller: %s",
+ nvmf_association_error(na));
+ nvmf_free_association(na);
+
+ /* Fetch Controller Capabilities Property */
+ error = nvmf_read_property(qp, NVMF_PROP_CAP, 8, &cap);
+ if (error != 0)
+ errc(EX_IOERR, error, "Failed to fetch CAP");
+
+ /* Set Controller Configuration Property (CC.EN=1) */
+ error = nvmf_read_property(qp, NVMF_PROP_CC, 4, &cc);
+ if (error != 0)
+ errc(EX_IOERR, error, "Failed to fetch CC");
+
+ /* Clear known fields preserving any reserved fields. */
+ cc &= ~(NVMEM(NVME_CC_REG_SHN) | NVMEM(NVME_CC_REG_AMS) |
+ NVMEM(NVME_CC_REG_MPS) | NVMEM(NVME_CC_REG_CSS));
+
+ /* Leave AMS, MPS, and CSS as 0. */
+
+ cc |= NVMEF(NVME_CC_REG_EN, 1);
+
+ error = nvmf_write_property(qp, NVMF_PROP_CC, 4, cc);
+ if (error != 0)
+ errc(EX_IOERR, error, "Failed to set CC");
+
+ /* Wait for CSTS.RDY in Controller Status */
+ timo = NVME_CAP_LO_TO(cap);
+ for (;;) {
+ error = nvmf_read_property(qp, NVMF_PROP_CSTS, 4, &csts);
+ if (error != 0)
+ errc(EX_IOERR, error, "Failed to fetch CSTS");
+
+ if (NVMEV(NVME_CSTS_REG_RDY, csts) != 0)
+ break;
+
+ if (timo == 0)
+ errx(EX_IOERR, "Controller failed to become ready");
+ timo--;
+ usleep(500 * 1000);
+ }
+
+ return (qp);
+}
+
+/* XXX: Should accept KATO and admin queue size as parameters. */
+static int
+connect_nvm_adminq(struct nvmf_association *na,
+ const struct nvmf_qpair_params *params, struct nvmf_qpair **qpp,
+ uint16_t cntlid, const char *subnqn, uint16_t *mqes)
+{
+ struct nvmf_qpair *qp;
+ uint64_t cap, cc, csts;
+ u_int mps, mpsmin, mpsmax;
+ int error, timo;
+
+ qp = nvmf_connect(na, params, 0, 32 /* XXX */, hostid, cntlid, subnqn,
+ nqn, NVMF_KATO_DEFAULT);
+ if (qp == NULL) {
+ warnx("Failed to connect to NVM controller %s: %s", subnqn,
+ nvmf_association_error(na));
+ return (EX_IOERR);
+ }
+
+ /* Fetch Controller Capabilities Property */
+ error = nvmf_read_property(qp, NVMF_PROP_CAP, 8, &cap);
+ if (error != 0) {
+ warnc(error, "Failed to fetch CAP");
+ nvmf_free_qpair(qp);
+ return (EX_IOERR);
+ }
+
+ /* Require the NVM command set. */
+ if (NVME_CAP_HI_CSS_NVM(cap >> 32) == 0) {
+ warnx("Controller %s does not support the NVM command set",
+ subnqn);
+ nvmf_free_qpair(qp);
+ return (EX_UNAVAILABLE);
+ }
+
+ *mqes = NVME_CAP_LO_MQES(cap);
+
+ /* Prefer native host page size if it fits. */
+ mpsmin = NVMEV(NVME_CAP_HI_REG_MPSMIN, cap >> 32);
+ mpsmax = NVMEV(NVME_CAP_HI_REG_MPSMAX, cap >> 32);
+ mps = ffs(getpagesize()) - 1;
+ if (mps < mpsmin + NVME_MPS_SHIFT)
+ mps = mpsmin;
+ else if (mps > mpsmax + NVME_MPS_SHIFT)
+ mps = mpsmax;
+ else
+ mps -= NVME_MPS_SHIFT;
+
+ /* Configure controller. */
+ error = nvmf_read_property(qp, NVMF_PROP_CC, 4, &cc);
+ if (error != 0) {
+ warnc(error, "Failed to fetch CC");
+ nvmf_free_qpair(qp);
+ return (EX_IOERR);
+ }
+
+ /* Clear known fields preserving any reserved fields. */
+ cc &= ~(NVMEM(NVME_CC_REG_IOCQES) | NVMEM(NVME_CC_REG_IOSQES) |
+ NVMEM(NVME_CC_REG_SHN) | NVMEM(NVME_CC_REG_AMS) |
+ NVMEM(NVME_CC_REG_MPS) | NVMEM(NVME_CC_REG_CSS));
+
+ cc |= NVMEF(NVME_CC_REG_IOCQES, 4); /* CQE entry size == 16 */
+ cc |= NVMEF(NVME_CC_REG_IOSQES, 6); /* SEQ entry size == 64 */
+ cc |= NVMEF(NVME_CC_REG_AMS, 0); /* AMS 0 (Round-robin) */
+ cc |= NVMEF(NVME_CC_REG_MPS, mps);
+ cc |= NVMEF(NVME_CC_REG_CSS, 0); /* NVM command set */
+ cc |= NVMEF(NVME_CC_REG_EN, 1); /* EN = 1 */
+
+ error = nvmf_write_property(qp, NVMF_PROP_CC, 4, cc);
+ if (error != 0) {
+ warnc(error, "Failed to set CC");
+ nvmf_free_qpair(qp);
+ return (EX_IOERR);
+ }
+
+ /* Wait for CSTS.RDY in Controller Status */
+ timo = NVME_CAP_LO_TO(cap);
+ for (;;) {
+ error = nvmf_read_property(qp, NVMF_PROP_CSTS, 4, &csts);
+ if (error != 0) {
+ warnc(error, "Failed to fetch CSTS");
+ nvmf_free_qpair(qp);
+ return (EX_IOERR);
+ }
+
+ if (NVMEV(NVME_CSTS_REG_RDY, csts) != 0)
+ break;
+
+ if (timo == 0) {
+ warnx("Controller failed to become ready");
+ nvmf_free_qpair(qp);
+ return (EX_IOERR);
+ }
+ timo--;
+ usleep(500 * 1000);
+ }
+
+ *qpp = qp;
+ return (0);
+}
+
+static void
+shutdown_controller(struct nvmf_qpair *qp)
+{
+ uint64_t cc;
+ int error;
+
+ error = nvmf_read_property(qp, NVMF_PROP_CC, 4, &cc);
+ if (error != 0) {
+ warnc(error, "Failed to fetch CC");
+ goto out;
+ }
+
+ cc |= NVMEF(NVME_CC_REG_SHN, NVME_SHN_NORMAL);
+
+ error = nvmf_write_property(qp, NVMF_PROP_CC, 4, cc);
+ if (error != 0) {
+ warnc(error, "Failed to set CC to trigger shutdown");
+ goto out;
+ }
+
+out:
+ nvmf_free_qpair(qp);
+}
+
+/* Returns a value from <sysexits.h> */
+int
+connect_nvm_queues(const struct nvmf_association_params *aparams,
+ enum nvmf_trtype trtype, int adrfam, const char *address,
+ const char *port, uint16_t cntlid, const char *subnqn,
+ struct nvmf_qpair **admin, struct nvmf_qpair **io, u_int num_io_queues,
+ struct nvme_controller_data *cdata)
+{
+ struct nvmf_qpair_params qparams;
+ struct nvmf_association *na;
+ u_int queues;
+ int error;
+ uint16_t mqes;
+
+ switch (trtype) {
+ case NVMF_TRTYPE_TCP:
+ break;
+ default:
+ warnx("Unsupported transport %s", nvmf_transport_type(trtype));
+ return (EX_UNAVAILABLE);
+ }
+
+ if (!init_hostid())
+ return (EX_IOERR);
+
+ /* Association. */
+ na = nvmf_allocate_association(trtype, false, aparams);
+ if (na == NULL) {
+ warn("Failed to create association for %s", subnqn);
+ return (EX_IOERR);
+ }
+
+ /* Admin queue. */
+ memset(&qparams, 0, sizeof(qparams));
+ qparams.admin = true;
+ if (!tcp_qpair_params(&qparams, adrfam, address, port)) {
+ nvmf_free_association(na);
+ return (EX_NOHOST);
+ }
+ error = connect_nvm_adminq(na, &qparams, admin, cntlid, subnqn, &mqes);
+ if (error != 0) {
+ nvmf_free_association(na);
+ return (error);
+ }
+
+ /* Fetch controller data. */
+ error = nvmf_host_identify_controller(*admin, cdata);
+ if (error != 0) {
+ shutdown_controller(*admin);
+ nvmf_free_association(na);
+ warnc(error, "Failed to fetch controller data for %s", subnqn);
+ return (EX_IOERR);
+ }
+
+ nvmf_update_assocation(na, cdata);
+
+ error = nvmf_host_request_queues(*admin, num_io_queues, &queues);
+ if (error != 0) {
+ shutdown_controller(*admin);
+ nvmf_free_association(na);
+ warnc(error, "Failed to request I/O queues");
+ return (EX_IOERR);
+ }
+ if (queues < num_io_queues) {
+ shutdown_controller(*admin);
+ nvmf_free_association(na);
+ warnx("Controller enabled fewer I/O queues (%u) than requested (%u)",
+ queues, num_io_queues);
+ return (EX_PROTOCOL);
+ }
+
+ /* I/O queues. */
+ memset(io, 0, sizeof(io) * num_io_queues);
+ for (u_int i = 0; i < num_io_queues; i++) {
+ memset(&qparams, 0, sizeof(qparams));
+ qparams.admin = false;
+ if (!tcp_qpair_params(&qparams, adrfam, address, port)) {
+ error = EX_NOHOST;
+ goto out;
+ }
+ io[i] = nvmf_connect(na, &qparams, i + 1, mqes + 1, hostid,
+ nvmf_cntlid(*admin), subnqn, nqn, 0);
+ if (io[i] == NULL) {
+ warnx("Failed to create I/O queue: %s",
+ nvmf_association_error(na));
+ error = EX_IOERR;
+ goto out;
+ }
+ }
+ nvmf_free_association(na);
+ return (0);
+
+out:
+ for (u_int i = 0; i < num_io_queues; i++) {
+ if (io[i] == NULL)
+ break;
+ nvmf_free_qpair(io[i]);
+ }
+ shutdown_controller(*admin);
+ nvmf_free_association(na);
+ return (error);
+}
diff --git a/sbin/nvmecontrol/nvmecontrol.8 b/sbin/nvmecontrol/nvmecontrol.8
--- a/sbin/nvmecontrol/nvmecontrol.8
+++ b/sbin/nvmecontrol/nvmecontrol.8
@@ -205,6 +205,26 @@
.Ic io-passthru
.Op args
.Aq Ar namespace-id
+.Nm
+.Ic discover
+.Op Fl t Ar transport
+.Op Fl v
+.Nm
+.Ic connect
+.Op Fl DFHd
+.Op Fl c Ar cntl-id
+.Op Fl t Ar transport
+.Aq Ar address
+.Op Ar SubNQN
+.Nm
+.Ic disconnect
+.Aq Ar device-id | Ar namespace-id
+.Nm
+.Ic reconnect
+.Op Fl DFH
+.Op Fl t Ar transport
+.Aq Ar device-id
+.Aq Ar address
.Sh DESCRIPTION
NVM Express (NVMe) is a storage protocol standard, for SSDs and other
high-speed storage devices over PCI Express.
@@ -597,6 +617,94 @@
Commands needing metadata are not supported by the
.Xr nvme 4
drive.
+.Ss discover
+List the remote controllers advertised by a remote Discovery Controller:
+.Bl -tag -width 6n
+.It Fl t Ar transport
+The transport to use.
+The default is
+.Ar tcp .
+.It Fl v
+Display the
+.Dv IDENTIFY_CONTROLLER
+data for the Discovery Controller.
+.El
+.Ss connect
+Establish an association with one or more remote controllers.
+.Pp
+If
+.Fl d
+is specified, then the Discovery Controller at
+.Ar address
+is queried and an association is established for each advertised controller.
+The
+.Fl t
+flag determines the transport used with the initial association with
+the Discovery Controller and defaults to
+.Ar tcp .
+Associations with I/O controllers use the transport from the
+corresponding log page entry.
+.Pp
+If
+.Fl d
+is not specified,
+a single association is established with the I/O Controller at
+.Ar address .
+The address must include a port,
+and a
+.Ar SubNQN
+identifying the remote NVM subsystem's NVMe qualified name must be
+specified.
+The
+.Fl c
+flag can request a remote controller ID via one of the following:
+.Bl -tag
+.It dynamic
+Request a dynamic controller ID for controllers using the dynamic
+controller model.
+This is the default.
+.It static
+Request a dynamic controller ID for controllers using the static
+controller model.
+.It Ar number
+Request a specific controller ID for controllers using the static
+controller model.
+.El
+.Pp
+For each association with an I/O controller, an admin queue pair and a
+single I/O queue pair are connected and handed off to the kernel to
+create a new controller device.
+The following flags control properties of each I/O association:
+.Bl -tag
+.It Fl D
+Enable TCP PDU data digests.
+.It Fl F
+Request submission queue flow control.
+By default submission queue flow control is disabled unless the remote
+controller requires it.
+.It Fl H
+Enable TCP PDU header digests.
+.El
+.Ss disconnect
+Delete the controller device associated with a remote I/O controller
+including any active association and open queues.
+.Ss reconnect
+Reestablish an association for the remote I/O controller associated with
+.Ar device-id
+at
+.Ar address .
+The address must include a port.
+The following flags control properties of the association:
+.Bl -tag
+.It Fl D
+Enable TCP PDU data digests.
+.It Fl F
+Request submission queue flow control.
+By default submission queue flow control is disabled unless the remote
+controller requires it.
+.It Fl H
+Enable TCP PDU header digests.
+.El
.Sh DEVICE NAMES
Where
.Aq Ar namespace-id
@@ -626,6 +734,37 @@
of
.Dq 0
means query the drive itself.
+.Sh FABRICS TRANSPORTS
+The following NVM Express over Fabrics transports are supported for
+accessing remote controllers:
+.Bl -tag
+.It tcp
+TCP transport
+.El
+.Sh NETWORK ADDRESSES
+Network addresses for remote controllers can use one of the following formats:
+.Bl -bullet
+.It
+.Bq Ar IPv6 address
+.Ns : Ns Ar port
+.It
+.Ar IPv4 address
+.Ns : Ns Ar port
+.It
+.Ar hostname Ns : Ns Ar port
+.It
+.Bq Ar IPv6 address
+.It
+.Ar IPv6 address
+.It
+.Ar IPv4 address
+.It
+.Ar hostname
+.El
+.Pp
+If a
+.Ar port
+is not provided, a default value is used if possible.
.Sh EXAMPLES
.Dl nvmecontrol devlist
.Pp
diff --git a/sbin/nvmecontrol/reconnect.c b/sbin/nvmecontrol/reconnect.c
new file mode 100644
--- /dev/null
+++ b/sbin/nvmecontrol/reconnect.c
@@ -0,0 +1,178 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2024 Chelsio Communications, Inc.
+ * Written by: John Baldwin <jhb@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <err.h>
+#include <libnvmf.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sysexits.h>
+#include <unistd.h>
+
+#include "nvmecontrol.h"
+#include "fabrics.h"
+
+/*
+ * See comment about other possible settings in connect.c.
+ */
+
+static struct options {
+ const char *dev;
+ const char *transport;
+ const char *address;
+ bool data_digests;
+ bool flow_control;
+ bool header_digests;
+} opt = {
+ .dev = NULL,
+ .transport = "tcp",
+ .address = NULL,
+ .data_digests = false,
+ .flow_control = false,
+ .header_digests = false,
+};
+
+static void
+tcp_association_params(struct nvmf_association_params *params)
+{
+ params->tcp.pda = 0;
+ params->tcp.header_digests = opt.header_digests;
+ params->tcp.data_digests = opt.data_digests;
+ /* XXX */
+ params->tcp.maxr2t = 1;
+}
+
+static int
+reconnect_nvm_controller(int fd, enum nvmf_trtype trtype, int adrfam,
+ const char *address, const char *port)
+{
+ struct nvme_controller_data cdata;
+ struct nvmf_association_params aparams;
+ struct nvmf_reconnect_params rparams;
+ struct nvmf_qpair *admin, *io[1];
+ int error;
+
+ error = nvmf_reconnect_params(fd, &rparams);
+ if (error != 0) {
+ warnc(error, "Failed to fetch reconnect parameters");
+ return (EX_IOERR);
+ }
+
+ memset(&aparams, 0, sizeof(aparams));
+ aparams.sq_flow_control = opt.flow_control;
+ switch (trtype) {
+ case NVMF_TRTYPE_TCP:
+ tcp_association_params(&aparams);
+ break;
+ default:
+ warnx("Unsupported transport %s", nvmf_transport_type(trtype));
+ return (EX_UNAVAILABLE);
+ }
+
+ error = connect_nvm_queues(&aparams, trtype, adrfam, address, port,
+ rparams.cntlid, rparams.subnqn, &admin, io, nitems(io), &cdata);
+ if (error != 0)
+ return (error);
+
+ error = nvmf_reconnect_host(fd, admin, 1, io, &cdata);
+ if (error != 0) {
+ warnc(error, "Failed to handoff queues to kernel");
+ return (EX_IOERR);
+ }
+ return (0);
+}
+
+static void
+reconnect_static(int fd, enum nvmf_trtype trtype, const char *address,
+ const char *port)
+{
+ int error;
+
+ if (port == NULL)
+ errx(EX_USAGE, "Explicit port required");
+
+ error = reconnect_nvm_controller(fd, trtype, AF_UNSPEC, address, port);
+ if (error != 0)
+ exit(error);
+}
+
+static void
+reconnect_fn(const struct cmd *f, int argc, char *argv[])
+{
+ enum nvmf_trtype trtype;
+ const char *address, *port;
+ char *tofree;
+ int fd;
+
+ if (arg_parse(argc, argv, f))
+ return;
+
+ if (strcasecmp(opt.transport, "tcp") == 0) {
+ trtype = NVMF_TRTYPE_TCP;
+ } else
+ errx(EX_USAGE, "Unsupported or invalid transport");
+
+ nvmf_parse_address(opt.address, &address, &port, &tofree);
+
+ open_dev(opt.dev, &fd, 1, 1);
+ reconnect_static(fd, trtype, address, port);
+
+ close(fd);
+ free(tofree);
+}
+
+static const struct opts reconnect_opts[] = {
+#define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc }
+ OPT("transport", 't', arg_string, opt, transport,
+ "Transport type"),
+ OPT("header_digests", 'H', arg_none, opt, header_digests,
+ "Enable TCP PDU header digests"),
+ OPT("data_digests", 'D', arg_none, opt, data_digests,
+ "Enable TCP PDU data digests"),
+ OPT("flow_control", 'F', arg_none, opt, flow_control,
+ "Request SQ flow control"),
+ { NULL, 0, arg_none, NULL, NULL }
+};
+#undef OPT
+
+static const struct args reconnect_args[] = {
+ { arg_string, &opt.dev, "controller-id" },
+ { arg_string, &opt.address, "address" },
+ { arg_none, NULL, NULL },
+};
+
+static struct cmd reconnect_cmd = {
+ .name = "reconnect",
+ .fn = reconnect_fn,
+ .descr = "Reconnect to a fabrics controller",
+ .ctx_size = sizeof(opt),
+ .opts = reconnect_opts,
+ .args = reconnect_args,
+};
+
+CMD_COMMAND(reconnect_cmd);

File Metadata

Mime Type
text/plain
Expires
Mon, Apr 13, 6:58 AM (15 h, 53 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
31398992
Default Alt Text
D44715.id136811.diff (44 KB)

Event Timeline