diff --git a/lib/libnvmf/libnvmf.h b/lib/libnvmf/libnvmf.h --- a/lib/libnvmf/libnvmf.h +++ b/lib/libnvmf/libnvmf.h @@ -332,7 +332,8 @@ * Handoff active host association to the kernel. This frees the * qpairs (even on error). */ -int nvmf_handoff_host(struct nvmf_qpair *admin_qp, u_int num_queues, +int nvmf_handoff_host(const struct nvme_discovery_log_entry *dle, + const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues, struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata); /* @@ -359,8 +360,8 @@ * Handoff active host association to an existing host in the kernel. * This frees the qpairs (even on error). */ -int nvmf_reconnect_host(int fd, struct nvmf_qpair *admin_qp, - u_int num_queues, struct nvmf_qpair **io_queues, - const struct nvme_controller_data *cdata); +int nvmf_reconnect_host(int fd, const struct nvme_discovery_log_entry *dle, + const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues, + struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata); #endif /* !__LIBNVMF_H__ */ diff --git a/lib/libnvmf/nvmf_host.c b/lib/libnvmf/nvmf_host.c --- a/lib/libnvmf/nvmf_host.c +++ b/lib/libnvmf/nvmf_host.c @@ -107,8 +107,8 @@ memset(&data, 0, sizeof(data)); memcpy(data.hostid, hostid, sizeof(data.hostid)); data.cntlid = htole16(cntlid); - strlcpy(data.subnqn, subnqn, sizeof(data.subnqn)); - strlcpy(data.hostnqn, hostnqn, sizeof(data.hostnqn)); + strncpy(data.subnqn, subnqn, sizeof(data.subnqn)); + strncpy(data.hostnqn, hostnqn, sizeof(data.hostnqn)); error = nvmf_capsule_append_data(cc, &data, sizeof(data), true); if (error != 0) { @@ -767,17 +767,23 @@ } static int -prepare_queues_for_handoff(struct nvmf_ioc_nv *nv, struct nvmf_qpair *admin_qp, - u_int num_queues, struct nvmf_qpair **io_queues, - const struct nvme_controller_data *cdata) +prepare_queues_for_handoff(struct nvmf_ioc_nv *nv, + const struct nvme_discovery_log_entry *dle, const char *hostnqn, + struct nvmf_qpair *admin_qp, u_int num_queues, + struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata) { - nvlist_t *nvl, *nvl_qp; + const struct nvmf_association *na = admin_qp->nq_association; + nvlist_t *nvl, *nvl_qp, *nvl_rparams; u_int i; int error; if (num_queues == 0) return (EINVAL); + /* Ensure trtype matches. */ + if (dle->trtype != na->na_trtype) + return (EINVAL); + /* All queue pairs must be idle. */ if (!is_queue_pair_idle(admin_qp)) return (EBUSY); @@ -786,9 +792,35 @@ return (EBUSY); } + /* Fill out reconnect parameters. */ + nvl_rparams = nvlist_create(0); + nvlist_add_binary(nvl_rparams, "dle", dle, sizeof(*dle)); + nvlist_add_string(nvl_rparams, "hostnqn", hostnqn); + nvlist_add_number(nvl_rparams, "num_io_queues", num_queues); + nvlist_add_number(nvl_rparams, "kato", admin_qp->nq_kato); + nvlist_add_number(nvl_rparams, "io_qsize", io_queues[0]->nq_qsize); + nvlist_add_bool(nvl_rparams, "sq_flow_control", + na->na_params.sq_flow_control); + switch (na->na_trtype) { + case NVMF_TRTYPE_TCP: + nvlist_add_bool(nvl_rparams, "header_digests", + na->na_params.tcp.header_digests); + nvlist_add_bool(nvl_rparams, "data_digests", + na->na_params.tcp.data_digests); + break; + default: + __unreachable(); + } + error = nvlist_error(nvl_rparams); + if (error != 0) { + nvlist_destroy(nvl_rparams); + return (error); + } + nvl = nvlist_create(0); - nvlist_add_number(nvl, "trtype", admin_qp->nq_association->na_trtype); + nvlist_add_number(nvl, "trtype", na->na_trtype); nvlist_add_number(nvl, "kato", admin_qp->nq_kato); + nvlist_move_nvlist(nvl, "rparams", nvl_rparams); /* First, the admin queue. */ error = nvmf_kernel_handoff_params(admin_qp, &nvl_qp); @@ -816,7 +848,8 @@ } int -nvmf_handoff_host(struct nvmf_qpair *admin_qp, u_int num_queues, +nvmf_handoff_host(const struct nvme_discovery_log_entry *dle, + const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues, struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata) { struct nvmf_ioc_nv nv; @@ -829,8 +862,8 @@ goto out; } - error = prepare_queues_for_handoff(&nv, admin_qp, num_queues, io_queues, - cdata); + error = prepare_queues_for_handoff(&nv, dle, hostnqn, admin_qp, + num_queues, io_queues, cdata); if (error != 0) goto out; @@ -924,15 +957,16 @@ } int -nvmf_reconnect_host(int fd, struct nvmf_qpair *admin_qp, u_int num_queues, +nvmf_reconnect_host(int fd, const struct nvme_discovery_log_entry *dle, + const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues, struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata) { struct nvmf_ioc_nv nv; u_int i; int error; - error = prepare_queues_for_handoff(&nv, admin_qp, num_queues, io_queues, - cdata); + error = prepare_queues_for_handoff(&nv, dle, hostnqn, admin_qp, + num_queues, io_queues, cdata); if (error != 0) goto out; diff --git a/sbin/nvmecontrol/connect.c b/sbin/nvmecontrol/connect.c --- a/sbin/nvmecontrol/connect.c +++ b/sbin/nvmecontrol/connect.c @@ -61,35 +61,39 @@ } static int -connect_nvm_controller(enum nvmf_trtype trtype, int adrfam, const char *address, - const char *port, uint16_t cntlid, const char *subnqn) +connect_nvm_controller(const struct nvme_discovery_log_entry *dle) { struct nvme_controller_data cdata; struct nvmf_association_params aparams; struct nvmf_qpair *admin, **io; + const char *hostnqn; int error; memset(&aparams, 0, sizeof(aparams)); aparams.sq_flow_control = opt.flow_control; - switch (trtype) { + switch (dle->trtype) { case NVMF_TRTYPE_TCP: tcp_association_params(&aparams); break; default: - warnx("Unsupported transport %s", nvmf_transport_type(trtype)); + warnx("Unsupported transport %s", + nvmf_transport_type(dle->trtype)); return (EX_UNAVAILABLE); } + hostnqn = opt.hostnqn; + if (hostnqn == NULL) + hostnqn = nvmf_default_hostnqn(); io = calloc(opt.num_io_queues, sizeof(*io)); - error = connect_nvm_queues(&aparams, trtype, adrfam, address, port, - cntlid, subnqn, opt.hostnqn, opt.kato * 1000, &admin, io, - opt.num_io_queues, opt.queue_size, &cdata); + error = connect_nvm_queues(&aparams, dle, hostnqn, opt.kato * 1000, + &admin, io, opt.num_io_queues, opt.queue_size, &cdata); if (error != 0) { free(io); return (error); } - error = nvmf_handoff_host(admin, opt.num_io_queues, io, &cdata); + error = nvmf_handoff_host(dle, hostnqn, admin, opt.num_io_queues, io, + &cdata); if (error != 0) { warnc(error, "Failed to handoff queues to kernel"); free(io); @@ -99,19 +103,29 @@ return (0); } +static int +connect_by_address(enum nvmf_trtype trtype, const char *address, + const char *port, uint16_t cntlid, const char *subnqn) +{ + struct nvme_discovery_log_entry dle; + int error; + + error = nvmf_init_dle_from_address(trtype, address, port, cntlid, + subnqn, &dle); + if (error != 0) + return (error); + + return (connect_nvm_controller(&dle)); +} + static void connect_discovery_entry(struct nvme_discovery_log_entry *entry) { - int adrfam; - switch (entry->trtype) { case NVMF_TRTYPE_TCP: switch (entry->adrfam) { case NVMF_ADRFAM_IPV4: - adrfam = AF_INET; - break; case NVMF_ADRFAM_IPV6: - adrfam = AF_INET6; break; default: warnx("Skipping unsupported address family for %s", @@ -139,8 +153,7 @@ */ /* XXX: Should this make use of entry->aqsz in some way? */ - connect_nvm_controller(entry->trtype, adrfam, entry->traddr, - entry->trsvcid, entry->cntlid, entry->subnqn); + connect_nvm_controller(entry); } static void @@ -197,8 +210,7 @@ cntlid = nvmf_parse_cntlid(opt.cntlid); - error = connect_nvm_controller(trtype, AF_UNSPEC, address, port, cntlid, - opt.subnqn); + error = connect_by_address(trtype, address, port, cntlid, opt.subnqn); if (error != 0) exit(error); diff --git a/sbin/nvmecontrol/fabrics.h b/sbin/nvmecontrol/fabrics.h --- a/sbin/nvmecontrol/fabrics.h +++ b/sbin/nvmecontrol/fabrics.h @@ -18,9 +18,11 @@ uint16_t nvmf_parse_cntlid(const char *cntlid); -/* Returns true if able to open a connection. */ -bool tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam, - const char *address, const char *port); +const char *nvmf_default_hostnqn(void); + +int nvmf_init_dle_from_address(enum nvmf_trtype trtype, const char *address, + const char *port, uint16_t cntlid, const char *subnqn, + struct nvme_discovery_log_entry *dle); /* Connect to a discovery controller and return the Admin qpair. */ struct nvmf_qpair *connect_discovery_adminq(enum nvmf_trtype trtype, @@ -33,8 +35,7 @@ * failure. */ int connect_nvm_queues(const struct nvmf_association_params *aparams, - enum nvmf_trtype trtype, int adrfam, const char *address, - const char *port, uint16_t cntlid, const char *subnqn, const char *hostnqn, + const struct nvme_discovery_log_entry *dle, const char *hostnqn, uint32_t kato, struct nvmf_qpair **admin, struct nvmf_qpair **io, u_int num_io_queues, u_int queue_size, struct nvme_controller_data *cdata); diff --git a/sbin/nvmecontrol/fabrics.c b/sbin/nvmecontrol/fabrics.c --- a/sbin/nvmecontrol/fabrics.c +++ b/sbin/nvmecontrol/fabrics.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,14 @@ return (true); } +const char * +nvmf_default_hostnqn(void) +{ + if (!init_hostid()) + exit(EX_IOERR); + return (nqn); +} + void nvmf_parse_address(const char *in_address, const char **address, const char **port, char **tofree) @@ -137,7 +146,80 @@ } } -bool +int +nvmf_init_dle_from_address(enum nvmf_trtype trtype, const char *address, + const char *port, uint16_t cntlid, const char *subnqn, + struct nvme_discovery_log_entry *dle) +{ + struct addrinfo hints, *ai, *list; + const struct sockaddr_in *sin; + const struct sockaddr_in6 *sin6; + const void *in_addr; + int error, s; + uint16_t tcp_port; + + switch (trtype) { + case NVMF_TRTYPE_TCP: + break; + default: + warnx("Unsupported transport %s", nvmf_transport_type(trtype)); + return (EX_UNAVAILABLE); + } + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_protocol = IPPROTO_TCP; + error = getaddrinfo(address, port, &hints, &list); + if (error != 0) { + warnx("%s", gai_strerror(error)); + return (EX_NOHOST); + } + + for (ai = list; ai != NULL; ai = ai->ai_next) { + s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); + if (s == -1) + continue; + + if (connect(s, ai->ai_addr, ai->ai_addrlen) != 0) { + close(s); + continue; + } + close(s); + + goto found; + } + warn("Failed to connect to controller at %s:%s", address, port); + freeaddrinfo(list); + return (EX_NOHOST); + +found: + memset(dle, 0, sizeof(*dle)); + dle->trtype = NVMF_TRTYPE_TCP; + switch (ai->ai_family) { + case AF_INET: + dle->adrfam = NVMF_ADRFAM_IPV4; + sin = (const void *)ai->ai_addr; + in_addr = &sin->sin_addr; + tcp_port = ntohs(sin->sin_port); + break; + case AF_INET6: + dle->adrfam = NVMF_ADRFAM_IPV6; + sin6 = (const void *)ai->ai_addr; + in_addr = &sin6->sin6_addr; + tcp_port = ntohs(sin6->sin6_port); + break; + default: + __unreachable(); + } + dle->cntlid = htole16(cntlid); + snprintf(dle->trsvcid, sizeof(dle->trsvcid), "%u", tcp_port); + strncpy(dle->subnqn, subnqn, sizeof(dle->subnqn)); + inet_ntop(ai->ai_family, in_addr, dle->traddr, sizeof(dle->traddr)); + freeaddrinfo(list); + return (0); +} + +static bool tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam, const char *address, const char *port) { @@ -287,7 +369,7 @@ qp = nvmf_connect(na, params, 0, NVMF_MIN_ADMIN_MAX_SQ_SIZE, hostid, cntlid, subnqn, hostnqn, kato); if (qp == NULL) { - warnx("Failed to connect to NVM controller %s: %s", subnqn, + warnx("Failed to connect to NVM controller %.256s: %s", subnqn, nvmf_association_error(na)); return (EX_IOERR); } @@ -302,7 +384,7 @@ /* Require the NVM command set. */ if (NVME_CAP_HI_CSS_NVM(cap >> 32) == 0) { - warnx("Controller %s does not support the NVM command set", + warnx("Controller %.256s does not support the NVM command set", subnqn); nvmf_free_qpair(qp); return (EX_UNAVAILABLE); @@ -401,51 +483,68 @@ /* Returns a value from */ int connect_nvm_queues(const struct nvmf_association_params *aparams, - enum nvmf_trtype trtype, int adrfam, const char *address, - const char *port, uint16_t cntlid, const char *subnqn, const char *hostnqn, + const struct nvme_discovery_log_entry *dle, const char *hostnqn, uint32_t kato, struct nvmf_qpair **admin, struct nvmf_qpair **io, u_int num_io_queues, u_int queue_size, struct nvme_controller_data *cdata) { struct nvmf_qpair_params qparams; struct nvmf_association *na; u_int queues; - int error; - uint16_t mqes; + int adrfam, error; + uint16_t cntlid, mqes; - switch (trtype) { + cntlid = le16toh(dle->cntlid); + switch (dle->trtype) { case NVMF_TRTYPE_TCP: + switch (dle->adrfam) { + case NVMF_ADRFAM_IPV4: + adrfam = AF_INET; + break; + case NVMF_ADRFAM_IPV6: + adrfam = AF_INET6; + break; + default: + warnx("Unsupported address family %u", dle->adrfam); + return (EX_UNAVAILABLE); + } + switch (dle->tsas.tcp.sectype) { + case NVME_TCP_SECURITY_NONE: + break; + default: + warnx("Unsupported TCP security type %u", + dle->tsas.tcp.sectype); + return (EX_UNAVAILABLE); + } break; default: - warnx("Unsupported transport %s", nvmf_transport_type(trtype)); + warnx("Unsupported transport %s", + nvmf_transport_type(dle->trtype)); return (EX_UNAVAILABLE); } if (!init_hostid()) return (EX_IOERR); - if (hostnqn != NULL) { - if (!nvmf_nqn_valid(hostnqn)) { - warnx("Invalid HostNQN %s", hostnqn); - return (EX_USAGE); - } - } else - hostnqn = nqn; + if (hostnqn == NULL || !nvmf_nqn_valid(hostnqn)) { + warnx("Invalid HostNQN %s", hostnqn); + return (EX_USAGE); + } /* Association. */ - na = nvmf_allocate_association(trtype, false, aparams); + na = nvmf_allocate_association(dle->trtype, false, aparams); if (na == NULL) { - warn("Failed to create association for %s", subnqn); + warn("Failed to create association for %.256s", dle->subnqn); return (EX_IOERR); } /* Admin queue. */ memset(&qparams, 0, sizeof(qparams)); qparams.admin = true; - if (!tcp_qpair_params(&qparams, adrfam, address, port)) { + if (!tcp_qpair_params(&qparams, adrfam, dle->traddr, dle->trsvcid)) { nvmf_free_association(na); return (EX_NOHOST); } - error = connect_nvm_adminq(na, &qparams, admin, cntlid, subnqn, hostnqn, - kato, &mqes); + error = connect_nvm_adminq(na, &qparams, admin, cntlid, dle->subnqn, + hostnqn, kato, &mqes); if (error != 0) { nvmf_free_association(na); return (error); @@ -467,7 +566,8 @@ if (error != 0) { shutdown_controller(*admin); nvmf_free_association(na); - warnc(error, "Failed to fetch controller data for %s", subnqn); + warnc(error, "Failed to fetch controller data for %.256s", + dle->subnqn); return (EX_IOERR); } @@ -493,12 +593,13 @@ for (u_int i = 0; i < num_io_queues; i++) { memset(&qparams, 0, sizeof(qparams)); qparams.admin = false; - if (!tcp_qpair_params(&qparams, adrfam, address, port)) { + if (!tcp_qpair_params(&qparams, adrfam, dle->traddr, + dle->trsvcid)) { error = EX_NOHOST; goto out; } io[i] = nvmf_connect(na, &qparams, i + 1, queue_size, hostid, - nvmf_cntlid(*admin), subnqn, hostnqn, 0); + nvmf_cntlid(*admin), dle->subnqn, hostnqn, 0); if (io[i] == NULL) { warnx("Failed to create I/O queue: %s", nvmf_association_error(na)); diff --git a/sbin/nvmecontrol/nvmecontrol.8 b/sbin/nvmecontrol/nvmecontrol.8 --- a/sbin/nvmecontrol/nvmecontrol.8 +++ b/sbin/nvmecontrol/nvmecontrol.8 @@ -235,6 +235,9 @@ .Aq Ar device-id | Ar namespace-id | Ar SubNQN .Nm .Ic reconnect +.Aq Ar device-id +.Nm +.Ic reconnect .Op Fl FGg .Op Fl i Ar queues .Op Fl k Ar seconds @@ -813,11 +816,20 @@ including any active association and open queues. .Ss reconnect Reestablish an association for the remote I/O controller associated with -.Ar device-id -at -.Ar address . -The address must include a port. -The flags have the same meaning for the new association as described above +.Ar device-id . +If an +.Ar address +is not provided, +the resolved address and settings from the previous association are used +to establish a new association. +If an +.Ar address +is provided, +the supplied address and command line flags are used to establish a new +association. +In this case, +the address must include a port and +the flags have the same meaning for the new association as described above for the .Cm connect command. diff --git a/sbin/nvmecontrol/reconnect.c b/sbin/nvmecontrol/reconnect.c --- a/sbin/nvmecontrol/reconnect.c +++ b/sbin/nvmecontrol/reconnect.c @@ -5,6 +5,7 @@ * Written by: John Baldwin */ +#include #include #include #include @@ -46,63 +47,54 @@ }; static void -tcp_association_params(struct nvmf_association_params *params) +tcp_association_params(struct nvmf_association_params *params, + const nvlist_t *rparams) { params->tcp.pda = 0; - params->tcp.header_digests = opt.header_digests; - params->tcp.data_digests = opt.data_digests; + params->tcp.header_digests = nvlist_get_bool(rparams, "header_digests"); + params->tcp.data_digests = nvlist_get_bool(rparams, "data_digests"); /* XXX */ params->tcp.maxr2t = 1; } static int -reconnect_nvm_controller(int fd, enum nvmf_trtype trtype, int adrfam, - const char *address, const char *port) +reconnect_nvm_controller(int fd, const nvlist_t *rparams) { + const struct nvme_discovery_log_entry *dle; struct nvme_controller_data cdata; struct nvmf_association_params aparams; - nvlist_t *rparams; struct nvmf_qpair *admin, **io; + const char *hostnqn; + u_int num_io_queues; int error; - error = nvmf_reconnect_params(fd, &rparams); - if (error != 0) { - warnc(error, "Failed to fetch reconnect parameters"); - return (EX_IOERR); - } - - if (!nvlist_exists_number(rparams, "cntlid") || - !nvlist_exists_string(rparams, "subnqn")) { - nvlist_destroy(rparams); - warnx("Missing required reconnect parameters"); - return (EX_IOERR); - } + dle = nvlist_get_binary(rparams, "dle", NULL); + num_io_queues = nvlist_get_number(rparams, "num_io_queues"); + hostnqn = nvlist_get_string(rparams, "hostnqn"); memset(&aparams, 0, sizeof(aparams)); - aparams.sq_flow_control = opt.flow_control; - switch (trtype) { + aparams.sq_flow_control = nvlist_get_bool(rparams, "sq_flow_control"); + switch (dle->trtype) { case NVMF_TRTYPE_TCP: - tcp_association_params(&aparams); + tcp_association_params(&aparams, rparams); break; default: - nvlist_destroy(rparams); - warnx("Unsupported transport %s", nvmf_transport_type(trtype)); + warnx("Unsupported transport %s", + nvmf_transport_type(dle->trtype)); return (EX_UNAVAILABLE); } - io = calloc(opt.num_io_queues, sizeof(*io)); - error = connect_nvm_queues(&aparams, trtype, adrfam, address, port, - nvlist_get_number(rparams, "cntlid"), - nvlist_get_string(rparams, "subnqn"), opt.hostnqn, opt.kato, - &admin, io, opt.num_io_queues, opt.queue_size, &cdata); + io = calloc(num_io_queues, sizeof(*io)); + error = connect_nvm_queues(&aparams, dle, hostnqn, + dnvlist_get_number(rparams, "kato", 0), &admin, io, num_io_queues, + nvlist_get_number(rparams, "io_qsize"), &cdata); if (error != 0) { free(io); - nvlist_destroy(rparams); return (error); } - nvlist_destroy(rparams); - error = nvmf_reconnect_host(fd, admin, opt.num_io_queues, io, &cdata); + error = nvmf_reconnect_host(fd, dle, hostnqn, admin, num_io_queues, io, + &cdata); if (error != 0) { warnc(error, "Failed to handoff queues to kernel"); free(io); @@ -112,34 +104,136 @@ return (0); } -static void -reconnect_fn(const struct cmd *f, int argc, char *argv[]) +static int +reconnect_params_from_address(nvlist_t **rparamsp) { + const struct nvme_discovery_log_entry *old_dle; + struct nvme_discovery_log_entry dle; enum nvmf_trtype trtype; - const char *address, *port; + const char *address, *hostnqn, *port; + nvlist_t *old_rparams, *new_rparams; char *tofree; - int error, fd; - - if (arg_parse(argc, argv, f)) - return; + int error; if (strcasecmp(opt.transport, "tcp") == 0) { trtype = NVMF_TRTYPE_TCP; - } else - errx(EX_USAGE, "Unsupported or invalid transport"); + } else { + warnx("Unsupported or invalid transport"); + return (EX_USAGE); + } nvmf_parse_address(opt.address, &address, &port, &tofree); + if (port == NULL) { + free(tofree); + warnx("Explicit port required"); + return (EX_USAGE); + } + + old_rparams = *rparamsp; + old_dle = nvlist_get_binary(old_rparams, "dle", NULL); + error = nvmf_init_dle_from_address(trtype, address, port, + le16toh(old_dle->cntlid), old_dle->subnqn, &dle); + free(tofree); + if (error != 0) + return (error); + + hostnqn = opt.hostnqn; + if (hostnqn == NULL) + hostnqn = nvmf_default_hostnqn(); + + new_rparams = nvlist_create(0); + nvlist_add_binary(new_rparams, "dle", &dle, sizeof(dle)); + nvlist_add_string(new_rparams, "hostnqn", hostnqn); + nvlist_add_number(new_rparams, "num_io_queues", opt.num_io_queues); + nvlist_add_number(new_rparams, "kato", opt.kato * 1000); + nvlist_add_number(new_rparams, "io_qsize", opt.queue_size); + nvlist_add_bool(new_rparams, "sq_flow_control", opt.flow_control); + nvlist_add_bool(new_rparams, "header_digests", opt.header_digests); + nvlist_add_bool(new_rparams, "data_digests", opt.data_digests); + nvlist_destroy(old_rparams); + *rparamsp = new_rparams; + return (0); +} + +static int +fetch_and_validate_rparams(int fd, nvlist_t **rparamsp) +{ + const struct nvme_discovery_log_entry *dle; + nvlist_t *rparams; + size_t len; + int error; + + error = nvmf_reconnect_params(fd, &rparams); + if (error != 0) { + warnc(error, "Failed to fetch reconnect parameters"); + return (EX_IOERR); + } + + if (!nvlist_exists_binary(rparams, "dle") || + !nvlist_exists_string(rparams, "hostnqn") || + !nvlist_exists_number(rparams, "num_io_queues") || + !nvlist_exists_number(rparams, "io_qsize") || + !nvlist_exists_bool(rparams, "sq_flow_control")) { + nvlist_destroy(rparams); + warnx("Missing required reconnect parameters"); + return (EX_IOERR); + } + + dle = nvlist_get_binary(rparams, "dle", &len); + if (len != sizeof(*dle)) { + nvlist_destroy(rparams); + warnx("Discovery Log entry reconnect parameter is wrong size"); + return (EX_IOERR); + } + + switch (dle->trtype) { + case NVMF_TRTYPE_TCP: + if (!nvlist_exists_bool(rparams, "header_digests") || + !nvlist_exists_bool(rparams, "data_digests")) { + nvlist_destroy(rparams); + warnx("Missing required reconnect parameters"); + return (EX_IOERR); + } + break; + default: + nvlist_destroy(rparams); + warnx("Unsupported transport %s", + nvmf_transport_type(dle->trtype)); + return (EX_UNAVAILABLE); + } + + *rparamsp = rparams; + return (0); +} + +static void +reconnect_fn(const struct cmd *f, int argc, char *argv[]) +{ + nvlist_t *rparams; + int error, fd; + + if (arg_parse(argc, argv, f)) + return; open_dev(opt.dev, &fd, 1, 1); - if (port == NULL) - errx(EX_USAGE, "Explicit port required"); + error = fetch_and_validate_rparams(fd, &rparams); + if (error != 0) + exit(error); + + /* Check for optional address. */ + if (optind < argc) { + opt.address = argv[optind]; + error = reconnect_params_from_address(&rparams); + if (error != 0) + exit(error); + } - error = reconnect_nvm_controller(fd, trtype, AF_UNSPEC, address, port); + error = reconnect_nvm_controller(fd, rparams); if (error != 0) exit(error); + nvlist_destroy(rparams); close(fd); - free(tofree); } static const struct opts reconnect_opts[] = { @@ -166,7 +260,6 @@ static const struct args reconnect_args[] = { { arg_string, &opt.dev, "controller-id" }, - { arg_string, &opt.address, "address" }, { arg_none, NULL, NULL }, }; diff --git a/sys/dev/nvmf/host/nvmf.c b/sys/dev/nvmf/host/nvmf.c --- a/sys/dev/nvmf/host/nvmf.c +++ b/sys/dev/nvmf/host/nvmf.c @@ -200,8 +200,10 @@ int nvmf_copyin_handoff(const struct nvmf_ioc_nv *nv, nvlist_t **nvlp) { + const struct nvme_discovery_log_entry *dle; + const struct nvme_controller_data *cdata; const nvlist_t *const *io; - const nvlist_t *admin; + const nvlist_t *admin, *rparams; nvlist_t *nvl; size_t i, num_io_queues; uint32_t qsize; @@ -214,7 +216,15 @@ if (!nvlist_exists_number(nvl, "trtype") || !nvlist_exists_nvlist(nvl, "admin") || !nvlist_exists_nvlist_array(nvl, "io") || - !nvlist_exists_binary(nvl, "cdata")) + !nvlist_exists_binary(nvl, "cdata") || + !nvlist_exists_nvlist(nvl, "rparams")) + goto invalid; + + rparams = nvlist_get_nvlist(nvl, "rparams"); + if (!nvlist_exists_binary(rparams, "dle") || + !nvlist_exists_string(rparams, "hostnqn") || + !nvlist_exists_number(rparams, "num_io_queues") || + !nvlist_exists_number(rparams, "io_qsize")) goto invalid; admin = nvlist_get_nvlist(nvl, "admin"); @@ -224,7 +234,8 @@ goto invalid; io = nvlist_get_nvlist_array(nvl, "io", &num_io_queues); - if (num_io_queues < 1) + if (num_io_queues < 1 || + num_io_queues != nvlist_get_number(rparams, "num_io_queues")) goto invalid; for (i = 0; i < num_io_queues; i++) { if (!nvmf_validate_qpair_nvlist(io[i], false)) @@ -232,14 +243,20 @@ } /* Require all I/O queues to be the same size. */ - qsize = nvlist_get_number(io[0], "qsize"); - for (i = 1; i < num_io_queues; i++) { + qsize = nvlist_get_number(rparams, "io_qsize"); + for (i = 0; i < num_io_queues; i++) { if (nvlist_get_number(io[i], "qsize") != qsize) goto invalid; } - nvlist_get_binary(nvl, "cdata", &i); - if (i != sizeof(struct nvme_controller_data)) + cdata = nvlist_get_binary(nvl, "cdata", &i); + if (i != sizeof(*cdata)) + goto invalid; + dle = nvlist_get_binary(rparams, "dle", &i); + if (i != sizeof(*dle)) + goto invalid; + + if (memcmp(dle->subnqn, cdata->subnqn, sizeof(cdata->subnqn)) != 0) goto invalid; *nvlp = nvl; @@ -264,7 +281,7 @@ } static int -nvmf_establish_connection(struct nvmf_softc *sc, const nvlist_t *nvl) +nvmf_establish_connection(struct nvmf_softc *sc, nvlist_t *nvl) { const nvlist_t *const *io; const nvlist_t *admin; @@ -294,7 +311,7 @@ sc->io[i] = nvmf_init_qp(sc, trtype, io[i], name, i); if (sc->io[i] == NULL) { device_printf(sc->dev, "Failed to setup I/O queue %u\n", - i + 1); + i); return (ENXIO); } } @@ -314,6 +331,10 @@ memcpy(sc->cdata, nvlist_get_binary(nvl, "cdata", NULL), sizeof(*sc->cdata)); + /* Save reconnect parameters. */ + nvlist_destroy(sc->rparams); + sc->rparams = nvlist_take_nvlist(nvl, "rparams"); + return (0); } @@ -467,7 +488,7 @@ { struct make_dev_args mda; struct nvmf_softc *sc = device_get_softc(dev); - const nvlist_t *nvl = device_get_ivars(dev); + nvlist_t *nvl = device_get_ivars(dev); const nvlist_t * const *io; struct sysctl_oid *oid; uint64_t val; @@ -584,6 +605,7 @@ taskqueue_drain(taskqueue_thread, &sc->disconnect_task); sx_destroy(&sc->connection_lock); + nvlist_destroy(sc->rparams); free(sc->cdata, M_NVMF); return (error); } @@ -825,6 +847,7 @@ nvmf_destroy_aer(sc); sx_destroy(&sc->connection_lock); + nvlist_destroy(sc->rparams); free(sc->cdata, M_NVMF); return (0); } @@ -1041,21 +1064,12 @@ static int nvmf_reconnect_params(struct nvmf_softc *sc, struct nvmf_ioc_nv *nv) { - nvlist_t *nvl; int error; - nvl = nvlist_create(0); - sx_slock(&sc->connection_lock); - if ((sc->cdata->fcatt & 1) == 0) - nvlist_add_number(nvl, "cntlid", NVMF_CNTLID_DYNAMIC); - else - nvlist_add_number(nvl, "cntlid", sc->cdata->ctrlr_id); - nvlist_add_stringf(nvl, "subnqn", "%.256s", sc->cdata->subnqn); + error = nvmf_pack_ioc_nvlist(sc->rparams, nv); sx_sunlock(&sc->connection_lock); - error = nvmf_pack_ioc_nvlist(nvl, nv); - nvlist_destroy(nvl); return (error); } diff --git a/sys/dev/nvmf/host/nvmf_var.h b/sys/dev/nvmf/host/nvmf_var.h --- a/sys/dev/nvmf/host/nvmf_var.h +++ b/sys/dev/nvmf/host/nvmf_var.h @@ -84,6 +84,8 @@ struct sysctl_oid_list *ioq_oid_list; + nvlist_t *rparams; + eventhandler_tag shutdown_pre_sync_eh; eventhandler_tag shutdown_post_sync_eh; }; diff --git a/sys/dev/nvmf/nvmf.h b/sys/dev/nvmf/nvmf.h --- a/sys/dev/nvmf/nvmf.h +++ b/sys/dev/nvmf/nvmf.h @@ -71,13 +71,23 @@ * qpair handoff nvlist admin * qpair handoff nvlist array io * binary cdata struct nvme_controller_data + * NVMF_RECONNECT_PARAMS nvlist rparams */ /* * The fields in the nvlist for NVMF_RECONNECT_PARAMS are: * - * number cntlid - * string subnqn + * binary dle struct nvme_discovery_log_entry + * string hostnqn + * number num_io_queues + * number kato (optional) + * number io_qsize + * bool sq_flow_control + * + * TCP transport: + * + * bool header_digests + * bool data_digests */ /*