nvmf: Refactor reconnection support

Save more data associated with a new association including the network
address of the remote controller.  This permits reconnecting an
association without providing the address or other details.  To use
this new mode, provide only an existing device ID to nvmecontrol's
reconnect command.  An address can still be provided to request a
different address or other different settings for the new association.

The saved data includes an entire Discovery Log page entry to aim to
be compatible with other transports in the future.  When a remote
controller is connected to via a Discovery Log page entry (nvmecontrol
connect-all), the raw entry is used.  When a remote controller is
connected to via an explicit address, an entry is synthesized from the
parameters.

Note that this is a pseudo-ABI break for the ioctls used by nvmf(4) in
that the nvlists for handoff and reconnect now use a slightly
different set of elements.  Since this is only present in main I did
not bother implementing compatability shims.

Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D48214

5 files changed, 262 insertions, 72 deletions

diff --git a/sbin/nvmecontrol/connect.c b/sbin/nvmecontrol/connect.c
index ef614eca2e2a..c1d5d2cbaf5a 100644
--- a/sbin/nvmecontrol/connect.c
+++ b/sbin/nvmecontrol/connect.c
@@ -62,11 +62,14 @@ tcp_association_params(struct nvmf_association_params *params)
 
 static int
 connect_nvm_controller(enum nvmf_trtype trtype, int adrfam, const char *address,
-    const char *port, uint16_t cntlid, const char *subnqn)
+    const char *port, uint16_t cntlid, const char *subnqn,
+    const struct nvme_discovery_log_entry *dle)
 {
 	struct nvme_controller_data cdata;
+	struct nvme_discovery_log_entry dle_thunk;
 	struct nvmf_association_params aparams;
 	struct nvmf_qpair *admin, **io;
+	const char *hostnqn;
 	int error;
 
 	memset(&aparams, 0, sizeof(aparams));
@@ -80,16 +83,31 @@ connect_nvm_controller(enum nvmf_trtype trtype, int adrfam, const char *address,
 		return (EX_UNAVAILABLE);
 	}
 
+	hostnqn = opt.hostnqn;
+	if (hostnqn == NULL)
+		hostnqn = nvmf_default_hostnqn();
 	io = calloc(opt.num_io_queues, sizeof(*io));
 	error = connect_nvm_queues(&aparams, trtype, adrfam, address, port,
-	    cntlid, subnqn, opt.hostnqn, opt.kato * 1000, &admin, io,
+	    cntlid, subnqn, hostnqn, opt.kato * 1000, &admin, io,
 	    opt.num_io_queues, opt.queue_size, &cdata);
 	if (error != 0) {
 		free(io);
 		return (error);
 	}
 
-	error = nvmf_handoff_host(admin, opt.num_io_queues, io, &cdata);
+	if (dle == NULL) {
+		error = nvmf_init_dle_from_admin_qp(admin, &cdata, &dle_thunk);
+		if (error != 0) {
+			warnc(error, "Failed to generate handoff parameters");
+			disconnect_nvm_queues(admin, io, opt.num_io_queues);
+			free(io);
+			return (EX_IOERR);
+		}
+		dle = &dle_thunk;
+	}
+
+	error = nvmf_handoff_host(dle, hostnqn, admin, opt.num_io_queues, io,
+	    &cdata);
 	if (error != 0) {
 		warnc(error, "Failed to handoff queues to kernel");
 		free(io);
@@ -140,7 +158,7 @@ connect_discovery_entry(struct nvme_discovery_log_entry *entry)
 
 	/* XXX: Should this make use of entry->aqsz in some way? */
 	connect_nvm_controller(entry->trtype, adrfam, entry->traddr,
-	    entry->trsvcid, entry->cntlid, entry->subnqn);
+	    entry->trsvcid, entry->cntlid, entry->subnqn, entry);
 }
 
 static void
@@ -198,7 +216,7 @@ connect_fn(const struct cmd *f, int argc, char *argv[])
 	cntlid = nvmf_parse_cntlid(opt.cntlid);
 
 	error = connect_nvm_controller(trtype, AF_UNSPEC, address, port, cntlid,
-	    opt.subnqn);
+	    opt.subnqn, NULL);
 	if (error != 0)
 		exit(error);
 
diff --git a/sbin/nvmecontrol/fabrics.c b/sbin/nvmecontrol/fabrics.c
index 27ecc8fb6cdc..1d90a62dd3e3 100644
--- a/sbin/nvmecontrol/fabrics.c
+++ b/sbin/nvmecontrol/fabrics.c
@@ -7,6 +7,7 @@
 
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <arpa/inet.h>
 #include <err.h>
 #include <libnvmf.h>
 #include <netdb.h>
@@ -48,6 +49,14 @@ init_hostid(void)
 	return (true);
 }
 
+const char *
+nvmf_default_hostnqn(void)
+{
+	if (!init_hostid())
+		exit(EX_IOERR);
+	return (nqn);
+}
+
 void
 nvmf_parse_address(const char *in_address, const char **address,
     const char **port, char **tofree)
@@ -137,7 +146,7 @@ nvmf_parse_cntlid(const char *cntlid)
 	}
 }
 
-bool
+static bool
 tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam,
     const char *address, const char *port)
 {
@@ -422,13 +431,10 @@ connect_nvm_queues(const struct nvmf_association_params *aparams,
 
 	if (!init_hostid())
 		return (EX_IOERR);
-	if (hostnqn != NULL) {
-		if (!nvmf_nqn_valid(hostnqn)) {
-			warnx("Invalid HostNQN %s", hostnqn);
-			return (EX_USAGE);
-		}
-	} else
-		hostnqn = nqn;
+	if (hostnqn == NULL || !nvmf_nqn_valid(hostnqn)) {
+		warnx("Invalid HostNQN %s", hostnqn);
+		return (EX_USAGE);
+	}
 
 	/* Association. */
 	na = nvmf_allocate_association(trtype, false, aparams);
@@ -510,12 +516,19 @@ connect_nvm_queues(const struct nvmf_association_params *aparams,
 	return (0);
 
 out:
+	disconnect_nvm_queues(*admin, io, num_io_queues);
+	nvmf_free_association(na);
+	return (error);
+}
+
+void
+disconnect_nvm_queues(struct nvmf_qpair *admin, struct nvmf_qpair **io,
+    u_int num_io_queues)
+{
 	for (u_int i = 0; i < num_io_queues; i++) {
 		if (io[i] == NULL)
 			break;
 		nvmf_free_qpair(io[i]);
 	}
-	shutdown_controller(*admin);
-	nvmf_free_association(na);
-	return (error);
+	shutdown_controller(admin);
 }
diff --git a/sbin/nvmecontrol/fabrics.h b/sbin/nvmecontrol/fabrics.h
index 9d6ee24b88fb..1f43fc53bb8f 100644
--- a/sbin/nvmecontrol/fabrics.h
+++ b/sbin/nvmecontrol/fabrics.h
@@ -18,9 +18,11 @@ void	nvmf_parse_address(const char *in_address, const char **address,
 
 uint16_t nvmf_parse_cntlid(const char *cntlid);
 
-/* Returns true if able to open a connection. */
-bool	tcp_qpair_params(struct nvmf_qpair_params *params, int adrfam,
-    const char *address, const char *port);
+const char *nvmf_default_hostnqn(void);
+
+int	nvmf_init_dle_from_address(enum nvmf_trtype trtype, const char *address,
+    const char *port, uint16_t cntlid, const char *subnqn,
+    struct nvme_discovery_log_entry *dle);
 
 /* Connect to a discovery controller and return the Admin qpair. */
 struct nvmf_qpair *connect_discovery_adminq(enum nvmf_trtype trtype,
@@ -38,4 +40,11 @@ int	connect_nvm_queues(const struct nvmf_association_params *aparams,
     uint32_t kato, struct nvmf_qpair **admin, struct nvmf_qpair **io,
     u_int num_io_queues, u_int queue_size, struct nvme_controller_data *cdata);
 
+/*
+ * Disconnect from an NVM controller disconnecting all queues and
+ * shutting down the controller.
+ */
+void	disconnect_nvm_queues(struct nvmf_qpair *admin, struct nvmf_qpair **io,
+    u_int num_io_queues);
+
 #endif /* !__FABRICS_H__ */
diff --git a/sbin/nvmecontrol/nvmecontrol.8 b/sbin/nvmecontrol/nvmecontrol.8
index e21079b17ed6..a9f4552fbdc6 100644
--- a/sbin/nvmecontrol/nvmecontrol.8
+++ b/sbin/nvmecontrol/nvmecontrol.8
@@ -235,6 +235,9 @@
 .Aq Ar device-id | Ar namespace-id | Ar SubNQN
 .Nm
 .Ic reconnect
+.Aq Ar device-id
+.Nm
+.Ic reconnect
 .Op Fl FGg
 .Op Fl i Ar queues
 .Op Fl k Ar seconds
@@ -813,11 +816,20 @@ Delete the controller device associated with a remote I/O controller
 including any active association and open queues.
 .Ss reconnect
 Reestablish an association for the remote I/O controller associated with
-.Ar device-id
-at
-.Ar address .
-The address must include a port.
-The flags have the same meaning for the new association as described above
+.Ar device-id .
+If an
+.Ar address
+is not provided,
+the resolved address and settings from the previous association are used
+to establish a new association.
+If an
+.Ar address
+is provided,
+the supplied address and command line flags are used to establish a new
+association.
+In this case,
+the address must include a port and
+the flags have the same meaning for the new association as described above
 for the
 .Cm connect
 command.
diff --git a/sbin/nvmecontrol/reconnect.c b/sbin/nvmecontrol/reconnect.c
index 4c9277bd34cb..adf1edac662b 100644
--- a/sbin/nvmecontrol/reconnect.c
+++ b/sbin/nvmecontrol/reconnect.c
@@ -5,6 +5,7 @@
  * Written by: John Baldwin <jhb@FreeBSD.org>
  */
 
+#include <sys/dnv.h>
 #include <sys/nv.h>
 #include <sys/socket.h>
 #include <err.h>
@@ -24,7 +25,6 @@
 static struct options {
 	const char	*dev;
 	const char	*transport;
-	const char	*address;
 	const char	*hostnqn;
 	uint32_t	kato;
 	uint16_t	num_io_queues;
@@ -35,7 +35,6 @@ static struct options {
 } opt = {
 	.dev = NULL,
 	.transport = "tcp",
-	.address = NULL,
 	.hostnqn = NULL,
 	.kato = NVMF_KATO_DEFAULT / 1000,
 	.num_io_queues = 1,
@@ -46,100 +45,240 @@ static struct options {
 };
 
 static void
-tcp_association_params(struct nvmf_association_params *params)
+tcp_association_params(struct nvmf_association_params *params,
+    bool header_digests, bool data_digests)
 {
 	params->tcp.pda = 0;
-	params->tcp.header_digests = opt.header_digests;
-	params->tcp.data_digests = opt.data_digests;
+	params->tcp.header_digests = header_digests;
+	params->tcp.data_digests = data_digests;
 	/* XXX */
 	params->tcp.maxr2t = 1;
 }
 
 static int
-reconnect_nvm_controller(int fd, enum nvmf_trtype trtype, int adrfam,
-    const char *address, const char *port)
+reconnect_nvm_controller(int fd, const struct nvmf_association_params *aparams,
+    enum nvmf_trtype trtype, int adrfam, const char *address, const char *port,
+    uint16_t cntlid, const char *subnqn, const char *hostnqn, uint32_t kato,
+    u_int num_io_queues, u_int queue_size,
+    const struct nvme_discovery_log_entry *dle)
 {
 	struct nvme_controller_data cdata;
-	struct nvmf_association_params aparams;
-	nvlist_t *rparams;
+	struct nvme_discovery_log_entry dle_thunk;
 	struct nvmf_qpair *admin, **io;
 	int error;
 
-	error = nvmf_reconnect_params(fd, &rparams);
+	io = calloc(num_io_queues, sizeof(*io));
+	error = connect_nvm_queues(aparams, trtype, adrfam, address, port,
+	    cntlid, subnqn, hostnqn, kato, &admin, io, num_io_queues,
+	    queue_size, &cdata);
 	if (error != 0) {
-		warnc(error, "Failed to fetch reconnect parameters");
-		return (EX_IOERR);
+		free(io);
+		return (error);
 	}
 
-	if (!nvlist_exists_number(rparams, "cntlid") ||
-	    !nvlist_exists_string(rparams, "subnqn")) {
-		nvlist_destroy(rparams);
-		warnx("Missing required reconnect parameters");
+	if (dle == NULL) {
+		error = nvmf_init_dle_from_admin_qp(admin, &cdata, &dle_thunk);
+		if (error != 0) {
+			warnc(error, "Failed to generate handoff parameters");
+			disconnect_nvm_queues(admin, io, num_io_queues);
+			free(io);
+			return (EX_IOERR);
+		}
+		dle = &dle_thunk;
+	}
+
+	error = nvmf_reconnect_host(fd, dle, hostnqn, admin, num_io_queues, io,
+	    &cdata);
+	if (error != 0) {
+		warnc(error, "Failed to handoff queues to kernel");
+		free(io);
 		return (EX_IOERR);
 	}
+	free(io);
+	return (0);
+}
+
+static int
+reconnect_by_address(int fd, const nvlist_t *rparams, const char *addr)
+{
+	const struct nvme_discovery_log_entry *dle;
+	struct nvmf_association_params aparams;
+	enum nvmf_trtype trtype;
+	const char *address, *hostnqn, *port;
+	char *subnqn, *tofree;
+	int error;
 
 	memset(&aparams, 0, sizeof(aparams));
 	aparams.sq_flow_control = opt.flow_control;
-	switch (trtype) {
+	if (strcasecmp(opt.transport, "tcp") == 0) {
+		trtype = NVMF_TRTYPE_TCP;
+		tcp_association_params(&aparams, opt.header_digests,
+		    opt.data_digests);
+	} else {
+		warnx("Unsupported or invalid transport");
+		return (EX_USAGE);
+	}
+
+	nvmf_parse_address(addr, &address, &port, &tofree);
+	if (port == NULL) {
+		free(tofree);
+		warnx("Explicit port required");
+		return (EX_USAGE);
+	}
+
+	dle = nvlist_get_binary(rparams, "dle", NULL);
+
+	hostnqn = opt.hostnqn;
+	if (hostnqn == NULL)
+		hostnqn = nvmf_default_hostnqn();
+
+	/* Ensure subnqn is a terminated C string. */
+	subnqn = strndup(dle->subnqn, sizeof(dle->subnqn));
+
+	error = reconnect_nvm_controller(fd, &aparams, trtype, AF_UNSPEC,
+	    address, port, le16toh(dle->cntlid), subnqn, hostnqn,
+	    opt.kato * 1000, opt.num_io_queues, opt.queue_size, NULL);
+	free(subnqn);
+	free(tofree);
+	return (error);
+}
+
+static int
+reconnect_by_params(int fd, const nvlist_t *rparams)
+{
+	struct nvmf_association_params aparams;
+	const struct nvme_discovery_log_entry *dle;
+	char *address, *port, *subnqn;
+	int adrfam, error;
+
+	dle = nvlist_get_binary(rparams, "dle", NULL);
+
+	memset(&aparams, 0, sizeof(aparams));
+	aparams.sq_flow_control = nvlist_get_bool(rparams, "sq_flow_control");
+	switch (dle->trtype) {
 	case NVMF_TRTYPE_TCP:
-		tcp_association_params(&aparams);
+		switch (dle->adrfam) {
+		case NVMF_ADRFAM_IPV4:
+			adrfam = AF_INET;
+			break;
+		case NVMF_ADRFAM_IPV6:
+			adrfam = AF_INET6;
+			break;
+		default:
+			warnx("Unsupported address family");
+			return (EX_UNAVAILABLE);
+		}
+		switch (dle->tsas.tcp.sectype) {
+		case NVME_TCP_SECURITY_NONE:
+			break;
+		default:
+			warnx("Unsupported TCP security type");
+			return (EX_UNAVAILABLE);
+		}
+		break;
+
+		tcp_association_params(&aparams,
+		    nvlist_get_bool(rparams, "header_digests"),
+		    nvlist_get_bool(rparams, "data_digests"));
 		break;
 	default:
-		nvlist_destroy(rparams);
-		warnx("Unsupported transport %s", nvmf_transport_type(trtype));
+		warnx("Unsupported transport %s",
+		    nvmf_transport_type(dle->trtype));
 		return (EX_UNAVAILABLE);
 	}
 
-	io = calloc(opt.num_io_queues, sizeof(*io));
-	error = connect_nvm_queues(&aparams, trtype, adrfam, address, port,
-	    nvlist_get_number(rparams, "cntlid"),
-	    nvlist_get_string(rparams, "subnqn"), opt.hostnqn, opt.kato,
-	    &admin, io, opt.num_io_queues, opt.queue_size, &cdata);
+	/* Ensure address, port, and subnqn is a terminated C string. */
+	address = strndup(dle->traddr, sizeof(dle->traddr));
+	port = strndup(dle->trsvcid, sizeof(dle->trsvcid));
+	subnqn = strndup(dle->subnqn, sizeof(dle->subnqn));
+
+	error = reconnect_nvm_controller(fd, &aparams, dle->trtype, adrfam,
+	    address, port, le16toh(dle->cntlid), dle->subnqn,
+	    nvlist_get_string(rparams, "hostnqn"),
+	    dnvlist_get_number(rparams, "kato", 0),
+	    nvlist_get_number(rparams, "num_io_queues"),
+	    nvlist_get_number(rparams, "io_qsize"), dle);
+	free(subnqn);
+	free(port);
+	free(address);
+	return (error);
+}
+
+static int
+fetch_and_validate_rparams(int fd, nvlist_t **rparamsp)
+{
+	const struct nvme_discovery_log_entry *dle;
+	nvlist_t *rparams;
+	size_t len;
+	int error;
+
+	error = nvmf_reconnect_params(fd, &rparams);
 	if (error != 0) {
-		free(io);
+		warnc(error, "Failed to fetch reconnect parameters");
+		return (EX_IOERR);
+	}
+
+	if (!nvlist_exists_binary(rparams, "dle") ||
+	    !nvlist_exists_string(rparams, "hostnqn") ||
+	    !nvlist_exists_number(rparams, "num_io_queues") ||
+	    !nvlist_exists_number(rparams, "io_qsize") ||
+	    !nvlist_exists_bool(rparams, "sq_flow_control")) {
 		nvlist_destroy(rparams);
-		return (error);
+		warnx("Missing required reconnect parameters");
+		return (EX_IOERR);
 	}
-	nvlist_destroy(rparams);
 
-	error = nvmf_reconnect_host(fd, admin, opt.num_io_queues, io, &cdata);
-	if (error != 0) {
-		warnc(error, "Failed to handoff queues to kernel");
-		free(io);
+	dle = nvlist_get_binary(rparams, "dle", &len);
+	if (len != sizeof(*dle)) {
+		nvlist_destroy(rparams);
+		warnx("Discovery Log entry reconnect parameter is wrong size");
 		return (EX_IOERR);
 	}
-	free(io);
+
+	switch (dle->trtype) {
+	case NVMF_TRTYPE_TCP:
+		if (!nvlist_exists_bool(rparams, "header_digests") ||
+		    !nvlist_exists_bool(rparams, "data_digests")) {
+			nvlist_destroy(rparams);
+			warnx("Missing required reconnect parameters");
+			return (EX_IOERR);
+		}
+		break;
+	default:
+		nvlist_destroy(rparams);
+		warnx("Unsupported transport %s",
+		    nvmf_transport_type(dle->trtype));
+		return (EX_UNAVAILABLE);
+	}
+
+	*rparamsp = rparams;
 	return (0);
 }
 
 static void
 reconnect_fn(const struct cmd *f, int argc, char *argv[])
 {
-	enum nvmf_trtype trtype;
-	const char *address, *port;
-	char *tofree;
+	nvlist_t *rparams;
 	int error, fd;
 
 	if (arg_parse(argc, argv, f))
 		return;
 
-	if (strcasecmp(opt.transport, "tcp") == 0) {
-		trtype = NVMF_TRTYPE_TCP;
-	} else
-		errx(EX_USAGE, "Unsupported or invalid transport");
-
-	nvmf_parse_address(opt.address, &address, &port, &tofree);
-
 	open_dev(opt.dev, &fd, 1, 1);
-	if (port == NULL)
-		errx(EX_USAGE, "Explicit port required");
+	error = fetch_and_validate_rparams(fd, &rparams);
+	if (error != 0)
+		exit(error);
 
-	error = reconnect_nvm_controller(fd, trtype, AF_UNSPEC, address, port);
+	/* Check for optional address. */
+	if (optind < argc)
+		error = reconnect_by_address(fd, rparams, argv[optind]);
+	else
+		error = reconnect_by_params(fd, rparams);
 	if (error != 0)
 		exit(error);
 
+	nvlist_destroy(rparams);
 	close(fd);
-	free(tofree);
 }
 
 static const struct opts reconnect_opts[] = {
@@ -166,7 +305,6 @@ static const struct opts reconnect_opts[] = {
 
 static const struct args reconnect_args[] = {
 	{ arg_string, &opt.dev, "controller-id" },
-	{ arg_string, &opt.address, "address" },
 	{ arg_none, NULL, NULL },
 };