nvmf: Refactor reconnection support

Save more data associated with a new association including the network
address of the remote controller.  This permits reconnecting an
association without providing the address or other details.  To use
this new mode, provide only an existing device ID to nvmecontrol's
reconnect command.  An address can still be provided to request a
different address or other different settings for the new association.

The saved data includes an entire Discovery Log page entry to aim to
be compatible with other transports in the future.  When a remote
controller is connected to via a Discovery Log page entry (nvmecontrol
connect-all), the raw entry is used.  When a remote controller is
connected to via an explicit address, an entry is synthesized from the
parameters.

Note that this is a pseudo-ABI break for the ioctls used by nvmf(4) in
that the nvlists for handoff and reconnect now use a slightly
different set of elements.  Since this is only present in main I did
not bother implementing compatability shims.

Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D48214

5 files changed, 114 insertions, 15 deletions

diff --git a/lib/libnvmf/internal.h b/lib/libnvmf/internal.h
index 7b3d4fbb03ef..300776c1213b 100644
--- a/lib/libnvmf/internal.h
+++ b/lib/libnvmf/internal.h
@@ -26,6 +26,8 @@ struct nvmf_transport_ops {
 
 	/* Add params for kernel handoff. */
 	void (*kernel_handoff_params)(struct nvmf_qpair *qp, nvlist_t *nvl);
+	int (*populate_dle)(struct nvmf_qpair *qp,
+	    struct nvme_discovery_log_entry *dle);
 
 	/* Capsule operations. */
 	struct nvmf_capsule *(*allocate_capsule)(struct nvmf_qpair *qp);
@@ -111,6 +113,8 @@ void	na_clear_error(struct nvmf_association *na);
 void	na_error(struct nvmf_association *na, const char *fmt, ...);
 
 int	nvmf_kernel_handoff_params(struct nvmf_qpair *qp, nvlist_t **nvlp);
+int	nvmf_populate_dle(struct nvmf_qpair *qp,
+    struct nvme_discovery_log_entry *dle);
 int	nvmf_pack_ioc_nvlist(struct nvmf_ioc_nv *nv, nvlist_t *nvl);
 
 #endif /* !__LIBNVMF_INTERNAL_H__ */
diff --git a/lib/libnvmf/libnvmf.h b/lib/libnvmf/libnvmf.h
index 44f13fda5ddd..f34ccdb177e7 100644
--- a/lib/libnvmf/libnvmf.h
+++ b/lib/libnvmf/libnvmf.h
@@ -321,6 +321,14 @@ int	nvmf_host_fetch_discovery_log_page(struct nvmf_qpair *qp,
     struct nvme_discovery_log **logp);
 
 /*
+ * Construct a discovery log page entry that describes the connection
+ * used by a host association's admin queue pair.
+ */
+int	nvmf_init_dle_from_admin_qp(struct nvmf_qpair *qp,
+    const struct nvme_controller_data *cdata,
+    struct nvme_discovery_log_entry *dle);
+
+/*
  * Request a desired number of I/O queues via SET_FEATURES.  The
  * number of actual I/O queues available is returned in *actual on
  * success.
@@ -332,7 +340,8 @@ int	nvmf_host_request_queues(struct nvmf_qpair *qp, u_int requested,
  * Handoff active host association to the kernel.  This frees the
  * qpairs (even on error).
  */
-int	nvmf_handoff_host(struct nvmf_qpair *admin_qp, u_int num_queues,
+int	nvmf_handoff_host(const struct nvme_discovery_log_entry *dle,
+    const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues,
     struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata);
 
 /*
@@ -359,8 +368,8 @@ int	nvmf_reconnect_params(int fd, nvlist_t **nvlp);
  * Handoff active host association to an existing host in the kernel.
  * This frees the qpairs (even on error).
  */
-int	nvmf_reconnect_host(int fd, struct nvmf_qpair *admin_qp,
-    u_int num_queues, struct nvmf_qpair **io_queues,
-    const struct nvme_controller_data *cdata);
+int	nvmf_reconnect_host(int fd, const struct nvme_discovery_log_entry *dle,
+    const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues,
+    struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata);
 
 #endif /* !__LIBNVMF_H__ */
diff --git a/lib/libnvmf/nvmf_host.c b/lib/libnvmf/nvmf_host.c
index c3668600c463..e194522870d1 100644
--- a/lib/libnvmf/nvmf_host.c
+++ b/lib/libnvmf/nvmf_host.c
@@ -710,6 +710,27 @@ nvmf_host_fetch_discovery_log_page(struct nvmf_qpair *qp,
 }
 
 int
+nvmf_init_dle_from_admin_qp(struct nvmf_qpair *qp,
+    const struct nvme_controller_data *cdata,
+    struct nvme_discovery_log_entry *dle)
+{
+	int error;
+	uint16_t cntlid;
+
+	memset(dle, 0, sizeof(*dle));
+	error = nvmf_populate_dle(qp, dle);
+	if (error != 0)
+		return (error);
+	if ((cdata->fcatt & 1) == 0)
+		cntlid = NVMF_CNTLID_DYNAMIC;
+	else
+		cntlid = cdata->ctrlr_id;
+	dle->cntlid = htole16(cntlid);
+	memcpy(dle->subnqn, cdata->subnqn, sizeof(dle->subnqn));
+	return (0);
+}
+
+int
 nvmf_host_request_queues(struct nvmf_qpair *qp, u_int requested, u_int *actual)
 {
 	struct nvme_command cmd;
@@ -767,17 +788,23 @@ is_queue_pair_idle(struct nvmf_qpair *qp)
 }
 
 static int
-prepare_queues_for_handoff(struct nvmf_ioc_nv *nv, struct nvmf_qpair *admin_qp,
-    u_int num_queues, struct nvmf_qpair **io_queues,
-    const struct nvme_controller_data *cdata)
+prepare_queues_for_handoff(struct nvmf_ioc_nv *nv,
+    const struct nvme_discovery_log_entry *dle, const char *hostnqn,
+    struct nvmf_qpair *admin_qp, u_int num_queues,
+    struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata)
 {
-	nvlist_t *nvl, *nvl_qp;
+	const struct nvmf_association *na = admin_qp->nq_association;
+	nvlist_t *nvl, *nvl_qp, *nvl_rparams;
 	u_int i;
 	int error;
 
 	if (num_queues == 0)
 		return (EINVAL);
 
+	/* Ensure trtype matches. */
+	if (dle->trtype != na->na_trtype)
+		return (EINVAL);
+
 	/* All queue pairs must be idle. */
 	if (!is_queue_pair_idle(admin_qp))
 		return (EBUSY);
@@ -786,9 +813,35 @@ prepare_queues_for_handoff(struct nvmf_ioc_nv *nv, struct nvmf_qpair *admin_qp,
 			return (EBUSY);
 	}
 
+	/* Fill out reconnect parameters. */
+	nvl_rparams = nvlist_create(0);
+	nvlist_add_binary(nvl_rparams, "dle", dle, sizeof(*dle));
+	nvlist_add_string(nvl_rparams, "hostnqn", hostnqn);
+	nvlist_add_number(nvl_rparams, "num_io_queues", num_queues);
+	nvlist_add_number(nvl_rparams, "kato", admin_qp->nq_kato);
+	nvlist_add_number(nvl_rparams, "io_qsize", io_queues[0]->nq_qsize);
+	nvlist_add_bool(nvl_rparams, "sq_flow_control",
+	    na->na_params.sq_flow_control);
+	switch (na->na_trtype) {
+	case NVMF_TRTYPE_TCP:
+		nvlist_add_bool(nvl_rparams, "header_digests",
+		    na->na_params.tcp.header_digests);
+		nvlist_add_bool(nvl_rparams, "data_digests",
+		    na->na_params.tcp.data_digests);
+		break;
+	default:
+		__unreachable();
+	}
+	error = nvlist_error(nvl_rparams);
+	if (error != 0) {
+		nvlist_destroy(nvl_rparams);
+		return (error);
+	}
+
 	nvl = nvlist_create(0);
-	nvlist_add_number(nvl, "trtype", admin_qp->nq_association->na_trtype);
+	nvlist_add_number(nvl, "trtype", na->na_trtype);
 	nvlist_add_number(nvl, "kato", admin_qp->nq_kato);
+	nvlist_move_nvlist(nvl, "rparams", nvl_rparams);
 
 	/* First, the admin queue. */
 	error = nvmf_kernel_handoff_params(admin_qp, &nvl_qp);
@@ -816,7 +869,8 @@ prepare_queues_for_handoff(struct nvmf_ioc_nv *nv, struct nvmf_qpair *admin_qp,
 }
 
 int
-nvmf_handoff_host(struct nvmf_qpair *admin_qp, u_int num_queues,
+nvmf_handoff_host(const struct nvme_discovery_log_entry *dle,
+    const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues,
     struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata)
 {
 	struct nvmf_ioc_nv nv;
@@ -829,8 +883,8 @@ nvmf_handoff_host(struct nvmf_qpair *admin_qp, u_int num_queues,
 		goto out;
 	}
 
-	error = prepare_queues_for_handoff(&nv, admin_qp, num_queues, io_queues,
-	    cdata);
+	error = prepare_queues_for_handoff(&nv, dle, hostnqn, admin_qp,
+	    num_queues, io_queues, cdata);
 	if (error != 0)
 		goto out;
 
@@ -924,15 +978,16 @@ nvmf_reconnect_params(int fd, nvlist_t **nvlp)
 }
 
 int
-nvmf_reconnect_host(int fd, struct nvmf_qpair *admin_qp, u_int num_queues,
+nvmf_reconnect_host(int fd, const struct nvme_discovery_log_entry *dle,
+    const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues,
     struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata)
 {
 	struct nvmf_ioc_nv nv;
 	u_int i;
 	int error;
 
-	error = prepare_queues_for_handoff(&nv, admin_qp, num_queues, io_queues,
-	    cdata);
+	error = prepare_queues_for_handoff(&nv, dle, hostnqn, admin_qp,
+	    num_queues, io_queues, cdata);
 	if (error != 0)
 		goto out;
 
diff --git a/lib/libnvmf/nvmf_tcp.c b/lib/libnvmf/nvmf_tcp.c
index 3f794b5d9750..6f41ca7ff502 100644
--- a/lib/libnvmf/nvmf_tcp.c
+++ b/lib/libnvmf/nvmf_tcp.c
@@ -8,9 +8,11 @@
 #include <sys/endian.h>
 #include <sys/gsb_crc32.h>
 #include <sys/queue.h>
+#include <sys/socket.h>
 #include <sys/uio.h>
 #include <assert.h>
 #include <errno.h>
+#include <netdb.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -1144,6 +1146,25 @@ tcp_kernel_handoff_params(struct nvmf_qpair *nq, nvlist_t *nvl)
 	nvlist_add_number(nvl, "max_icd", qp->max_icd);
 }
 
+static int
+tcp_populate_dle(struct nvmf_qpair *nq, struct nvme_discovery_log_entry *dle)
+{
+	struct nvmf_tcp_qpair *qp = TQP(nq);
+	struct sockaddr_storage ss;
+	socklen_t ss_len;
+
+	ss_len = sizeof(ss);
+	if (getpeername(qp->s, (struct sockaddr *)&ss, &ss_len) == -1)
+		return (errno);
+
+	if (getnameinfo((struct sockaddr *)&ss, ss_len, dle->traddr,
+	    sizeof(dle->traddr), dle->trsvcid, sizeof(dle->trsvcid),
+	    NI_NUMERICHOST | NI_NUMERICSERV) != 0)
+		return (EINVAL);
+
+	return (0);
+}
+
 static struct nvmf_capsule *
 tcp_allocate_capsule(struct nvmf_qpair *qp __unused)
 {
@@ -1468,6 +1489,7 @@ struct nvmf_transport_ops tcp_ops = {
 	.allocate_qpair = tcp_allocate_qpair,
 	.free_qpair = tcp_free_qpair,
 	.kernel_handoff_params = tcp_kernel_handoff_params,
+	.populate_dle = tcp_populate_dle,
 	.allocate_capsule = tcp_allocate_capsule,
 	.free_capsule = tcp_free_capsule,
 	.transmit_capsule = tcp_transmit_capsule,
diff --git a/lib/libnvmf/nvmf_transport.c b/lib/libnvmf/nvmf_transport.c
index fa3826b8c50d..b105c17d2efb 100644
--- a/lib/libnvmf/nvmf_transport.c
+++ b/lib/libnvmf/nvmf_transport.c
@@ -259,6 +259,15 @@ nvmf_kernel_handoff_params(struct nvmf_qpair *qp, nvlist_t **nvlp)
 	return (0);
 }
 
+int
+nvmf_populate_dle(struct nvmf_qpair *qp, struct nvme_discovery_log_entry *dle)
+{
+	struct nvmf_association *na = qp->nq_association;
+
+	dle->trtype = na->na_trtype;
+	return (na->na_ops->populate_dle(qp, dle));
+}
+
 const char *
 nvmf_transport_type(uint8_t trtype)
 {