Rework CTL frontend & backend options to use nv(3), allow creating multiple

ioctl frontend ports.

This revision introduces two changes to CTL:
- Changes the way options are passed to CTL_LUN_REQ and CTL_PORT_REQ ioctls.
  Removes ctl_be_arg structure and associated logic and replaces it with
  nv(3)-based logic for passing in and out arguments.
- Allows creating multiple ioctl frontend ports using either ctladm(8) or
  ctld(8).
  New frontend ports are represented by /dev/cam/ctl<pp>.<vp> nodes, eg /dev/cam/ctl5.3.
  Those device nodes respond only to CTL_IO ioctl.

New command-line options for ctladm:
# creates new ioctl frontend port with using free pp and vp=0
ctladm port -c
# creates new ioctl frontend port with pp=10 and vp=0
ctladm port -c -O pp=10
# creates new ioctl frontend port with pp=11 and vp=12
ctladm port -c -O pp=11 -O vp=12
# removes port with number 4 (it's a "targ_port" number, not pp number)
ctladm port -r -p 4

New syntax for ctl.conf:
target ... {
    port ioctl/<pp>
    ...
}

target ... {
    port ioctl/<pp>/<vp>
    ...

Note: Most of this work was made by jceel@, thank you.

Submitted by:	jceel
Reworked by:	myself
Reviewed by:	mav (earlier versions and recently during the rework)
Obtained from:  FreeNAS and TrueOS
Relnotes:	Yes
Sponsored by:	iXsystems Inc.
Differential Revision:	https://reviews.freebsd.org/D9299

22 files changed, 933 insertions, 711 deletions

diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index 1e1343d4c0ae..c1e986ea42f1 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
  * Copyright (c) 2012 The FreeBSD Foundation
  * Copyright (c) 2014-2017 Alexander Motin <mav@FreeBSD.org>
+ * Copyright (c) 2017 Jakub Wojciech Klama <jceel@FreeBSD.org>
+ * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>
  * All rights reserved.
  *
  * Portions of this software were developed by Edward Tomasz Napierala
@@ -65,6 +67,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/smp.h>
 #include <sys/endian.h>
 #include <sys/sysctl.h>
+#include <sys/nv.h>
+#include <sys/dnv.h>
 #include <vm/uma.h>
 
 #include <cam/cam.h>
@@ -1869,6 +1873,7 @@ ctl_init(void)
 	args.mda_gid = GID_OPERATOR;
 	args.mda_mode = 0600;
 	args.mda_si_drv1 = softc;
+	args.mda_si_drv2 = NULL;
 	error = make_dev_s(&args, &softc->dev, "cam/ctl");
 	if (error != 0) {
 		free(softc, M_DEVBUF);
@@ -2468,105 +2473,6 @@ ctl_copyin_alloc(void *user_addr, unsigned int len, char *error_str,
 	return (kptr);
 }
 
-static void
-ctl_free_args(int num_args, struct ctl_be_arg *args)
-{
-	int i;
-
-	if (args == NULL)
-		return;
-
-	for (i = 0; i < num_args; i++) {
-		free(args[i].kname, M_CTL);
-		free(args[i].kvalue, M_CTL);
-	}
-
-	free(args, M_CTL);
-}
-
-static struct ctl_be_arg *
-ctl_copyin_args(int num_args, struct ctl_be_arg *uargs,
-		char *error_str, size_t error_str_len)
-{
-	struct ctl_be_arg *args;
-	int i;
-
-	args = ctl_copyin_alloc(uargs, num_args * sizeof(*args),
-				error_str, error_str_len);
-
-	if (args == NULL)
-		goto bailout;
-
-	for (i = 0; i < num_args; i++) {
-		args[i].kname = NULL;
-		args[i].kvalue = NULL;
-	}
-
-	for (i = 0; i < num_args; i++) {
-		uint8_t *tmpptr;
-
-		if (args[i].namelen == 0) {
-			snprintf(error_str, error_str_len, "Argument %d "
-				 "name length is zero", i);
-			goto bailout;
-		}
-
-		args[i].kname = ctl_copyin_alloc(args[i].name,
-			args[i].namelen, error_str, error_str_len);
-		if (args[i].kname == NULL)
-			goto bailout;
-
-		if (args[i].kname[args[i].namelen - 1] != '\0') {
-			snprintf(error_str, error_str_len, "Argument %d "
-				 "name is not NUL-terminated", i);
-			goto bailout;
-		}
-
-		if (args[i].flags & CTL_BEARG_RD) {
-			if (args[i].vallen == 0) {
-				snprintf(error_str, error_str_len, "Argument %d "
-					 "value length is zero", i);
-				goto bailout;
-			}
-
-			tmpptr = ctl_copyin_alloc(args[i].value,
-				args[i].vallen, error_str, error_str_len);
-			if (tmpptr == NULL)
-				goto bailout;
-
-			if ((args[i].flags & CTL_BEARG_ASCII)
-			 && (tmpptr[args[i].vallen - 1] != '\0')) {
-				snprintf(error_str, error_str_len, "Argument "
-				    "%d value is not NUL-terminated", i);
-				free(tmpptr, M_CTL);
-				goto bailout;
-			}
-			args[i].kvalue = tmpptr;
-		} else {
-			args[i].kvalue = malloc(args[i].vallen,
-			    M_CTL, M_WAITOK | M_ZERO);
-		}
-	}
-
-	return (args);
-bailout:
-
-	ctl_free_args(num_args, args);
-
-	return (NULL);
-}
-
-static void
-ctl_copyout_args(int num_args, struct ctl_be_arg *args)
-{
-	int i;
-
-	for (i = 0; i < num_args; i++) {
-		if (args[i].flags & CTL_BEARG_WR)
-			copyout(args[i].kvalue, args[i].value, args[i].vallen);
-	}
-}
-
 /*
  * Escape characters that are illegal or not recommended in XML.
  */
@@ -3038,8 +2944,12 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	case CTL_LUN_REQ: {
 		struct ctl_lun_req *lun_req;
 		struct ctl_backend_driver *backend;
+		void *packed;
+		nvlist_t *tmp_args_nvl;
+		size_t packed_len;
 
 		lun_req = (struct ctl_lun_req *)addr;
+		tmp_args_nvl = lun_req->args_nvl;
 
 		backend = ctl_backend_find(lun_req->backend);
 		if (backend == NULL) {
@@ -3050,32 +2960,68 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 				 lun_req->backend);
 			break;
 		}
-		if (lun_req->num_be_args > 0) {
-			lun_req->kern_be_args = ctl_copyin_args(
-				lun_req->num_be_args,
-				lun_req->be_args,
-				lun_req->error_str,
-				sizeof(lun_req->error_str));
-			if (lun_req->kern_be_args == NULL) {
+
+		if (lun_req->args != NULL) {
+			lun_req->args_nvl = nvlist_unpack(lun_req->args,
+			    lun_req->args_len, 0);
+
+			if (lun_req->args_nvl == NULL) {
 				lun_req->status = CTL_LUN_ERROR;
+				snprintf(lun_req->error_str, sizeof(lun_req->error_str),
+				    "Cannot unpack args nvlist.");
 				break;
 			}
-		}
+		} else
+			lun_req->args_nvl = nvlist_create(0);
 
 		retval = backend->ioctl(dev, cmd, addr, flag, td);
+		nvlist_destroy(lun_req->args_nvl);
+		lun_req->args_nvl = tmp_args_nvl;
+
+		if (lun_req->result_nvl != NULL) {
+			if (lun_req->result != NULL) {
+				packed = nvlist_pack(lun_req->result_nvl,
+				    &packed_len);
+				if (packed == NULL) {
+					lun_req->status = CTL_LUN_ERROR;
+					snprintf(lun_req->error_str,
+					    sizeof(lun_req->error_str),
+					    "Cannot pack result nvlist.");
+					break;
+				}
+
+				if (packed_len > lun_req->result_len) {
+					lun_req->status = CTL_LUN_ERROR;
+					snprintf(lun_req->error_str,
+					    sizeof(lun_req->error_str),
+					    "Result nvlist too large.");
+					free(packed, M_NVLIST);
+					break;
+				}
 
-		if (lun_req->num_be_args > 0) {
-			ctl_copyout_args(lun_req->num_be_args,
-				      lun_req->kern_be_args);
-			ctl_free_args(lun_req->num_be_args,
-				      lun_req->kern_be_args);
+				if (copyout(packed, lun_req->result, packed_len)) {
+					lun_req->status = CTL_LUN_ERROR;
+					snprintf(lun_req->error_str,
+					    sizeof(lun_req->error_str),
+					    "Cannot copyout() the result.");
+					free(packed, M_NVLIST);
+					break;
+				}
+
+				lun_req->result_len = packed_len;
+				free(packed, M_NVLIST);
+			}
+
+			nvlist_destroy(lun_req->result_nvl);
 		}
 		break;
 	}
 	case CTL_LUN_LIST: {
 		struct sbuf *sb;
 		struct ctl_lun_list *list;
-		struct ctl_option *opt;
+		const char *name, *value;
+		void *cookie;
+		int type;
 
 		list = (struct ctl_lun_list *)addr;
 
@@ -3201,11 +3147,20 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 				if (retval != 0)
 					break;
 			}
-			STAILQ_FOREACH(opt, &lun->be_lun->options, links) {
-				retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
-				    opt->name, opt->value, opt->name);
-				if (retval != 0)
-					break;
+
+			cookie = NULL;
+			while ((name = nvlist_next(lun->be_lun->options, &type,
+			    &cookie)) != NULL) {
+				sbuf_printf(sb, "\t<%s>", name);
+
+				if (type == NV_TYPE_STRING) {
+					value = dnvlist_get_string(
+					    lun->be_lun->options, name, NULL);
+					if (value != NULL)
+						sbuf_printf(sb, "%s", value);
+				}
+
+				sbuf_printf(sb, "</%s>\n", name);
 			}
 
 			retval = sbuf_printf(sb, "</lun>\n");
@@ -3259,8 +3214,12 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	case CTL_PORT_REQ: {
 		struct ctl_req *req;
 		struct ctl_frontend *fe;
+		void *packed;
+		nvlist_t *tmp_args_nvl;
+		size_t packed_len;
 
 		req = (struct ctl_req *)addr;
+		tmp_args_nvl = req->args_nvl;
 
 		fe = ctl_frontend_find(req->driver);
 		if (fe == NULL) {
@@ -3269,23 +3228,63 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 			    "Frontend \"%s\" not found.", req->driver);
 			break;
 		}
-		if (req->num_args > 0) {
-			req->kern_args = ctl_copyin_args(req->num_args,
-			    req->args, req->error_str, sizeof(req->error_str));
-			if (req->kern_args == NULL) {
+
+		if (req->args != NULL) {
+			req->args_nvl = nvlist_unpack(req->args,
+			    req->args_len, 0);
+
+			if (req->args_nvl == NULL) {
 				req->status = CTL_LUN_ERROR;
+				snprintf(req->error_str, sizeof(req->error_str),
+				    "Cannot unpack args nvlist.");
 				break;
 			}
-		}
+		} else
+			req->args_nvl = nvlist_create(0);
 
 		if (fe->ioctl)
 			retval = fe->ioctl(dev, cmd, addr, flag, td);
 		else
 			retval = ENODEV;
 
-		if (req->num_args > 0) {
-			ctl_copyout_args(req->num_args, req->kern_args);
-			ctl_free_args(req->num_args, req->kern_args);
+		nvlist_destroy(req->args_nvl);
+		req->args_nvl = tmp_args_nvl;
+
+		if (req->result_nvl != NULL) {
+			if (req->result != NULL) {
+				packed = nvlist_pack(req->result_nvl,
+				    &packed_len);
+				if (packed == NULL) {
+					req->status = CTL_LUN_ERROR;
+					snprintf(req->error_str,
+					    sizeof(req->error_str),
+					    "Cannot pack result nvlist.");
+					break;
+				}
+
+				if (packed_len > req->result_len) {
+					req->status = CTL_LUN_ERROR;
+					snprintf(req->error_str,
+					    sizeof(req->error_str),
+					    "Result nvlist too large.");
+					free(packed, M_NVLIST);
+					break;
+				}
+
+				if (copyout(packed, req->result, packed_len)) {
+					req->status = CTL_LUN_ERROR;
+					snprintf(req->error_str,
+					    sizeof(req->error_str),
+					    "Cannot copyout() the result.");
+					free(packed, M_NVLIST);
+					break;
+				}
+
+				req->result_len = packed_len;
+				free(packed, M_NVLIST);
+			}
+
+			nvlist_destroy(req->result_nvl);
 		}
 		break;
 	}
@@ -3293,8 +3292,9 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		struct sbuf *sb;
 		struct ctl_port *port;
 		struct ctl_lun_list *list;
-		struct ctl_option *opt;
-		int j;
+		const char *name, *value;
+		void *cookie;
+		int j, type;
 		uint32_t plun;
 
 		list = (struct ctl_lun_list *)addr;
@@ -3369,11 +3369,20 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 				if (retval != 0)
 					break;
 			}
-			STAILQ_FOREACH(opt, &port->options, links) {
-				retval = sbuf_printf(sb, "\t<%s>%s</%s>\n",
-				    opt->name, opt->value, opt->name);
-				if (retval != 0)
-					break;
+
+			cookie = NULL;
+			while ((name = nvlist_next(port->options, &type,
+			    &cookie)) != NULL) {
+				sbuf_printf(sb, "\t<%s>", name);
+
+				if (type == NV_TYPE_STRING) {
+					value = dnvlist_get_string(port->options,
+					    name, NULL);
+					if (value != NULL)
+						sbuf_printf(sb, "%s", value);
+				}
+
+				sbuf_printf(sb, "</%s>\n", name);
 			}
 
 			if (port->lun_map != NULL) {
@@ -4180,8 +4189,8 @@ ctl_init_page_index(struct ctl_lun *lun)
 				CTL_PAGE_DEFAULT];
 			scsi_ulto3b(cylinders, rigid_disk_page->cylinders);
 
-			if ((value = ctl_get_opt(&lun->be_lun->options,
-			    "rpm")) != NULL) {
+			if ((value = dnvlist_get_string(lun->be_lun->options,
+			    "rpm", NULL)) != NULL) {
 				scsi_ulto2b(strtol(value, NULL, 0),
 				     rigid_disk_page->rotation_rate);
 			}
@@ -4234,10 +4243,12 @@ ctl_init_page_index(struct ctl_lun *lun)
 			       sizeof(caching_page_default));
 			caching_page = &lun->mode_pages.caching_page[
 			    CTL_PAGE_SAVED];
-			value = ctl_get_opt(&lun->be_lun->options, "writecache");
+			value = dnvlist_get_string(lun->be_lun->options,
+			    "writecache", NULL);
 			if (value != NULL && strcmp(value, "off") == 0)
 				caching_page->flags1 &= ~SCP_WCE;
-			value = ctl_get_opt(&lun->be_lun->options, "readcache");
+			value = dnvlist_get_string(lun->be_lun->options,
+			    "readcache", NULL);
 			if (value != NULL && strcmp(value, "off") == 0)
 				caching_page->flags1 |= SCP_RCD;
 			memcpy(&lun->mode_pages.caching_page[CTL_PAGE_CURRENT],
@@ -4266,8 +4277,8 @@ ctl_init_page_index(struct ctl_lun *lun)
 				       sizeof(control_page_default));
 				control_page = &lun->mode_pages.control_page[
 				    CTL_PAGE_SAVED];
-				value = ctl_get_opt(&lun->be_lun->options,
-				    "reordering");
+				value = dnvlist_get_string(lun->be_lun->options,
+				    "reordering", NULL);
 				if (value != NULL &&
 				    strcmp(value, "unrestricted") == 0) {
 					control_page->queue_flags &=
@@ -4342,8 +4353,8 @@ ctl_init_page_index(struct ctl_lun *lun)
 				       &lbp_page_default,
 				       sizeof(lbp_page_default));
 				page = &lun->mode_pages.lbp_page[CTL_PAGE_SAVED];
-				value = ctl_get_opt(&lun->be_lun->options,
-				    "avail-threshold");
+				value = dnvlist_get_string(lun->be_lun->options,
+				    "avail-threshold", NULL);
 				if (value != NULL &&
 				    ctl_expand_number(value, &ival) == 0) {
 					page->descr[0].flags |= SLBPPD_ENABLED |
@@ -4355,8 +4366,8 @@ ctl_init_page_index(struct ctl_lun *lun)
 					scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
 					    page->descr[0].count);
 				}
-				value = ctl_get_opt(&lun->be_lun->options,
-				    "used-threshold");
+				value = dnvlist_get_string(lun->be_lun->options,
+				    "used-threshold", NULL);
 				if (value != NULL &&
 				    ctl_expand_number(value, &ival) == 0) {
 					page->descr[1].flags |= SLBPPD_ENABLED |
@@ -4368,8 +4379,8 @@ ctl_init_page_index(struct ctl_lun *lun)
 					scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
 					    page->descr[1].count);
 				}
-				value = ctl_get_opt(&lun->be_lun->options,
-				    "pool-avail-threshold");
+				value = dnvlist_get_string(lun->be_lun->options,
+				    "pool-avail-threshold", NULL);
 				if (value != NULL &&
 				    ctl_expand_number(value, &ival) == 0) {
 					page->descr[2].flags |= SLBPPD_ENABLED |
@@ -4381,8 +4392,8 @@ ctl_init_page_index(struct ctl_lun *lun)
 					scsi_ulto4b(ival >> CTL_LBP_EXPONENT,
 					    page->descr[2].count);
 				}
-				value = ctl_get_opt(&lun->be_lun->options,
-				    "pool-used-threshold");
+				value = dnvlist_get_string(lun->be_lun->options,
+				    "pool-used-threshold", NULL);
 				if (value != NULL &&
 				    ctl_expand_number(value, &ival) == 0) {
 					page->descr[3].flags |= SLBPPD_ENABLED |
@@ -4581,20 +4592,20 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
 	    strnlen(be_lun->device_id, CTL_DEVID_LEN));
 	idlen1 = sizeof(*t10id) + devidlen;
 	len = sizeof(struct scsi_vpd_id_descriptor) + idlen1;
-	scsiname = ctl_get_opt(&be_lun->options, "scsiname");
+	scsiname = dnvlist_get_string(be_lun->options, "scsiname", NULL);
 	if (scsiname != NULL) {
 		idlen2 = roundup2(strlen(scsiname) + 1, 4);
 		len += sizeof(struct scsi_vpd_id_descriptor) + idlen2;
 	}
-	eui = ctl_get_opt(&be_lun->options, "eui");
+	eui = dnvlist_get_string(be_lun->options, "eui", NULL);
 	if (eui != NULL) {
 		len += sizeof(struct scsi_vpd_id_descriptor) + 16;
 	}
-	naa = ctl_get_opt(&be_lun->options, "naa");
+	naa = dnvlist_get_string(be_lun->options, "naa", NULL);
 	if (naa != NULL) {
 		len += sizeof(struct scsi_vpd_id_descriptor) + 16;
 	}
-	uuid = ctl_get_opt(&be_lun->options, "uuid");
+	uuid = dnvlist_get_string(be_lun->options, "uuid", NULL);
 	if (uuid != NULL) {
 		len += sizeof(struct scsi_vpd_id_descriptor) + 18;
 	}
@@ -4606,7 +4617,7 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
 	desc->length = idlen1;
 	t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
 	memset(t10id->vendor, ' ', sizeof(t10id->vendor));
-	if ((vendor = ctl_get_opt(&be_lun->options, "vendor")) == NULL) {
+	if ((vendor = dnvlist_get_string(be_lun->options, "vendor", NULL)) == NULL) {
 		strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
 	} else {
 		strncpy(t10id->vendor, vendor,
@@ -4719,7 +4730,7 @@ fail:
 	if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
 		lun->flags |= CTL_LUN_PRIMARY_SC;
 
-	value = ctl_get_opt(&be_lun->options, "removable");
+	value = dnvlist_get_string(be_lun->options, "removable", NULL);
 	if (value != NULL) {
 		if (strcmp(value, "on") == 0)
 			lun->flags |= CTL_LUN_REMOVABLE;
@@ -9772,6 +9783,7 @@ ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
 {
 	struct ctl_lun *lun = CTL_LUN(ctsio);
 	struct scsi_vpd_block_limits *bl_ptr;
+	const char *val;
 	uint64_t ival;
 
 	ctsio->kern_data_ptr = malloc(sizeof(*bl_ptr), M_CTL, M_WAITOK | M_ZERO);
@@ -9801,12 +9813,16 @@ ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
 		scsi_ulto4b(lun->be_lun->opttxferlen, bl_ptr->opt_txfer_len);
 		if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
 			ival = 0xffffffff;
-			ctl_get_opt_number(&lun->be_lun->options,
-			    "unmap_max_lba", &ival);
+			val = dnvlist_get_string(lun->be_lun->options,
+			    "unmap_max_lba", NULL);
+			if (val != NULL)
+				ctl_expand_number(val, &ival);
 			scsi_ulto4b(ival, bl_ptr->max_unmap_lba_cnt);
 			ival = 0xffffffff;
-			ctl_get_opt_number(&lun->be_lun->options,
-			    "unmap_max_descr", &ival);
+			val = dnvlist_get_string(lun->be_lun->options,
+			    "unmap_max_descr", NULL);
+			if (val != NULL)
+				ctl_expand_number(val, &ival);
 			scsi_ulto4b(ival, bl_ptr->max_unmap_blk_cnt);
 			if (lun->be_lun->ublockexp != 0) {
 				scsi_ulto4b((1 << lun->be_lun->ublockexp),
@@ -9822,7 +9838,10 @@ ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
 		scsi_ulto4b(0, bl_ptr->max_atomic_transfer_length_with_atomic_boundary);
 		scsi_ulto4b(0, bl_ptr->max_atomic_boundary_size);
 		ival = UINT64_MAX;
-		ctl_get_opt_number(&lun->be_lun->options, "write_same_max_lba", &ival);
+		val = dnvlist_get_string(lun->be_lun->options,
+		    "write_same_max_lba", NULL);
+		if (val != NULL)
+			ctl_expand_number(val, &ival);
 		scsi_u64to8b(ival, bl_ptr->max_write_same_length);
 	}
 
@@ -9861,13 +9880,13 @@ ctl_inquiry_evpd_bdc(struct ctl_scsiio *ctsio, int alloc_len)
 	bdc_ptr->page_code = SVPD_BDC;
 	scsi_ulto2b(sizeof(*bdc_ptr) - 4, bdc_ptr->page_length);
 	if (lun != NULL &&
-	    (value = ctl_get_opt(&lun->be_lun->options, "rpm")) != NULL)
+	    (value = dnvlist_get_string(lun->be_lun->options, "rpm", NULL)) != NULL)
 		i = strtol(value, NULL, 0);
 	else
 		i = CTL_DEFAULT_ROTATION_RATE;
 	scsi_ulto2b(i, bdc_ptr->medium_rotation_rate);
 	if (lun != NULL &&
-	    (value = ctl_get_opt(&lun->be_lun->options, "formfactor")) != NULL)
+	    (value = dnvlist_get_string(lun->be_lun->options, "formfactor", NULL)) != NULL)
 		i = strtol(value, NULL, 0);
 	else
 		i = 0;
@@ -9912,7 +9931,8 @@ ctl_inquiry_evpd_lbp(struct ctl_scsiio *ctsio, int alloc_len)
 	if (lun != NULL && lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
 		lbp_ptr->flags = SVPD_LBP_UNMAP | SVPD_LBP_WS16 |
 		    SVPD_LBP_WS10 | SVPD_LBP_RZ | SVPD_LBP_ANC_SUP;
-		value = ctl_get_opt(&lun->be_lun->options, "provisioning_type");
+		value = dnvlist_get_string(lun->be_lun->options,
+		    "provisioning_type", NULL);
 		if (value != NULL) {
 			if (strcmp(value, "resource") == 0)
 				lbp_ptr->prov_type = SVPD_LBP_RESOURCE;
@@ -10006,7 +10026,7 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 	struct ctl_lun *lun = CTL_LUN(ctsio);
 	struct scsi_inquiry_data *inq_ptr;
 	struct scsi_inquiry *cdb;
-	char *val;
+	const char *val;
 	uint32_t alloc_len, data_len;
 	ctl_port_type port_type;
 
@@ -10084,8 +10104,8 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 	 * We have 8 bytes for the vendor name, and 16 bytes for the device
 	 * name and 4 bytes for the revision.
 	 */
-	if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
-	    "vendor")) == NULL) {
+	if (lun == NULL || (val = dnvlist_get_string(lun->be_lun->options,
+	    "vendor", NULL)) == NULL) {
 		strncpy(inq_ptr->vendor, CTL_VENDOR, sizeof(inq_ptr->vendor));
 	} else {
 		memset(inq_ptr->vendor, ' ', sizeof(inq_ptr->vendor));
@@ -10095,7 +10115,8 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 	if (lun == NULL) {
 		strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
 		    sizeof(inq_ptr->product));
-	} else if ((val = ctl_get_opt(&lun->be_lun->options, "product")) == NULL) {
+	} else if ((val = dnvlist_get_string(lun->be_lun->options, "product",
+	    NULL)) == NULL) {
 		switch (lun->be_lun->lun_type) {
 		case T_DIRECT:
 			strncpy(inq_ptr->product, CTL_DIRECT_PRODUCT,
@@ -10124,8 +10145,8 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 	 * XXX make this a macro somewhere so it automatically gets
 	 * incremented when we make changes.
 	 */
-	if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
-	    "revision")) == NULL) {
+	if (lun == NULL || (val = dnvlist_get_string(lun->be_lun->options,
+	    "revision", NULL)) == NULL) {
 		strncpy(inq_ptr->revision, "0001", sizeof(inq_ptr->revision));
 	} else {
 		memset(inq_ptr->revision, ' ', sizeof(inq_ptr->revision));
diff --git a/sys/cam/ctl/ctl.h b/sys/cam/ctl/ctl.h
index 05f62e75c862..9f8fd32e72f1 100644
--- a/sys/cam/ctl/ctl.h
+++ b/sys/cam/ctl/ctl.h
@@ -196,24 +196,6 @@ void ctl_isc_announce_iid(struct ctl_port *port, int iid);
 void ctl_isc_announce_mode(struct ctl_lun *lun, uint32_t initidx,
     uint8_t page, uint8_t subpage);
 
-/*
- * KPI to manipulate LUN/port options
- */
-
-struct ctl_option {
-	STAILQ_ENTRY(ctl_option)	links;
-	char			*name;
-	char			*value;
-};
-typedef STAILQ_HEAD(ctl_options, ctl_option) ctl_options_t;
-
-struct ctl_be_arg;
-void ctl_init_opts(ctl_options_t *opts, int num_args, struct ctl_be_arg *args);
-void ctl_update_opts(ctl_options_t *opts, int num_args,
-    struct ctl_be_arg *args);
-void ctl_free_opts(ctl_options_t *opts);
-char * ctl_get_opt(ctl_options_t *opts, const char *name);
-int ctl_get_opt_number(ctl_options_t *opts, const char *name, uint64_t *num);
 int ctl_expand_number(const char *buf, uint64_t *num);
 
 #endif	/* _KERNEL */
diff --git a/sys/cam/ctl/ctl_backend.c b/sys/cam/ctl/ctl_backend.c
index 9918ce404d58..0a0645556636 100644
--- a/sys/cam/ctl/ctl_backend.c
+++ b/sys/cam/ctl/ctl_backend.c
@@ -141,93 +141,3 @@ ctl_backend_find(char *backend_name)
 	return (NULL);
 }
 
-void
-ctl_init_opts(ctl_options_t *opts, int num_args, struct ctl_be_arg *args)
-{
-	struct ctl_option *opt;
-	int i;
-
-	STAILQ_INIT(opts);
-	for (i = 0; i < num_args; i++) {
-		if ((args[i].flags & CTL_BEARG_RD) == 0)
-			continue;
-		if ((args[i].flags & CTL_BEARG_ASCII) == 0)
-			continue;
-		opt = malloc(sizeof(*opt), M_CTL, M_WAITOK);
-		opt->name = strdup(args[i].kname, M_CTL);
-		opt->value = strdup(args[i].kvalue, M_CTL);
-		STAILQ_INSERT_TAIL(opts, opt, links);
-	}
-}
-
-void
-ctl_update_opts(ctl_options_t *opts, int num_args, struct ctl_be_arg *args)
-{
-	struct ctl_option *opt;
-	int i;
-
-	for (i = 0; i < num_args; i++) {
-		if ((args[i].flags & CTL_BEARG_RD) == 0)
-			continue;
-		if ((args[i].flags & CTL_BEARG_ASCII) == 0)
-			continue;
-		STAILQ_FOREACH(opt, opts, links) {
-			if (strcmp(opt->name, args[i].kname) == 0)
-				break;
-		}
-		if (args[i].kvalue != NULL &&
-		    ((char *)args[i].kvalue)[0] != 0) {
-			if (opt) {
-				free(opt->value, M_CTL);
-				opt->value = strdup(args[i].kvalue, M_CTL);
-			} else {
-				opt = malloc(sizeof(*opt), M_CTL, M_WAITOK);
-				opt->name = strdup(args[i].kname, M_CTL);
-				opt->value = strdup(args[i].kvalue, M_CTL);
-				STAILQ_INSERT_TAIL(opts, opt, links);
-			}
-		} else if (opt) {
-			STAILQ_REMOVE(opts, opt, ctl_option, links);
-			free(opt->name, M_CTL);
-			free(opt->value, M_CTL);
-			free(opt, M_CTL);
-		}
-	}
-}
-
-void
-ctl_free_opts(ctl_options_t *opts)
-{
-	struct ctl_option *opt;
-
-	while ((opt = STAILQ_FIRST(opts)) != NULL) {
-		STAILQ_REMOVE_HEAD(opts, links);
-		free(opt->name, M_CTL);
-		free(opt->value, M_CTL);
-		free(opt, M_CTL);
-	}
-}
-
-char *
-ctl_get_opt(ctl_options_t *opts, const char *name)
-{
-	struct ctl_option *opt;
-
-	STAILQ_FOREACH(opt, opts, links) {
-		if (strcmp(opt->name, name) == 0) {
-			return (opt->value);
-		}
-	}
-	return (NULL);
-}
-
-int
-ctl_get_opt_number(ctl_options_t *opts, const char *name, uint64_t *val)
-{
-	const char *value;
-
-	value = ctl_get_opt(opts, name);
-	if (value == NULL)
-		return (-2);
-	return (ctl_expand_number(value, val));
-}
diff --git a/sys/cam/ctl/ctl_backend.h b/sys/cam/ctl/ctl_backend.h
index a330497a604f..587bb70a8b81 100644
--- a/sys/cam/ctl/ctl_backend.h
+++ b/sys/cam/ctl/ctl_backend.h
@@ -43,6 +43,7 @@
 #define	_CTL_BACKEND_H_
 
 #include <cam/ctl/ctl_ioctl.h>
+#include <sys/nv.h>
 
 typedef enum {
 	CTL_LUN_SERSEQ_OFF,
@@ -175,7 +176,7 @@ struct ctl_be_lun {
 	be_lun_config_t		lun_config_status; /* passed to CTL */
 	struct ctl_backend_driver *be;		/* passed to CTL */
 	void			*ctl_lun;	/* used by CTL */
-	ctl_options_t		options;	/* passed to CTL */
+	nvlist_t	 	*options;	/* passed to CTL */
 	STAILQ_ENTRY(ctl_be_lun) links;		/* used by CTL */
 };
 
diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c
index b415424d2a07..a0cb085e0ca2 100644
--- a/sys/cam/ctl/ctl_backend_block.c
+++ b/sys/cam/ctl/ctl_backend_block.c
@@ -78,6 +78,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/sdt.h>
 #include <sys/devicestat.h>
 #include <sys/sysctl.h>
+#include <sys/nv.h>
+#include <sys/dnv.h>
 
 #include <geom/geom.h>
 
@@ -1817,7 +1819,7 @@ ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	struct ctl_be_lun *cbe_lun;
 	struct ctl_be_block_filedata *file_data;
 	struct ctl_lun_create_params *params;
-	char			     *value;
+	const char		     *value;
 	struct vattr		      vattr;
 	off_t			      ps, pss, po, pos, us, uss, uo, uos;
 	int			      error;
@@ -1867,10 +1869,10 @@ ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	us = ps = vattr.va_blocksize;
 	uo = po = 0;
 
-	value = ctl_get_opt(&cbe_lun->options, "pblocksize");
+	value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &ps);
-	value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
+	value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &po);
 	pss = ps / cbe_lun->blocksize;
@@ -1881,10 +1883,10 @@ ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 		cbe_lun->pblockoff = (pss - pos) % pss;
 	}
 
-	value = ctl_get_opt(&cbe_lun->options, "ublocksize");
+	value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &us);
-	value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
+	value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &uo);
 	uss = us / cbe_lun->blocksize;
@@ -1917,7 +1919,7 @@ ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	struct ctl_lun_create_params *params;
 	struct cdevsw		     *csw;
 	struct cdev		     *dev;
-	char			     *value;
+	const char		     *value;
 	int			      error, atomic, maxio, ref, unmap, tmp;
 	off_t			      ps, pss, po, pos, us, uss, uo, uos, otmp;
 
@@ -2033,10 +2035,10 @@ ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	us = ps;
 	uo = po;
 
-	value = ctl_get_opt(&cbe_lun->options, "pblocksize");
+	value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &ps);
-	value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
+	value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &po);
 	pss = ps / cbe_lun->blocksize;
@@ -2047,10 +2049,10 @@ ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 		cbe_lun->pblockoff = (pss - pos) % pss;
 	}
 
-	value = ctl_get_opt(&cbe_lun->options, "ublocksize");
+	value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &us);
-	value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
+	value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL);
 	if (value != NULL)
 		ctl_expand_number(value, &uo);
 	uss = us / cbe_lun->blocksize;
@@ -2075,7 +2077,7 @@ ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 		    curthread);
 		unmap = (error == 0) ? arg.value.i : 0;
 	}
-	value = ctl_get_opt(&cbe_lun->options, "unmap");
+	value = dnvlist_get_string(cbe_lun->options, "unmap", NULL);
 	if (value != NULL)
 		unmap = (strcmp(value, "on") == 0);
 	if (unmap)
@@ -2125,7 +2127,7 @@ ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 {
 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
 	struct nameidata nd;
-	char		*value;
+	const char	*value;
 	int		 error, flags;
 
 	error = 0;
@@ -2136,7 +2138,7 @@ ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	}
 	pwd_ensure_dirs();
 
-	value = ctl_get_opt(&cbe_lun->options, "file");
+	value = dnvlist_get_string(cbe_lun->options, "file", NULL);
 	if (value == NULL) {
 		snprintf(req->error_str, sizeof(req->error_str),
 			 "no file argument specified");
@@ -2146,7 +2148,7 @@ ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	be_lun->dev_path = strdup(value, M_CTLBLK);
 
 	flags = FREAD;
-	value = ctl_get_opt(&cbe_lun->options, "readonly");
+	value = dnvlist_get_string(cbe_lun->options, "readonly", NULL);
 	if (value != NULL) {
 		if (strcmp(value, "on") != 0)
 			flags |= FWRITE;
@@ -2205,7 +2207,7 @@ again:
 	cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
 	if (be_lun->dispatch != ctl_be_block_dispatch_dev)
 		cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
-	value = ctl_get_opt(&cbe_lun->options, "serseq");
+	value = dnvlist_get_string(cbe_lun->options, "serseq", NULL);
 	if (value != NULL && strcmp(value, "on") == 0)
 		cbe_lun->serseq = CTL_LUN_SERSEQ_ON;
 	else if (value != NULL && strcmp(value, "read") == 0)
@@ -2223,7 +2225,7 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	struct ctl_lun_create_params *params;
 	char num_thread_str[16];
 	char tmpstr[32];
-	char *value;
+	const char *value;
 	int retval, num_threads;
 	int tmp_num_threads;
 
@@ -2243,8 +2245,7 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	sprintf(be_lun->lunname, "cblk%d", softc->num_luns);
 	mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF);
 	mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF);
-	ctl_init_opts(&cbe_lun->options,
-	    req->num_be_args, req->kern_be_args);
+	cbe_lun->options = nvlist_clone(req->args_nvl);
 	be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG,
 	    NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
 	if (be_lun->lun_zone == NULL) {
@@ -2259,7 +2260,7 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 		cbe_lun->lun_type = T_DIRECT;
 	be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED;
 	cbe_lun->flags = 0;
-	value = ctl_get_opt(&cbe_lun->options, "ha_role");
+	value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL);
 	if (value != NULL) {
 		if (strcmp(value, "primary") == 0)
 			cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
@@ -2292,7 +2293,7 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 		num_threads = 1;
 	}
 
-	value = ctl_get_opt(&cbe_lun->options, "num_threads");
+	value = dnvlist_get_string(cbe_lun->options, "num_threads", NULL);
 	if (value != NULL) {
 		tmp_num_threads = strtol(value, NULL, 0);
 
@@ -2457,7 +2458,7 @@ bailout_error:
 		free(be_lun->dev_path, M_CTLBLK);
 	if (be_lun->lun_zone != NULL)
 		uma_zdestroy(be_lun->lun_zone);
-	ctl_free_opts(&cbe_lun->options);
+	nvlist_destroy(cbe_lun->options);
 	mtx_destroy(&be_lun->queue_lock);
 	mtx_destroy(&be_lun->io_lock);
 	free(be_lun, M_CTLBLK);
@@ -2541,7 +2542,7 @@ ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 
 	uma_zdestroy(be_lun->lun_zone);
 
-	ctl_free_opts(&cbe_lun->options);
+	nvlist_destroy(cbe_lun->options);
 	free(be_lun->dev_path, M_CTLBLK);
 	mtx_destroy(&be_lun->queue_lock);
 	mtx_destroy(&be_lun->io_lock);
@@ -2561,7 +2562,7 @@ ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	struct ctl_lun_modify_params *params;
 	struct ctl_be_block_lun *be_lun;
 	struct ctl_be_lun *cbe_lun;
-	char *value;
+	const char *value;
 	uint64_t oldsize;
 	int error, wasprim;
 
@@ -2583,10 +2584,12 @@ ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 
 	if (params->lun_size_bytes != 0)
 		be_lun->params.lun_size_bytes = params->lun_size_bytes;
-	ctl_update_opts(&cbe_lun->options, req->num_be_args, req->kern_be_args);
+
+	nvlist_destroy(cbe_lun->options);
+	cbe_lun->options = nvlist_clone(req->args_nvl);
 
 	wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY);
-	value = ctl_get_opt(&cbe_lun->options, "ha_role");
+	value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL);
 	if (value != NULL) {
 		if (strcmp(value, "primary") == 0)
 			cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c
index 9a29a0ff9315..e7ac5ec53f69 100644
--- a/sys/cam/ctl/ctl_backend_ramdisk.c
+++ b/sys/cam/ctl/ctl_backend_ramdisk.c
@@ -62,6 +62,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/ioccom.h>
 #include <sys/module.h>
 #include <sys/sysctl.h>
+#include <sys/nv.h>
+#include <sys/dnv.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_da.h>
@@ -956,7 +958,7 @@ ctl_backend_ramdisk_rm(struct ctl_be_ramdisk_softc *softc,
 	if (retval == 0) {
 		taskqueue_drain_all(be_lun->io_taskqueue);
 		taskqueue_free(be_lun->io_taskqueue);
-		ctl_free_opts(&be_lun->cbe_lun.options);
+		nvlist_destroy(be_lun->cbe_lun.options);
 		free(be_lun->zero_page, M_RAMDISK);
 		ctl_backend_ramdisk_freeallpages(be_lun->pages, be_lun->indir);
 		sx_destroy(&be_lun->page_lock);
@@ -979,7 +981,7 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 	struct ctl_be_ramdisk_lun *be_lun;
 	struct ctl_be_lun *cbe_lun;
 	struct ctl_lun_create_params *params;
-	char *value;
+	const char *value;
 	char tmpstr[32];
 	uint64_t t;
 	int retval;
@@ -990,10 +992,10 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 	be_lun = malloc(sizeof(*be_lun), M_RAMDISK, M_ZERO | M_WAITOK);
 	cbe_lun = &be_lun->cbe_lun;
 	cbe_lun->be_lun = be_lun;
+	cbe_lun->options = nvlist_clone(req->args_nvl);
 	be_lun->params = req->reqdata.create;
 	be_lun->softc = softc;
 	sprintf(be_lun->lunname, "cram%d", softc->num_luns);
-	ctl_init_opts(&cbe_lun->options, req->num_be_args, req->kern_be_args);
 
 	if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
 		cbe_lun->lun_type = params->device_type;
@@ -1001,7 +1003,7 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 		cbe_lun->lun_type = T_DIRECT;
 	be_lun->flags = CTL_BE_RAMDISK_LUN_UNCONFIGURED;
 	cbe_lun->flags = 0;
-	value = ctl_get_opt(&cbe_lun->options, "ha_role");
+	value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL);
 	if (value != NULL) {
 		if (strcmp(value, "primary") == 0)
 			cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
@@ -1009,7 +1011,7 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 		cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
 
 	be_lun->pblocksize = PAGE_SIZE;
-	value = ctl_get_opt(&cbe_lun->options, "pblocksize");
+	value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL);
 	if (value != NULL) {
 		ctl_expand_number(value, &t);
 		be_lun->pblocksize = t;
@@ -1058,7 +1060,7 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 		cbe_lun->ublockoff = 0;
 		cbe_lun->atomicblock = be_lun->pblocksize;
 		cbe_lun->opttxferlen = SGPP * be_lun->pblocksize;
-		value = ctl_get_opt(&cbe_lun->options, "capacity");
+		value = dnvlist_get_string(cbe_lun->options, "capacity", NULL);
 		if (value != NULL)
 			ctl_expand_number(value, &be_lun->cap_bytes);
 	} else {
@@ -1070,17 +1072,17 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 	params->blocksize_bytes = cbe_lun->blocksize;
 	params->lun_size_bytes = be_lun->size_bytes;
 
-	value = ctl_get_opt(&cbe_lun->options, "unmap");
-	if (value == NULL || strcmp(value, "off") != 0)
+	value = dnvlist_get_string(cbe_lun->options, "unmap", NULL);
+	if (value != NULL && strcmp(value, "off") != 0)
 		cbe_lun->flags |= CTL_LUN_FLAG_UNMAP;
-	value = ctl_get_opt(&cbe_lun->options, "readonly");
+	value = dnvlist_get_string(cbe_lun->options, "readonly", NULL);
 	if (value != NULL) {
 		if (strcmp(value, "on") == 0)
 			cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
 	} else if (cbe_lun->lun_type != T_DIRECT)
 		cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
 	cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
-	value = ctl_get_opt(&cbe_lun->options, "serseq");
+	value = dnvlist_get_string(cbe_lun->options, "serseq", NULL);
 	if (value != NULL && strcmp(value, "on") == 0)
 		cbe_lun->serseq = CTL_LUN_SERSEQ_ON;
 	else if (value != NULL && strcmp(value, "read") == 0)
@@ -1209,7 +1211,7 @@ bailout_error:
 	if (be_lun != NULL) {
 		if (be_lun->io_taskqueue != NULL)
 			taskqueue_free(be_lun->io_taskqueue);
-		ctl_free_opts(&cbe_lun->options);
+		nvlist_destroy(cbe_lun->options);
 		free(be_lun->zero_page, M_RAMDISK);
 		ctl_backend_ramdisk_freeallpages(be_lun->pages, be_lun->indir);
 		sx_destroy(&be_lun->page_lock);
@@ -1226,7 +1228,7 @@ ctl_backend_ramdisk_modify(struct ctl_be_ramdisk_softc *softc,
 	struct ctl_be_ramdisk_lun *be_lun;
 	struct ctl_be_lun *cbe_lun;
 	struct ctl_lun_modify_params *params;
-	char *value;
+	const char *value;
 	uint32_t blocksize;
 	int wasprim;
 
@@ -1248,10 +1250,12 @@ ctl_backend_ramdisk_modify(struct ctl_be_ramdisk_softc *softc,
 
 	if (params->lun_size_bytes != 0)
 		be_lun->params.lun_size_bytes = params->lun_size_bytes;
-	ctl_update_opts(&cbe_lun->options, req->num_be_args, req->kern_be_args);
+
+	nvlist_destroy(cbe_lun->options);
+	cbe_lun->options = nvlist_clone(req->args_nvl);
 
 	wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY);
-	value = ctl_get_opt(&cbe_lun->options, "ha_role");
+	value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL);
 	if (value != NULL) {
 		if (strcmp(value, "primary") == 0)
 			cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
diff --git a/sys/cam/ctl/ctl_frontend.c b/sys/cam/ctl/ctl_frontend.c
index 9a206d0c1423..d481c1b8f003 100644
--- a/sys/cam/ctl/ctl_frontend.c
+++ b/sys/cam/ctl/ctl_frontend.c
@@ -52,6 +52,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/endian.h>
 #include <sys/queue.h>
 #include <sys/sysctl.h>
+#include <sys/nv.h>
+#include <sys/dnv.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_da.h>
@@ -200,8 +202,8 @@ error:
 	}
 	port->targ_port = port_num;
 	port->ctl_pool_ref = pool;
-	if (port->options.stqh_first == NULL)
-		STAILQ_INIT(&port->options);
+	if (port->options == NULL)
+		port->options = nvlist_create(0);
 	port->stats.item = port_num;
 	mtx_init(&port->port_lock, "CTL port", NULL, MTX_DEF);
 
@@ -240,7 +242,7 @@ ctl_port_deregister(struct ctl_port *port)
 	mtx_unlock(&softc->ctl_lock);
 
 	ctl_pool_free(pool);
-	ctl_free_opts(&port->options);
+	nvlist_destroy(port->options);
 
 	ctl_lun_map_deinit(port);
 	free(port->port_devid, M_CTL);
@@ -333,7 +335,7 @@ ctl_port_online(struct ctl_port *port)
 		port->port_online(port->onoff_arg);
 	mtx_lock(&softc->ctl_lock);
 	if (softc->is_single == 0) {
-		value = ctl_get_opt(&port->options, "ha_shared");
+		value = dnvlist_get_string(port->options, "ha_shared", NULL);
 		if (value != NULL && strcmp(value, "on") == 0)
 			port->status |= CTL_PORT_STATUS_HA_SHARED;
 		else
diff --git a/sys/cam/ctl/ctl_frontend.h b/sys/cam/ctl/ctl_frontend.h
index 22980d555d11..bdcb7a2e1abd 100644
--- a/sys/cam/ctl/ctl_frontend.h
+++ b/sys/cam/ctl/ctl_frontend.h
@@ -43,6 +43,7 @@
 #define	_CTL_FRONTEND_H_
 
 #include <cam/ctl/ctl_ioctl.h>
+#include <sys/nv.h>
 
 typedef enum {
 	CTL_PORT_STATUS_NONE		= 0x00,
@@ -237,7 +238,7 @@ struct ctl_port {
 	uint64_t	wwnn;			/* set by CTL before online */
 	uint64_t	wwpn;			/* set by CTL before online */
 	ctl_port_status	status;			/* used by CTL */
-	ctl_options_t	options;		/* passed to CTL */
+	nvlist_t	*options;		/* passed to CTL */
 	struct ctl_devid *port_devid;		/* passed to CTL */
 	struct ctl_devid *target_devid;		/* passed to CTL */
 	struct ctl_devid *init_devid;		/* passed to CTL */
diff --git a/sys/cam/ctl/ctl_frontend_ioctl.c b/sys/cam/ctl/ctl_frontend_ioctl.c
index 63a9504016ef..221d912714b7 100644
--- a/sys/cam/ctl/ctl_frontend_ioctl.c
+++ b/sys/cam/ctl/ctl_frontend_ioctl.c
@@ -1,7 +1,10 @@
 /*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
  * Copyright (c) 2012 The FreeBSD Foundation
  * Copyright (c) 2015 Alexander Motin <mav@FreeBSD.org>
+ * Copyright (c) 2017 Jakub Wojciech Klama <jceel@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,6 +44,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/queue.h>
 #include <sys/sysctl.h>
+#include <sys/nv.h>
+#include <sys/dnv.h>
 
 #include <cam/cam.h>
 #include <cam/scsi/scsi_all.h>
@@ -68,22 +73,41 @@ struct ctl_fe_ioctl_params {
 	ctl_fe_ioctl_state	state;
 };
 
-struct cfi_softc {
+struct cfi_port {
+	TAILQ_ENTRY(cfi_port)	link;
 	uint32_t		cur_tag_num;
+	struct cdev *		dev;
 	struct ctl_port		port;
 };
 
+struct cfi_softc {
+	TAILQ_HEAD(, cfi_port)	ports;
+};
+
+
 static struct cfi_softc cfi_softc;
 
+
 static int cfi_init(void);
 static int cfi_shutdown(void);
 static void cfi_datamove(union ctl_io *io);
 static void cfi_done(union ctl_io *io);
+static int cfi_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
+    struct thread *td);
+static void cfi_ioctl_port_create(struct ctl_req *req);
+static void cfi_ioctl_port_remove(struct ctl_req *req);
+
+static struct cdevsw cfi_cdevsw = {
+	.d_version = D_VERSION,
+	.d_flags = 0,
+	.d_ioctl = ctl_ioctl_io
+};
 
 static struct ctl_frontend cfi_frontend =
 {
 	.name = "ioctl",
 	.init = cfi_init,
+	.ioctl = cfi_ioctl,
 	.shutdown = cfi_shutdown,
 };
 CTL_FRONTEND_DECLARE(ctlioctl, cfi_frontend);
@@ -92,26 +116,31 @@ static int
 cfi_init(void)
 {
 	struct cfi_softc *isoftc = &cfi_softc;
+	struct cfi_port *cfi;
 	struct ctl_port *port;
 	int error = 0;
 
 	memset(isoftc, 0, sizeof(*isoftc));
+	TAILQ_INIT(&isoftc->ports);
 
-	port = &isoftc->port;
+	cfi = malloc(sizeof(*cfi), M_CTL, M_WAITOK | M_ZERO);
+	port = &cfi->port;
 	port->frontend = &cfi_frontend;
 	port->port_type = CTL_PORT_IOCTL;
 	port->num_requested_ctl_io = 100;
 	port->port_name = "ioctl";
 	port->fe_datamove = cfi_datamove;
 	port->fe_done = cfi_done;
+	port->physical_port = 0;
 	port->targ_port = -1;
-	port->max_initiators = 1;
 
 	if ((error = ctl_port_register(port)) != 0) {
 		printf("%s: ioctl port registration failed\n", __func__);
 		return (error);
 	}
+
 	ctl_port_online(port);
+	TAILQ_INSERT_TAIL(&isoftc->ports, cfi, link);
 	return (0);
 }
 
@@ -119,13 +148,185 @@ static int
 cfi_shutdown(void)
 {
 	struct cfi_softc *isoftc = &cfi_softc;
-	struct ctl_port *port = &isoftc->port;
-	int error = 0;
+	struct cfi_port *cfi, *temp;
+	struct ctl_port *port;
+	int error;
+
+	TAILQ_FOREACH_SAFE(cfi, &isoftc->ports, link, temp) {
+		port = &cfi->port;
+		ctl_port_offline(port);
+		error = ctl_port_deregister(port);
+		if (error != 0) {
+			printf("%s: ctl_frontend_deregister() failed\n",
+			   __func__);
+			return (error);
+		}
 
-	ctl_port_offline(port);
-	if ((error = ctl_port_deregister(port)) != 0)
-		printf("%s: ioctl port deregistration failed\n", __func__);
-	return (error);
+		TAILQ_REMOVE(&isoftc->ports, cfi, link);
+		free(cfi, M_CTL);
+	}
+
+	return (0);
+}
+
+static void
+cfi_ioctl_port_create(struct ctl_req *req)
+{
+	struct cfi_softc *isoftc = &cfi_softc;
+	struct cfi_port *cfi;
+	struct ctl_port *port;
+	struct make_dev_args args;
+	const char *val;
+	int retval;
+	int pp = -1, vp = 0;
+
+	val = dnvlist_get_string(req->args_nvl, "pp", NULL);
+	if (val != NULL)
+		pp = strtol(val, NULL, 10);
+	
+	val = dnvlist_get_string(req->args_nvl, "vp", NULL);
+	if (val != NULL)
+		vp = strtol(val, NULL, 10);
+
+	if (pp != -1) {
+		/* Check for duplicates */
+		TAILQ_FOREACH(cfi, &isoftc->ports, link) {
+			if (pp == cfi->port.physical_port && 
+			    vp == cfi->port.virtual_port) {
+				req->status = CTL_LUN_ERROR;
+				snprintf(req->error_str, sizeof(req->error_str),
+				    "port %d already exists", pp);
+
+				return;
+			}
+		}
+	} else {
+		/* Find free port number */
+		TAILQ_FOREACH(cfi, &isoftc->ports, link) {
+			pp = MAX(pp, cfi->port.physical_port);
+		}
+
+		pp++;
+	}
+
+	cfi = malloc(sizeof(*cfi), M_CTL, M_WAITOK | M_ZERO);
+	port = &cfi->port;
+	port->frontend = &cfi_frontend;
+	port->port_type = CTL_PORT_IOCTL;
+	port->num_requested_ctl_io = 100;
+	port->port_name = "ioctl";
+	port->fe_datamove = cfi_datamove;
+	port->fe_done = cfi_done;
+	port->physical_port = pp;
+	port->virtual_port = vp;
+	port->targ_port = -1;
+
+	retval = ctl_port_register(port);
+	if (retval != 0) {
+		req->status = CTL_LUN_ERROR;
+		snprintf(req->error_str, sizeof(req->error_str),
+		    "ctl_port_register() failed with error %d", retval);
+		free(port, M_CTL);
+		return;
+	}
+
+	req->result_nvl = nvlist_create(0);
+	nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
+	ctl_port_online(port);
+
+	make_dev_args_init(&args);
+	args.mda_devsw = &cfi_cdevsw;
+	args.mda_uid = UID_ROOT;
+	args.mda_gid = GID_OPERATOR;
+	args.mda_mode = 0600;
+	args.mda_si_drv1 = NULL;
+	args.mda_si_drv2 = cfi;
+
+	retval = make_dev_s(&args, &cfi->dev, "cam/ctl%d.%d", pp, vp);
+	if (retval != 0) {
+		req->status = CTL_LUN_ERROR;
+		snprintf(req->error_str, sizeof(req->error_str),
+		    "make_dev_s() failed with error %d", retval);
+		free(port, M_CTL);
+		return;
+	}
+
+	req->status = CTL_LUN_OK;
+	TAILQ_INSERT_TAIL(&isoftc->ports, cfi, link);
+}
+
+static void
+cfi_ioctl_port_remove(struct ctl_req *req)
+{
+	struct cfi_softc *isoftc = &cfi_softc;
+	struct cfi_port *cfi = NULL;
+	const char *val;
+	int port_id = -1;
+
+	val = dnvlist_get_string(req->args_nvl, "port_id", NULL);
+	if (val != NULL)
+		port_id = strtol(val, NULL, 10);
+
+	if (port_id == -1) {
+		req->status = CTL_LUN_ERROR;
+		snprintf(req->error_str, sizeof(req->error_str),
+		    "port_id not provided");
+		return;
+	}
+
+	TAILQ_FOREACH(cfi, &isoftc->ports, link) {
+		if (cfi->port.targ_port == port_id)
+			break;
+	}
+
+	if (cfi == NULL) {
+		req->status = CTL_LUN_ERROR;
+		snprintf(req->error_str, sizeof(req->error_str),
+		    "cannot find port %d", port_id);
+
+		return;
+	}
+
+	if (cfi->port.physical_port == 0 && cfi->port.virtual_port == 0) {
+		req->status = CTL_LUN_ERROR;
+		snprintf(req->error_str, sizeof(req->error_str),
+		    "cannot destroy default ioctl port");
+
+		return;
+	}
+
+	ctl_port_offline(&cfi->port);
+	ctl_port_deregister(&cfi->port);
+	TAILQ_REMOVE(&isoftc->ports, cfi, link);
+	destroy_dev(cfi->dev);
+	free(cfi, M_CTL);
+	req->status = CTL_LUN_OK;
+}
+
+static int
+cfi_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
+    struct thread *td)
+{
+	struct ctl_req *req;
+
+	if (cmd == CTL_PORT_REQ) {
+		req = (struct ctl_req *)addr;
+		switch (req->reqtype) {
+		case CTL_REQ_CREATE:
+			cfi_ioctl_port_create(req);
+			break;
+		case CTL_REQ_REMOVE:
+			cfi_ioctl_port_remove(req);
+			break;
+		default:
+			req->status = CTL_LUN_ERROR;
+			snprintf(req->error_str, sizeof(req->error_str),
+			    "Unsupported request type %d", req->reqtype);
+		}
+		return (0);
+	}
+
+	return (ENOTTY);
 }
 
 /*
@@ -389,18 +590,26 @@ int
 ctl_ioctl_io(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
     struct thread *td)
 {
+	struct cfi_port *cfi;
 	union ctl_io *io;
 	void *pool_tmp, *sc_tmp;
 	int retval = 0;
 
+	if (cmd != CTL_IO)
+		return (ENOTTY);
+
+	cfi = dev->si_drv2 == NULL
+	    ? TAILQ_FIRST(&cfi_softc.ports)
+	    : dev->si_drv2;
+
 	/*
 	 * If we haven't been "enabled", don't allow any SCSI I/O
 	 * to this FETD.
 	 */
-	if ((cfi_softc.port.status & CTL_PORT_STATUS_ONLINE) == 0)
+	if ((cfi->port.status & CTL_PORT_STATUS_ONLINE) == 0)
 		return (EPERM);
 
-	io = ctl_alloc_io(cfi_softc.port.ctl_pool_ref);
+	io = ctl_alloc_io(cfi->port.ctl_pool_ref);
 
 	/*
 	 * Need to save the pool reference so it doesn't get
@@ -420,15 +629,16 @@ ctl_ioctl_io(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	/*
 	 * The user sets the initiator ID, target and LUN IDs.
 	 */
-	io->io_hdr.nexus.targ_port = cfi_softc.port.targ_port;
+	io->io_hdr.nexus.targ_port = cfi->port.targ_port;
 	io->io_hdr.flags |= CTL_FLAG_USER_REQ;
 	if ((io->io_hdr.io_type == CTL_IO_SCSI) &&
 	    (io->scsiio.tag_type != CTL_TAG_UNTAGGED))
-		io->scsiio.tag_num = cfi_softc.cur_tag_num++;
+		io->scsiio.tag_num = cfi->cur_tag_num++;
 
 	retval = cfi_submit_wait(io);
 	if (retval == 0)
 		memcpy((void *)addr, io, sizeof(*io));
+
 	ctl_free_io(io);
 	return (retval);
 }
diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c
index a46aa002c136..22a5b3e1fb9d 100644
--- a/sys/cam/ctl/ctl_frontend_iscsi.c
+++ b/sys/cam/ctl/ctl_frontend_iscsi.c
@@ -56,6 +56,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/uio.h>
 #include <sys/unistd.h>
+#include <sys/nv.h>
+#include <sys/dnv.h>
 #include <vm/uma.h>
 
 #include <cam/scsi/scsi_all.h>
@@ -2105,30 +2107,30 @@ cfiscsi_ioctl_port_create(struct ctl_req *req)
 {
 	struct cfiscsi_target *ct;
 	struct ctl_port *port;
-	const char *target, *alias, *tags;
+	const char *target, *alias, *val;
 	struct scsi_vpd_id_descriptor *desc;
-	ctl_options_t opts;
 	int retval, len, idlen;
 	uint16_t tag;
 
-	ctl_init_opts(&opts, req->num_args, req->kern_args);
-	target = ctl_get_opt(&opts, "cfiscsi_target");
-	alias = ctl_get_opt(&opts, "cfiscsi_target_alias");
-	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
-	if (target == NULL || tags == NULL) {
+	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
+	alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);
+	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
+	    NULL);
+
+
+	if (target == NULL || val == NULL) {
 		req->status = CTL_LUN_ERROR;
 		snprintf(req->error_str, sizeof(req->error_str),
 		    "Missing required argument");
-		ctl_free_opts(&opts);
 		return;
 	}
-	tag = strtol(tags, (char **)NULL, 10);
+
+	tag = strtoul(val, NULL, 0);
 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
 	if (ct == NULL) {
 		req->status = CTL_LUN_ERROR;
 		snprintf(req->error_str, sizeof(req->error_str),
 		    "failed to create target \"%s\"", target);
-		ctl_free_opts(&opts);
 		return;
 	}
 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
@@ -2137,7 +2139,6 @@ cfiscsi_ioctl_port_create(struct ctl_req *req)
 		    "target \"%s\" for portal group tag %u already exists",
 		    target, tag);
 		cfiscsi_target_release(ct);
-		ctl_free_opts(&opts);
 		return;
 	}
 	port = &ct->ct_port;
@@ -2150,7 +2151,7 @@ cfiscsi_ioctl_port_create(struct ctl_req *req)
 	/* XXX KDM what should the real number be here? */
 	port->num_requested_ctl_io = 4096;
 	port->port_name = "iscsi";
-	port->physical_port = tag;
+	port->physical_port = (int)tag;
 	port->virtual_port = ct->ct_target_id;
 	port->port_online = cfiscsi_online;
 	port->port_offline = cfiscsi_offline;
@@ -2159,9 +2160,7 @@ cfiscsi_ioctl_port_create(struct ctl_req *req)
 	port->fe_datamove = cfiscsi_datamove;
 	port->fe_done = cfiscsi_done;
 	port->targ_port = -1;
-
-	port->options = opts;
-	STAILQ_INIT(&opts);
+	port->options = nvlist_clone(req->args_nvl);
 
 	/* Generate Port ID. */
 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
@@ -2193,7 +2192,6 @@ cfiscsi_ioctl_port_create(struct ctl_req *req)
 
 	retval = ctl_port_register(port);
 	if (retval != 0) {
-		ctl_free_opts(&port->options);
 		free(port->port_devid, M_CFISCSI);
 		free(port->target_devid, M_CFISCSI);
 		cfiscsi_target_release(ct);
@@ -2205,45 +2203,42 @@ cfiscsi_ioctl_port_create(struct ctl_req *req)
 done:
 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
 	req->status = CTL_LUN_OK;
-	memcpy(req->kern_args[0].kvalue, &port->targ_port,
-	    sizeof(port->targ_port)); //XXX
+	req->result_nvl = nvlist_create(0);
+	nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
 }
 
 static void
 cfiscsi_ioctl_port_remove(struct ctl_req *req)
 {
 	struct cfiscsi_target *ct;
-	const char *target, *tags;
-	ctl_options_t opts;
+	const char *target, *val;
 	uint16_t tag;
 
-	ctl_init_opts(&opts, req->num_args, req->kern_args);
-	target = ctl_get_opt(&opts, "cfiscsi_target");
-	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
-	if (target == NULL || tags == NULL) {
-		ctl_free_opts(&opts);
+	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
+	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
+	    NULL);
+
+	if (target == NULL || val == NULL) {
 		req->status = CTL_LUN_ERROR;
 		snprintf(req->error_str, sizeof(req->error_str),
 		    "Missing required argument");
 		return;
 	}
-	tag = strtol(tags, (char **)NULL, 10);
+
+	tag = strtoul(val, NULL, 0);
 	ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
 	if (ct == NULL) {
-		ctl_free_opts(&opts);
 		req->status = CTL_LUN_ERROR;
 		snprintf(req->error_str, sizeof(req->error_str),
 		    "can't find target \"%s\"", target);
 		return;
 	}
 	if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) {
-		ctl_free_opts(&opts);
 		req->status = CTL_LUN_ERROR;
 		snprintf(req->error_str, sizeof(req->error_str),
 		    "target \"%s\" is already dying", target);
 		return;
 	}
-	ctl_free_opts(&opts);
 
 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
 	ctl_port_offline(&ct->ct_port);
diff --git a/sys/cam/ctl/ctl_ioctl.h b/sys/cam/ctl/ctl_ioctl.h
index f7297504dcc1..0c3aa1e1095c 100644
--- a/sys/cam/ctl/ctl_ioctl.h
+++ b/sys/cam/ctl/ctl_ioctl.h
@@ -48,6 +48,7 @@
 #endif
 
 #include <sys/ioccom.h>
+#include <sys/nv.h>
 
 #define	CTL_DEFAULT_DEV		"/dev/cam/ctl"
 /*
@@ -316,39 +317,6 @@ typedef enum {
 
 #define	CTL_ERROR_STR_LEN	160
 
-#define	CTL_BEARG_RD		0x01
-#define	CTL_BEARG_WR		0x02
-#define	CTL_BEARG_RW		(CTL_BEARG_RD|CTL_BEARG_WR)
-#define	CTL_BEARG_ASCII		0x04
-
-/*
- * Backend Argument:
- *
- * namelen:	Length of the name field, including the terminating NUL.
- *
- * name:	Name of the parameter.  This must be NUL-terminated.
- *
- * flags:	Flags for the parameter, see above for values.
- *
- * vallen:	Length of the value in bytes, including the terminating NUL.
- *
- * value:	Value to be set/fetched. This must be NUL-terminated.
- *
- * kname:	For kernel use only.
- *
- * kvalue:	For kernel use only.
- */
-struct ctl_be_arg {
-	unsigned int	namelen;
-	char		*name;
-	int		flags;
-	unsigned int	vallen;
-	void		*value;
-
-	char		*kname;
-	void		*kvalue;
-};
-
 typedef enum {
 	CTL_LUNREQ_CREATE,
 	CTL_LUNREQ_RM,
@@ -524,11 +492,14 @@ struct ctl_lun_req {
 	char			backend[CTL_BE_NAME_LEN];
 	ctl_lunreq_type		reqtype;
 	union ctl_lunreq_data	reqdata;
-	int			num_be_args;
-	struct ctl_be_arg	*be_args;
+	void *			args;
+	nvlist_t *		args_nvl;
+	size_t			args_len;
+	void *			result;
+	nvlist_t *		result_nvl;
+	size_t			result_len;
 	ctl_lun_status		status;
 	char			error_str[CTL_ERROR_STR_LEN];
-	struct ctl_be_arg	*kern_be_args;
 };
 
 /*
@@ -617,11 +588,14 @@ typedef enum {
 struct ctl_req {
 	char			driver[CTL_DRIVER_NAME_LEN];
 	ctl_req_type		reqtype;
-	int			num_args;
-	struct ctl_be_arg	*args;
+	void *			args;
+	nvlist_t *		args_nvl;
+	size_t			args_len;
+	void *			result;
+	nvlist_t *		result_nvl;
+	size_t			result_len;
 	ctl_lun_status		status;
 	char			error_str[CTL_ERROR_STR_LEN];
-	struct ctl_be_arg	*kern_args;
 };
 
 /*
diff --git a/sys/cam/ctl/ctl_tpc.c b/sys/cam/ctl/ctl_tpc.c
index 572d93a90078..00fb3a52b3d4 100644
--- a/sys/cam/ctl/ctl_tpc.c
+++ b/sys/cam/ctl/ctl_tpc.c
@@ -41,6 +41,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/queue.h>
 #include <sys/sysctl.h>
+#include <sys/nv.h>
+#include <sys/dnv.h>
 #include <machine/atomic.h>
 
 #include <cam/cam.h>
@@ -1668,7 +1670,7 @@ ctl_extended_copy_lid1(struct ctl_scsiio *ctsio)
 	struct scsi_ec_segment *seg;
 	struct tpc_list *list, *tlist;
 	uint8_t *ptr;
-	char *value;
+	const char *value;
 	int len, off, lencscd, lenseg, leninl, nseg;
 
 	CTL_DEBUG_PRINT(("ctl_extended_copy_lid1\n"));
@@ -1731,7 +1733,7 @@ ctl_extended_copy_lid1(struct ctl_scsiio *ctsio)
 
 	list = malloc(sizeof(struct tpc_list), M_CTL, M_WAITOK | M_ZERO);
 	list->service_action = cdb->service_action;
-	value = ctl_get_opt(&lun->be_lun->options, "insecure_tpc");
+	value = dnvlist_get_string(lun->be_lun->options, "insecure_tpc", NULL);
 	if (value != NULL && strcmp(value, "on") == 0)
 		list->init_port = -1;
 	else
@@ -1822,7 +1824,7 @@ ctl_extended_copy_lid4(struct ctl_scsiio *ctsio)
 	struct scsi_ec_segment *seg;
 	struct tpc_list *list, *tlist;
 	uint8_t *ptr;
-	char *value;
+	const char *value;
 	int len, off, lencscd, lenseg, leninl, nseg;
 
 	CTL_DEBUG_PRINT(("ctl_extended_copy_lid4\n"));
@@ -1885,7 +1887,7 @@ ctl_extended_copy_lid4(struct ctl_scsiio *ctsio)
 
 	list = malloc(sizeof(struct tpc_list), M_CTL, M_WAITOK | M_ZERO);
 	list->service_action = cdb->service_action;
-	value = ctl_get_opt(&lun->be_lun->options, "insecure_tpc");
+	value = dnvlist_get_string(lun->be_lun->options, "insecure_tpc", NULL);
 	if (value != NULL && strcmp(value, "on") == 0)
 		list->init_port = -1;
 	else
diff --git a/sys/sys/param.h b/sys/sys/param.h
index cb226c4891f8..00b1de8edf21 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -60,7 +60,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1200062	/* Master, propagated to newvers */
+#define __FreeBSD_version 1200063	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
diff --git a/usr.sbin/ctladm/Makefile b/usr.sbin/ctladm/Makefile
index 5b74668bf571..9f7386bdc36c 100644
--- a/usr.sbin/ctladm/Makefile
+++ b/usr.sbin/ctladm/Makefile
@@ -16,7 +16,7 @@ CFLAGS+=	-I${SDIR}
 WARNS?= 3
 .endif
 
-LIBADD=		cam sbuf bsdxml util
+LIBADD=		cam sbuf bsdxml util nv
 MAN=		ctladm.8
 
 .if ${MK_ISCSI} != "no"
diff --git a/usr.sbin/ctladm/ctladm.8 b/usr.sbin/ctladm/ctladm.8
index e9dd3620d99f..d9b0c075eeed 100644
--- a/usr.sbin/ctladm/ctladm.8
+++ b/usr.sbin/ctladm/ctladm.8
@@ -1,6 +1,7 @@
 .\"
 .\" Copyright (c) 2003 Silicon Graphics International Corp.
 .\" Copyright (c) 2015 Alexander Motin <mav@FreeBSD.org>
+.\" Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -35,7 +36,7 @@
 .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.sbin/ctladm/ctladm.8#3 $
 .\" $FreeBSD$
 .\"
-.Dd July 15, 2017
+.Dd May 10, 2018
 .Dt CTLADM 8
 .Os
 .Sh NAME
@@ -161,10 +162,13 @@
 .Op Fl x
 .Nm
 .Ic port
+.Op Fl c
 .Op Fl o Ar on|off
 .Op Fl w Ar wwpn
 .Op Fl W Ar wwnn
+.Op Fl O Ar pp|vp
 .Op Fl p Ar targ_port
+.Op Fl r Ar targ_port
 .Op Fl t Ar fe_type
 .Nm
 .Ic portlist
@@ -593,13 +597,21 @@ must be specified.
 The WWNN and WWPN may both be specified at the same time, but cannot be
 combined with enabling/disabling or listing ports.
 .Bl -tag -width 12n
+.It Fl c
+Create new frontend port using free pp and vp=0.
 .It Fl o Ar on|off
-Turn the specified CTL frontend ports off or on.
+Turn the specified CTL frontend ports on or off.
 If no port number or port type is specified, all ports are turned on or
 off.
+.It Fl O Ar pp|vp
+Specify generic options on the ioctl frontend port.
+At present, only pp and vp port numbers can be set.
 .It Fl p Ar targ_port
 Specify the frontend port number.
 The port numbers can be found in the frontend port list.
+.It Fl r
+Remove port specified with
+.Pq Fl p Ar targ_port .
 .It Fl t Ar fe_type
 Specify the frontend type.
 Currently defined port types are
@@ -950,29 +962,26 @@ The default value is zero, that disables backing store completely,
 making all writes go to nowhere, while all reads return zeroes.
 .El
 .Sh EXAMPLES
-.Dl ctladm tur 1
 .Pp
 Send a
 .Tn SCSI
 TEST UNIT READY command to LUN 1.
 .Pp
-.Dl ctladm modesense 1 -l
+.Dl ctladm tur 1
 .Pp
 Display the list of mode pages supported by LUN 1.
 .Pp
-.Dl ctladm modesense 0 -m 10 -P 3 -d -c 10
+.Dl ctladm modesense 1 -l
 .Pp
 Display the saved version of the Control mode page (page 10) on LUN 0.
 Disable fetching block descriptors, and use a 10 byte MODE SENSE command
 instead of the default 6 byte command.
-.Bd -literal
-ctladm read 2 -l 0 -d 1 -b 512 -f - > foo
-.Ed
+.Pp
+.Dl ctladm modesense 0 -m 10 -P 3 -d -c 10
 .Pp
 Read the first 512 byte block from LUN 2 and dump it to the file
-.Pa foo .
 .Bd -literal
-ctladm write 3 -l 0xff432140 -d 20 -b 512 -f /tmp/bar
+.Dl ctladm read 2 -l 0 -d 1 -b 512 -f - > foo
 .Ed
 .Pp
 Read 10240 bytes from the file
@@ -980,7 +989,9 @@ Read 10240 bytes from the file
 and write it to LUN 3.
 starting at LBA 0xff432140.
 .Pp
-.Dl ctladm create -b ramdisk -s 10485760000000000
+.Bd -literal
+.Dl ctladm write 3 -l 0xff432140 -d 20 -b 512 -f /tmp/bar
+.Ed
 .Pp
 Create a LUN with the
 .Dq fake
@@ -988,20 +999,20 @@ ramdisk as a backing store.
 The LUN will claim to have a size of approximately 10 terabytes,
 while having no real data store (all written data are lost).
 .Pp
-.Dl ctladm create -b ramdisk -s 10T -o capacity=10G
+.Dl ctladm create -b ramdisk -s 10485760000000000
 .Pp
 Create a thin provisioned LUN with a ramdisk as a backing store.
 The LUN will have maximal backing store capacity of 10 gigabytes,
 while reporting size of 10 terabytes,
 .Pp
-.Dl ctladm create -b block -o file=src/usr.sbin/ctladm/ctladm.8
+.Dl ctladm create -b ramdisk -s 10T -o capacity=10G
 .Pp
 Create a LUN using the block backend, and specify the file
 .Pa src/usr.sbin/ctladm/ctladm.8
 as the backing store.
 The size of the LUN will be derived from the size of the file.
 .Pp
-.Dl ctladm create -b block -o file=src/usr.sbin/ctladm/ctladm.8 -S MYSERIAL321 -d MYDEVID123
+.Dl ctladm create -b block -o file=src/usr.sbin/ctladm/ctladm.8
 .Pp
 Create a LUN using the block backend, specify the file
 .Pa src/usr.sbin/ctladm/ctladm.8
@@ -1012,33 +1023,46 @@ VPD page 0x80 and 0x83 serial number
 and device ID
 .Fl ( d ) .
 .Pp
-.Dl ctladm remove -b block -l 12
+.Dl ctladm create -b block -o file=src/usr.sbin/ctladm/ctladm.8 -S MYSERIAL321 -d MYDEVID123
+.Pp
+Use to specify generic options on ioctl frontend port, now it is
+only possible to set pp and/or vp port number.
+.Pp
+.Dl ctladm port -c -O pp=11 -O vp=12
+.Pp
+Remove specified targ_port.
+.Pp
+.Dl ctladm port -r -p 4
+.Pp
 .Pp
 Remove LUN 12, which is handled by the block backend, from the system.
 .Pp
-.Dl ctladm devlist
+.Dl ctladm remove -b block -l 12
 .Pp
 List configured LUNs in the system, along with their backend and serial
 number.
 This works when the Front End Target Drivers are enabled or disabled.
 .Pp
-.Dl ctladm lunlist
+.Dl ctladm devlist
 .Pp
 List all LUNs in the system, along with their inquiry data and device type.
 This only works when the FETDs are enabled, since the commands go through the
 ioctl port.
 .Pp
-.Dl ctladm inject 6 -i mediumerr -p read -r 0,512 -c
+.Dl ctladm lunlist
 .Pp
 Inject a medium error on LUN 6 for every read that covers the first 512
 blocks of the LUN.
-.Bd -literal -offset indent
-ctladm inject 6 -i custom -p tur -s 18 "f0 0 02 s12 04 02"
-.Ed
+.Pp
+.Dl ctladm inject 6 -i mediumerr -p read -r 0,512 -c
 .Pp
 Inject a custom error on LUN 6 for the next TEST UNIT READY command only.
 This will result in a sense key of NOT READY (0x02), and an ASC/ASCQ of
 0x04,0x02 ("Logical unit not ready, initializing command required").
+.Pp
+.Bd -literal -offset indent
+ctladm inject 6 -i custom -p tur -s 18 "f0 0 02 s12 04 02"
+.Ed
 .Sh SEE ALSO
 .Xr cam 3 ,
 .Xr cam_cdbparse 3 ,
diff --git a/usr.sbin/ctladm/ctladm.c b/usr.sbin/ctladm/ctladm.c
index 8bad0452b4a8..2761b2c763b0 100644
--- a/usr.sbin/ctladm/ctladm.c
+++ b/usr.sbin/ctladm/ctladm.c
@@ -1,7 +1,10 @@
 /*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
  * Copyright (c) 2003, 2004 Silicon Graphics International Corp.
  * Copyright (c) 1997-2007 Kenneth D. Merry
  * Copyright (c) 2012 The FreeBSD Foundation
+ * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>
  * All rights reserved.
  *
  * Portions of this software were developed by Edward Tomasz Napierala
@@ -50,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/module.h>
 #include <sys/queue.h>
 #include <sys/sbuf.h>
+#include <sys/nv.h>
 #include <sys/stat.h>
 #include <bsdxml.h>
 #include <ctype.h>
@@ -178,7 +182,7 @@ static struct ctladm_opts option_table[] = {
 	{"lunmap", CTLADM_CMD_LUNMAP, CTLADM_ARG_NONE, "p:l:L:"},
 	{"modesense", CTLADM_CMD_MODESENSE, CTLADM_ARG_NEED_TL, "P:S:dlm:c:"},
 	{"modify", CTLADM_CMD_MODIFY, CTLADM_ARG_NONE, "b:l:o:s:"},
-	{"port", CTLADM_CMD_PORT, CTLADM_ARG_NONE, "lo:p:qt:w:W:x"},
+	{"port", CTLADM_CMD_PORT, CTLADM_ARG_NONE, "lo:O:d:crp:qt:w:W:x"},
 	{"portlist", CTLADM_CMD_PORTLIST, CTLADM_ARG_NONE, "f:ilp:qvx"},
 	{"prin", CTLADM_CMD_PRES_IN, CTLADM_ARG_NEED_TL, "a:"},
 	{"prout", CTLADM_CMD_PRES_OUT, CTLADM_ARG_NEED_TL, "a:k:r:s:"},
@@ -369,7 +373,9 @@ typedef enum {
 	CCTL_PORT_MODE_LIST,
 	CCTL_PORT_MODE_SET,
 	CCTL_PORT_MODE_ON,
-	CCTL_PORT_MODE_OFF
+	CCTL_PORT_MODE_OFF,
+	CCTL_PORT_MODE_CREATE,
+	CCTL_PORT_MODE_REMOVE
 } cctl_port_mode;
 
 static struct ctladm_opts cctl_fe_table[] = {
@@ -392,9 +398,16 @@ cctl_port(int fd, int argc, char **argv, char *combinedopt)
 	uint64_t wwnn = 0, wwpn = 0;
 	cctl_port_mode port_mode = CCTL_PORT_MODE_NONE;
 	struct ctl_port_entry entry;
+	struct ctl_req req;
+	char *driver = NULL;
+	nvlist_t *option_list;
 	ctl_port_type port_type = CTL_PORT_NONE;
 	int quiet = 0, xml = 0;
 
+	option_list = nvlist_create(0);
+	if (option_list == NULL)
+		err(1, "%s: unable to allocate nvlist", __func__);
+
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'l':
@@ -403,6 +416,12 @@ cctl_port(int fd, int argc, char **argv, char *combinedopt)
 
 			port_mode = CCTL_PORT_MODE_LIST;
 			break;
+		case 'c':
+			port_mode = CCTL_PORT_MODE_CREATE;
+			break;
+		case 'r':
+			port_mode = CCTL_PORT_MODE_REMOVE;
+			break;
 		case 'o':
 			if (port_mode != CCTL_PORT_MODE_NONE)
 				goto bailout_badarg;
@@ -419,6 +438,40 @@ cctl_port(int fd, int argc, char **argv, char *combinedopt)
 				goto bailout;
 			}
 			break;
+		case 'O': {
+			char *tmpstr;
+			char *name, *value;
+
+			tmpstr = strdup(optarg);
+			name = strsep(&tmpstr, "=");
+			if (name == NULL) {
+				warnx("%s: option -O takes \"name=value\""
+				      "argument", __func__);
+				retval = 1;
+				goto bailout;
+			}
+			value = strsep(&tmpstr, "=");
+			if (value == NULL) {
+				warnx("%s: option -O takes \"name=value\""
+				      "argument", __func__);
+				retval = 1;
+				goto bailout;
+			}
+
+			free(tmpstr);
+			nvlist_add_string(option_list, name, value);
+			break;
+		}
+		case 'd':
+			if (driver != NULL) {
+				warnx("%s: option -d cannot be specified twice",
+				    __func__);
+				retval = 1;
+				goto bailout;
+			}
+
+			driver = strdup(optarg);
+			break;
 		case 'p':
 			targ_port = strtol(optarg, NULL, 0);
 			break;
@@ -474,6 +527,9 @@ cctl_port(int fd, int argc, char **argv, char *combinedopt)
 		}
 	}
 
+	if (driver == NULL)
+		driver = strdup("ioctl");
+
 	/*
 	 * The user can specify either one or more frontend types (-t), or
 	 * a specific frontend, but not both.
@@ -515,6 +571,56 @@ cctl_port(int fd, int argc, char **argv, char *combinedopt)
 		cctl_portlist(fd, argcx, argvx, opts);
 		break;
 	}
+	case CCTL_PORT_MODE_REMOVE:
+		if (targ_port == -1) {
+			warnx("%s: -r require -p", __func__);
+			retval = 1;
+			goto bailout;
+		}
+	case CCTL_PORT_MODE_CREATE: {
+		bzero(&req, sizeof(req));
+		strlcpy(req.driver, driver, sizeof(req.driver));
+
+		if (port_mode == CCTL_PORT_MODE_REMOVE) {
+			req.reqtype = CTL_REQ_REMOVE;
+			nvlist_add_stringf(option_list, "port_id", "%d",
+			    targ_port);
+		} else
+			req.reqtype = CTL_REQ_CREATE;
+
+		req.args = nvlist_pack(option_list, &req.args_len);
+		if (req.args == NULL) {
+			warn("%s: error packing nvlist", __func__);
+			retval = 1;
+			goto bailout;
+		}
+
+		retval = ioctl(fd, CTL_PORT_REQ, &req);
+		free(req.args);
+		if (retval == -1) {
+			warn("%s: CTL_PORT_REQ ioctl failed", __func__);
+			retval = 1;
+			goto bailout;
+		}
+
+		switch (req.status) {
+		case CTL_LUN_ERROR:
+			warnx("error: %s", req.error_str);
+			retval = 1;
+			goto bailout;
+		case CTL_LUN_WARNING:
+			warnx("warning: %s", req.error_str);
+			break;
+		case CTL_LUN_OK:
+			break;
+		default:
+			warnx("unknown status: %d", req.status);
+			retval = 1;
+			goto bailout;
+		}
+
+		break;
+	}
 	case CCTL_PORT_MODE_SET:
 		if (targ_port == -1) {
 			warnx("%s: -w and -W require -n", __func__);
@@ -561,7 +667,8 @@ cctl_port(int fd, int argc, char **argv, char *combinedopt)
 	}
 
 bailout:
-
+	nvlist_destroy(req.args_nvl);
+	free(driver);
 	return (retval);
 
 bailout_badarg:
@@ -2271,14 +2378,6 @@ bailout:
 	return (retval);
 }
 
-struct cctl_req_option {
-	char			     *name;
-	int			      namelen;
-	char			     *value;
-	int			      vallen;
-	STAILQ_ENTRY(cctl_req_option) links;
-};
-
 static int
 cctl_create_lun(int fd, int argc, char **argv, char *combinedopt)
 {
@@ -2290,11 +2389,12 @@ cctl_create_lun(int fd, int argc, char **argv, char *combinedopt)
 	char *device_id = NULL;
 	int lun_size_set = 0, blocksize_set = 0, lun_id_set = 0;
 	char *backend_name = NULL;
-	STAILQ_HEAD(, cctl_req_option) option_list;
-	int num_options = 0;
+	nvlist_t *option_list;
 	int retval = 0, c;
 
-	STAILQ_INIT(&option_list);
+	option_list = nvlist_create(0);
+	if (option_list == NULL)
+		err(1, "%s: unable to allocate nvlist", __func__);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
@@ -2313,7 +2413,6 @@ cctl_create_lun(int fd, int argc, char **argv, char *combinedopt)
 			lun_id_set = 1;
 			break;
 		case 'o': {
-			struct cctl_req_option *option;
 			char *tmpstr;
 			char *name, *value;
 
@@ -2332,21 +2431,8 @@ cctl_create_lun(int fd, int argc, char **argv, char *combinedopt)
 				retval = 1;
 				goto bailout;
 			}
-			option = malloc(sizeof(*option));
-			if (option == NULL) {
-				warn("%s: error allocating %zd bytes",
-				     __func__, sizeof(*option));
-				retval = 1;
-				goto bailout;
-			}
-			option->name = strdup(name);
-			option->namelen = strlen(name) + 1;
-			option->value = strdup(value);
-			option->vallen = strlen(value) + 1;
 			free(tmpstr);
-
-			STAILQ_INSERT_TAIL(&option_list, option, links);
-			num_options++;
+			nvlist_add_string(option_list, name, value);
 			break;
 		}
 		case 's':
@@ -2413,43 +2499,16 @@ cctl_create_lun(int fd, int argc, char **argv, char *combinedopt)
 		req.reqdata.create.flags |= CTL_LUN_FLAG_DEVID;
 	}
 
-	req.num_be_args = num_options;
-	if (num_options > 0) {
-		struct cctl_req_option *option, *next_option;
-		int i;
-
-		req.be_args = malloc(num_options * sizeof(*req.be_args));
-		if (req.be_args == NULL) {
-			warn("%s: error allocating %zd bytes", __func__,
-			     num_options * sizeof(*req.be_args));
-			retval = 1;
-			goto bailout;
-		}
-
-		for (i = 0, option = STAILQ_FIRST(&option_list);
-		     i < num_options; i++, option = next_option) {
-			next_option = STAILQ_NEXT(option, links);
-
-			req.be_args[i].namelen = option->namelen;
-			req.be_args[i].name = strdup(option->name);
-			req.be_args[i].vallen = option->vallen;
-			req.be_args[i].value = strdup(option->value);
-			/*
-			 * XXX KDM do we want a way to specify a writeable
-			 * flag of some sort?  Do we want a way to specify
-			 * binary data?
-			 */
-			req.be_args[i].flags = CTL_BEARG_ASCII | CTL_BEARG_RD;
-
-			STAILQ_REMOVE(&option_list, option, cctl_req_option,
-				      links);
-			free(option->name);
-			free(option->value);
-			free(option);
-		}
+	req.args = nvlist_pack(option_list, &req.args_len);
+	if (req.args == NULL) {
+		warn("%s: error packing nvlist", __func__);
+		retval = 1;
+		goto bailout;
 	}
 
-	if (ioctl(fd, CTL_LUN_REQ, &req) == -1) {
+	retval = ioctl(fd, CTL_LUN_REQ, &req);
+	free(req.args);
+	if (retval == -1) {
 		warn("%s: error issuing CTL_LUN_REQ ioctl", __func__);
 		retval = 1;
 		goto bailout;
@@ -2480,9 +2539,10 @@ cctl_create_lun(int fd, int argc, char **argv, char *combinedopt)
 		req.reqdata.create.blocksize_bytes);
 	fprintf(stdout, "LUN ID:        %d\n", req.reqdata.create.req_lun_id);
 	fprintf(stdout, "Serial Number: %s\n", req.reqdata.create.serial_num);
-	fprintf(stdout, "Device ID;     %s\n", req.reqdata.create.device_id);
+	fprintf(stdout, "Device ID:     %s\n", req.reqdata.create.device_id);
 
 bailout:
+	nvlist_destroy(req.args_nvl);
 	return (retval);
 }
 
@@ -2493,11 +2553,12 @@ cctl_rm_lun(int fd, int argc, char **argv, char *combinedopt)
 	uint32_t lun_id = 0;
 	int lun_id_set = 0;
 	char *backend_name = NULL;
-	STAILQ_HEAD(, cctl_req_option) option_list;
-	int num_options = 0;
+	nvlist_t *option_list;
 	int retval = 0, c;
 
-	STAILQ_INIT(&option_list);
+	option_list = nvlist_create(0);
+	if (option_list == NULL)
+		err(1, "%s: unable to allocate nvlist", __func__);
 
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
@@ -2509,7 +2570,6 @@ cctl_rm_lun(int fd, int argc, char **argv, char *combinedopt)
 			lun_id_set = 1;
 			break;
 		case 'o': {
-			struct cctl_req_option *option;
 			char *tmpstr;
 			char *name, *value;
 
@@ -2528,21 +2588,8 @@ cctl_rm_lun(int fd, int argc, char **argv, char *combinedopt)
 				retval = 1;
 				goto bailout;
 			}
-			option = malloc(sizeof(*option));
-			if (option == NULL) {
-				warn("%s: error allocating %zd bytes",
-				     __func__, sizeof(*option));
-				retval = 1;
-				goto bailout;
-			}
-			option->name = strdup(name);
-			option->namelen = strlen(name) + 1;
-			option->value = strdup(value);
-			option->vallen = strlen(value) + 1;
 			free(tmpstr);
-
-			STAILQ_INSERT_TAIL(&option_list, option, links);
-			num_options++;
+			nvlist_add_string(option_list, name, value);
 			break;
 		}
 		default:
@@ -2562,44 +2609,17 @@ cctl_rm_lun(int fd, int argc, char **argv, char *combinedopt)
 	req.reqtype = CTL_LUNREQ_RM;
 
 	req.reqdata.rm.lun_id = lun_id;
-
-	req.num_be_args = num_options;
-	if (num_options > 0) {
-		struct cctl_req_option *option, *next_option;
-		int i;
-
-		req.be_args = malloc(num_options * sizeof(*req.be_args));
-		if (req.be_args == NULL) {
-			warn("%s: error allocating %zd bytes", __func__,
-			     num_options * sizeof(*req.be_args));
-			retval = 1;
-			goto bailout;
-		}
-
-		for (i = 0, option = STAILQ_FIRST(&option_list);
-		     i < num_options; i++, option = next_option) {
-			next_option = STAILQ_NEXT(option, links);
-
-			req.be_args[i].namelen = option->namelen;
-			req.be_args[i].name = strdup(option->name);
-			req.be_args[i].vallen = option->vallen;
-			req.be_args[i].value = strdup(option->value);
-			/*
-			 * XXX KDM do we want a way to specify a writeable
-			 * flag of some sort?  Do we want a way to specify
-			 * binary data?
-			 */
-			req.be_args[i].flags = CTL_BEARG_ASCII | CTL_BEARG_RD;
-
-			STAILQ_REMOVE(&option_list, option, cctl_req_option,
-				      links);
-			free(option->name);
-			free(option->value);
-			free(option);
-		}
+		
+	req.args = nvlist_pack(option_list, &req.args_len);
+	if (req.args == NULL) {
+		warn("%s: error packing nvlist", __func__);
+		retval = 1;
+		goto bailout;
 	}
 
-	if (ioctl(fd, CTL_LUN_REQ, &req) == -1) {
+	retval = ioctl(fd, CTL_LUN_REQ, &req);
+	free(req.args);
+	if (retval == -1) {
 		warn("%s: error issuing CTL_LUN_REQ ioctl", __func__);
 		retval = 1;
 		goto bailout;
@@ -2624,6 +2644,7 @@ cctl_rm_lun(int fd, int argc, char **argv, char *combinedopt)
 	printf("LUN %d removed successfully\n", lun_id);
 
 bailout:
+	nvlist_destroy(req.args_nvl);
 	return (retval);
 }
 
@@ -2635,11 +2656,13 @@ cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt)
 	uint32_t lun_id = 0;
 	int lun_id_set = 0, lun_size_set = 0;
 	char *backend_name = NULL;
-	STAILQ_HEAD(, cctl_req_option) option_list;
-	int num_options = 0;
+	nvlist_t *option_list;
 	int retval = 0, c;
 
-	STAILQ_INIT(&option_list);
+	option_list = nvlist_create(0);
+	if (option_list == NULL)
+		err(1, "%s: unable to allocate nvlist", __func__);
+
 	while ((c = getopt(argc, argv, combinedopt)) != -1) {
 		switch (c) {
 		case 'b':
@@ -2650,7 +2673,6 @@ cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt)
 			lun_id_set = 1;
 			break;
 		case 'o': {
-			struct cctl_req_option *option;
 			char *tmpstr;
 			char *name, *value;
 
@@ -2669,21 +2691,8 @@ cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt)
 				retval = 1;
 				goto bailout;
 			}
-			option = malloc(sizeof(*option));
-			if (option == NULL) {
-				warn("%s: error allocating %zd bytes",
-				     __func__, sizeof(*option));
-				retval = 1;
-				goto bailout;
-			}
-			option->name = strdup(name);
-			option->namelen = strlen(name) + 1;
-			option->value = strdup(value);
-			option->vallen = strlen(value) + 1;
 			free(tmpstr);
-
-			STAILQ_INSERT_TAIL(&option_list, option, links);
-			num_options++;
+			nvlist_add_string(option_list, name, value);
 			break;
 		}
 		case 's':
@@ -2709,7 +2718,7 @@ cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt)
 	if (lun_id_set == 0)
 		errx(1, "%s: LUN id (-l) must be specified", __func__);
 
-	if (lun_size_set == 0 && num_options == 0)
+	if (lun_size_set == 0 && nvlist_empty(option_list))
 		errx(1, "%s: size (-s) or options (-o) must be specified",
 		    __func__);
 
@@ -2721,43 +2730,16 @@ cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt)
 	req.reqdata.modify.lun_id = lun_id;
 	req.reqdata.modify.lun_size_bytes = lun_size;
 
-	req.num_be_args = num_options;
-	if (num_options > 0) {
-		struct cctl_req_option *option, *next_option;
-		int i;
-
-		req.be_args = malloc(num_options * sizeof(*req.be_args));
-		if (req.be_args == NULL) {
-			warn("%s: error allocating %zd bytes", __func__,
-			     num_options * sizeof(*req.be_args));
-			retval = 1;
-			goto bailout;
-		}
-
-		for (i = 0, option = STAILQ_FIRST(&option_list);
-		     i < num_options; i++, option = next_option) {
-			next_option = STAILQ_NEXT(option, links);
-
-			req.be_args[i].namelen = option->namelen;
-			req.be_args[i].name = strdup(option->name);
-			req.be_args[i].vallen = option->vallen;
-			req.be_args[i].value = strdup(option->value);
-			/*
-			 * XXX KDM do we want a way to specify a writeable
-			 * flag of some sort?  Do we want a way to specify
-			 * binary data?
-			 */
-			req.be_args[i].flags = CTL_BEARG_ASCII | CTL_BEARG_RD;
-
-			STAILQ_REMOVE(&option_list, option, cctl_req_option,
-				      links);
-			free(option->name);
-			free(option->value);
-			free(option);
-		}
+	req.args = nvlist_pack(option_list, &req.args_len);
+	if (req.args == NULL) {
+		warn("%s: error packing nvlist", __func__);
+		retval = 1;
+		goto bailout;
 	}
 
-	if (ioctl(fd, CTL_LUN_REQ, &req) == -1) {
+	retval = ioctl(fd, CTL_LUN_REQ, &req);
+	free(req.args);
+	if (retval == -1) {
 		warn("%s: error issuing CTL_LUN_REQ ioctl", __func__);
 		retval = 1;
 		goto bailout;
@@ -2782,6 +2764,7 @@ cctl_modify_lun(int fd, int argc, char **argv, char *combinedopt)
 	printf("LUN %d modified successfully\n", lun_id);
 
 bailout:
+	nvlist_destroy(req.args_nvl);
 	return (retval);
 }
 
@@ -3661,7 +3644,7 @@ cctl_portlist(int fd, int argc, char **argv, char *combinedopt)
 	struct cctl_portlist_data portlist;
 	struct cctl_port *port;
 	XML_Parser parser;
-	char *port_str;
+	char *port_str = NULL;
 	int port_len;
 	int dump_xml = 0;
 	int retval, c;
@@ -3704,7 +3687,7 @@ cctl_portlist(int fd, int argc, char **argv, char *combinedopt)
 	}
 
 retry:
-	port_str = malloc(port_len);
+	port_str = (char *)realloc(port_str, port_len);
 
 	bzero(&list, sizeof(list));
 	list.alloc_len = port_len;
@@ -3876,6 +3859,8 @@ usage(int error)
 "                            [-s len fmt [args]] [-c] [-d delete_id]\n"
 "         ctladm port        <-o <on|off> | [-w wwnn][-W wwpn]>\n"
 "                            [-p targ_port] [-t port_type]\n"
+"                            <-c> [-d driver] [-O name=value]\n"
+"                            <-r> <-p targ_port>\n"
 "         ctladm portlist    [-f frontend] [-i] [-p targ_port] [-q] [-v] [-x]\n"
 "         ctladm islist      [-v | -x]\n"
 "         ctladm islogout    <-a | -c connection-id | -i name | -p portal>\n"
@@ -3954,12 +3939,16 @@ usage(int error)
 "-c                       : continuous operation\n"
 "-d delete_id             : error id to delete\n"
 "port options:\n"
+"-c                       : create new ioctl or iscsi frontend port\n"
+"-d                       : specify ioctl or iscsi frontend type\n"
 "-l                       : list frontend ports\n"
 "-o on|off                : turn frontend ports on or off\n"
+"-O pp|vp                 : create new frontend port using pp and/or vp\n"
 "-w wwnn                  : set WWNN for one frontend\n"
 "-W wwpn                  : set WWPN for one frontend\n"
 "-t port_type             : specify fc, scsi, ioctl, internal frontend type\n"
 "-p targ_port             : specify target port number\n"
+"-r                       : remove frontend port\n" 
 "-q                       : omit header in list output\n"
 "-x                       : output port list in XML format\n"
 "portlist options:\n"
diff --git a/usr.sbin/ctld/Makefile b/usr.sbin/ctld/Makefile
index 944b94d663c8..2d13099c7a9a 100644
--- a/usr.sbin/ctld/Makefile
+++ b/usr.sbin/ctld/Makefile
@@ -15,7 +15,7 @@ CFLAGS+=	-I${SRCTOP}/sys/dev/iscsi
 #CFLAGS+=	-DICL_KERNEL_PROXY
 MAN=		ctld.8 ctl.conf.5
 
-LIBADD=		bsdxml l md sbuf util ucl m
+LIBADD=		bsdxml l md sbuf util ucl m nv
 
 YFLAGS+=	-v
 CLEANFILES=	y.tab.c y.tab.h y.output
diff --git a/usr.sbin/ctld/ctld.c b/usr.sbin/ctld/ctld.c
index a728682a9e63..c1dbac6f7956 100644
--- a/usr.sbin/ctld/ctld.c
+++ b/usr.sbin/ctld/ctld.c
@@ -1234,6 +1234,7 @@ port_new(struct conf *conf, struct target *target, struct portal_group *pg)
 		log_err(1, "calloc");
 	port->p_conf = conf;
 	port->p_name = name;
+	port->p_ioctl_port = 0;
 	TAILQ_INSERT_TAIL(&conf->conf_ports, port, p_next);
 	TAILQ_INSERT_TAIL(&target->t_ports, port, p_ts);
 	port->p_target = target;
@@ -1243,6 +1244,51 @@ port_new(struct conf *conf, struct target *target, struct portal_group *pg)
 }
 
 struct port *
+port_new_ioctl(struct conf *conf, struct target *target, int pp, int vp)
+{
+	struct pport *pport;
+	struct port *port;
+	char *pname;
+	char *name;
+	int ret;
+
+	ret = asprintf(&pname, "ioctl/%d/%d", pp, vp);
+	if (ret <= 0) {
+		log_err(1, "asprintf");
+		return (NULL);
+	}
+
+	pport = pport_find(conf, pname);
+	if (pport != NULL) {
+		free(pname);
+		return (port_new_pp(conf, target, pport));
+	}
+
+	ret = asprintf(&name, "%s-%s", pname, target->t_name);
+	free(pname);
+
+	if (ret <= 0)
+		log_err(1, "asprintf");
+	if (port_find(conf, name) != NULL) {
+		log_warnx("duplicate port \"%s\"", name);
+		free(name);
+		return (NULL);
+	}
+	port = calloc(1, sizeof(*port));
+	if (port == NULL)
+		log_err(1, "calloc");
+	port->p_conf = conf;
+	port->p_name = name;
+	port->p_ioctl_port = 1;
+	port->p_ioctl_pp = pp;
+	port->p_ioctl_vp = vp;
+	TAILQ_INSERT_TAIL(&conf->conf_ports, port, p_next);
+	TAILQ_INSERT_TAIL(&target->t_ports, port, p_ts);
+	port->p_target = target;
+	return (port);
+}
+
+struct port *
 port_new_pp(struct conf *conf, struct target *target, struct pport *pp)
 {
 	struct port *port;
@@ -1627,9 +1673,9 @@ conf_print(struct conf *conf)
 		TAILQ_FOREACH(auth_name, &ag->ag_names, an_next)
 			fprintf(stderr, "\t initiator-name %s\n",
 			    auth_name->an_initator_name);
-		TAILQ_FOREACH(auth_portal, &ag->ag_portals, an_next)
+		TAILQ_FOREACH(auth_portal, &ag->ag_portals, ap_next)
 			fprintf(stderr, "\t initiator-portal %s\n",
-			    auth_portal->an_initator_portal);
+			    auth_portal->ap_initator_portal);
 		fprintf(stderr, "}\n");
 	}
 	TAILQ_FOREACH(pg, &conf->conf_portal_groups, pg_next) {
@@ -1643,7 +1689,7 @@ conf_print(struct conf *conf)
 		fprintf(stderr, "\t\tpath %s\n", lun->l_path);
 		TAILQ_FOREACH(o, &lun->l_options, o_next)
 			fprintf(stderr, "\t\toption %s %s\n",
-			    lo->o_name, lo->o_value);
+			    o->o_name, o->o_value);
 		fprintf(stderr, "\t}\n");
 	}
 	TAILQ_FOREACH(targ, &conf->conf_targets, t_next) {
diff --git a/usr.sbin/ctld/ctld.h b/usr.sbin/ctld/ctld.h
index 8644a806f932..60dac68c0e0c 100644
--- a/usr.sbin/ctld/ctld.h
+++ b/usr.sbin/ctld/ctld.h
@@ -152,6 +152,9 @@ struct port {
 	struct pport			*p_pport;
 	struct target			*p_target;
 
+	int				p_ioctl_port;
+	int				p_ioctl_pp;
+	int				p_ioctl_vp;
 	uint32_t			p_ctl_port;
 };
 
@@ -368,6 +371,8 @@ void			pport_delete(struct pport *pport);
 
 struct port		*port_new(struct conf *conf, struct target *target,
 			    struct portal_group *pg);
+struct port		*port_new_ioctl(struct conf *conf, struct target *target,
+			    int pp, int vp);
 struct port		*port_new_pp(struct conf *conf, struct target *target,
 			    struct pport *pp);
 struct port		*port_find(const struct conf *conf, const char *name);
diff --git a/usr.sbin/ctld/kernel.c b/usr.sbin/ctld/kernel.c
index f300eff35bb9..166025f4587e 100644
--- a/usr.sbin/ctld/kernel.c
+++ b/usr.sbin/ctld/kernel.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2003, 2004 Silicon Graphics International Corp.
  * Copyright (c) 1997-2007 Kenneth D. Merry
  * Copyright (c) 2012 The FreeBSD Foundation
+ * Copyright (c) 2017 Jakub Wojciech Klama <jceel@FreeBSD.org>
  * All rights reserved.
  *
  * Portions of this software were developed by Edward Tomasz Napierala
@@ -47,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/module.h>
 #include <sys/queue.h>
 #include <sys/sbuf.h>
+#include <sys/nv.h>
 #include <sys/stat.h>
 #include <assert.h>
 #include <bsdxml.h>
@@ -73,6 +75,8 @@ __FBSDID("$FreeBSD$");
 #include <netdb.h>
 #endif
 
+#define	NVLIST_BUFSIZE	1024
+
 extern bool proxy_mode;
 
 static int	ctl_fd = 0;
@@ -652,23 +656,12 @@ retry_port:
 	return (conf);
 }
 
-static void
-str_arg(struct ctl_be_arg *arg, const char *name, const char *value)
-{
-
-	arg->namelen = strlen(name) + 1;
-	arg->name = __DECONST(char *, name);
-	arg->vallen = strlen(value) + 1;
-	arg->value = __DECONST(char *, value);
-	arg->flags = CTL_BEARG_ASCII | CTL_BEARG_RD;
-}
-
 int
 kernel_lun_add(struct lun *lun)
 {
 	struct option *o;
 	struct ctl_lun_req req;
-	int error, i, num_options;
+	int error;
 
 	bzero(&req, sizeof(req));
 
@@ -724,29 +717,26 @@ kernel_lun_add(struct lun *lun)
 		assert(o != NULL);
 	}
 
-	num_options = 0;
-	TAILQ_FOREACH(o, &lun->l_options, o_next)
-		num_options++;
-
-	req.num_be_args = num_options;
-	if (num_options > 0) {
-		req.be_args = malloc(num_options * sizeof(*req.be_args));
-		if (req.be_args == NULL) {
-			log_warn("error allocating %zd bytes",
-			    num_options * sizeof(*req.be_args));
+	if (!TAILQ_EMPTY(&lun->l_options)) {
+		req.args_nvl = nvlist_create(0);
+		if (req.args_nvl == NULL) {
+			log_warn("error allocating nvlist");
 			return (1);
 		}
 
-		i = 0;
-		TAILQ_FOREACH(o, &lun->l_options, o_next) {
-			str_arg(&req.be_args[i], o->o_name, o->o_value);
-			i++;
+		TAILQ_FOREACH(o, &lun->l_options, o_next)
+			nvlist_add_string(req.args_nvl, o->o_name, o->o_value);
+
+		req.args = nvlist_pack(req.args_nvl, &req.args_len);
+		if (req.args == NULL) {
+			log_warn("error packing nvlist");
+			return (1);
 		}
-		assert(i == num_options);
 	}
 
 	error = ioctl(ctl_fd, CTL_LUN_REQ, &req);
-	free(req.be_args);
+	nvlist_destroy(req.args_nvl);
+
 	if (error != 0) {
 		log_warn("error issuing CTL_LUN_REQ ioctl");
 		return (1);
@@ -776,7 +766,7 @@ kernel_lun_modify(struct lun *lun)
 {
 	struct option *o;
 	struct ctl_lun_req req;
-	int error, i, num_options;
+	int error;
 
 	bzero(&req, sizeof(req));
 
@@ -786,29 +776,26 @@ kernel_lun_modify(struct lun *lun)
 	req.reqdata.modify.lun_id = lun->l_ctl_lun;
 	req.reqdata.modify.lun_size_bytes = lun->l_size;
 
-	num_options = 0;
-	TAILQ_FOREACH(o, &lun->l_options, o_next)
-		num_options++;
-
-	req.num_be_args = num_options;
-	if (num_options > 0) {
-		req.be_args = malloc(num_options * sizeof(*req.be_args));
-		if (req.be_args == NULL) {
-			log_warn("error allocating %zd bytes",
-			    num_options * sizeof(*req.be_args));
+	if (!TAILQ_EMPTY(&lun->l_options)) {
+		req.args_nvl = nvlist_create(0);
+		if (req.args_nvl == NULL) {
+			log_warn("error allocating nvlist");
 			return (1);
 		}
 
-		i = 0;
-		TAILQ_FOREACH(o, &lun->l_options, o_next) {
-			str_arg(&req.be_args[i], o->o_name, o->o_value);
-			i++;
+		TAILQ_FOREACH(o, &lun->l_options, o_next)
+			nvlist_add_string(req.args_nvl, o->o_name, o->o_value);
+
+		req.args = nvlist_pack(req.args_nvl, &req.args_len);
+		if (req.args == NULL) {
+			log_warn("error packing nvlist");
+			return (1);
 		}
-		assert(i == num_options);
 	}
 
 	error = ioctl(ctl_fd, CTL_LUN_REQ, &req);
-	free(req.be_args);
+	nvlist_destroy(req.args_nvl);
+
 	if (error != 0) {
 		log_warn("error issuing CTL_LUN_REQ ioctl");
 		return (1);
@@ -988,37 +975,54 @@ kernel_port_add(struct port *port)
 	struct ctl_lun_map lm;
 	struct target *targ = port->p_target;
 	struct portal_group *pg = port->p_portal_group;
-	char tagstr[16];
-	int error, i, n;
+	char result_buf[NVLIST_BUFSIZE];
+	int error, i;
 
 	/* Create iSCSI port. */
-	if (port->p_portal_group) {
+	if (port->p_portal_group || port->p_ioctl_port) {
 		bzero(&req, sizeof(req));
-		strlcpy(req.driver, "iscsi", sizeof(req.driver));
 		req.reqtype = CTL_REQ_CREATE;
-		req.num_args = 5;
-		TAILQ_FOREACH(o, &pg->pg_options, o_next)
-			req.num_args++;
-		req.args = malloc(req.num_args * sizeof(*req.args));
-		if (req.args == NULL)
-			log_err(1, "malloc");
-		n = 0;
-		req.args[n].namelen = sizeof("port_id");
-		req.args[n].name = __DECONST(char *, "port_id");
-		req.args[n].vallen = sizeof(port->p_ctl_port);
-		req.args[n].value = &port->p_ctl_port;
-		req.args[n++].flags = CTL_BEARG_WR;
-		str_arg(&req.args[n++], "cfiscsi_target", targ->t_name);
-		snprintf(tagstr, sizeof(tagstr), "%d", pg->pg_tag);
-		str_arg(&req.args[n++], "cfiscsi_portal_group_tag", tagstr);
-		if (targ->t_alias)
-			str_arg(&req.args[n++], "cfiscsi_target_alias", targ->t_alias);
-		str_arg(&req.args[n++], "ctld_portal_group_name", pg->pg_name);
-		TAILQ_FOREACH(o, &pg->pg_options, o_next)
-			str_arg(&req.args[n++], o->o_name, o->o_value);
-		req.num_args = n;
+
+		if (port->p_portal_group) {
+			strlcpy(req.driver, "iscsi", sizeof(req.driver));
+			req.args_nvl = nvlist_create(0);
+			nvlist_add_string(req.args_nvl, "cfiscsi_target",
+			    targ->t_name);
+			nvlist_add_string(req.args_nvl,
+			    "ctld_portal_group_name", pg->pg_name);
+			nvlist_add_stringf(req.args_nvl,
+			    "cfiscsi_portal_group_tag", "%u", pg->pg_tag);
+
+			if (targ->t_alias) {
+				nvlist_add_string(req.args_nvl,
+				    "cfiscsi_target_alias", targ->t_alias);
+			}
+
+			TAILQ_FOREACH(o, &pg->pg_options, o_next)
+				nvlist_add_string(req.args_nvl, o->o_name,
+				    o->o_value);
+		}
+
+		if (port->p_ioctl_port) {
+			strlcpy(req.driver, "ioctl", sizeof(req.driver));
+			req.args_nvl = nvlist_create(0);
+			nvlist_add_stringf(req.args_nvl, "pp", "%d",
+			    port->p_ioctl_pp);
+			nvlist_add_stringf(req.args_nvl, "vp", "%d",
+			    port->p_ioctl_vp);
+		}
+
+		req.args = nvlist_pack(req.args_nvl, &req.args_len);
+		if (req.args == NULL) {
+			log_warn("error packing nvlist");
+			return (1);
+		}
+
+		req.result = result_buf;
+		req.result_len = sizeof(result_buf);
 		error = ioctl(ctl_fd, CTL_PORT_REQ, &req);
-		free(req.args);
+		nvlist_destroy(req.args_nvl);
+
 		if (error != 0) {
 			log_warn("error issuing CTL_PORT_REQ ioctl");
 			return (1);
@@ -1033,6 +1037,15 @@ kernel_port_add(struct port *port)
 			    req.status);
 			return (1);
 		}
+
+		req.result_nvl = nvlist_unpack(result_buf, req.result_len, 0);
+		if (req.result_nvl == NULL) {
+			log_warnx("error unpacking result nvlist");
+			return (1);
+		}
+
+		port->p_ctl_port = nvlist_get_number(req.result_nvl, "port_id");
+		nvlist_destroy(req.result_nvl);
 	} else if (port->p_pport) {
 		port->p_ctl_port = port->p_pport->pp_ctl_port;
 
@@ -1116,7 +1129,6 @@ kernel_port_remove(struct port *port)
 	struct ctl_port_entry entry;
 	struct ctl_lun_map lm;
 	struct ctl_req req;
-	char tagstr[16];
 	struct target *targ = port->p_target;
 	struct portal_group *pg = port->p_portal_group;
 	int error;
@@ -1130,20 +1142,35 @@ kernel_port_remove(struct port *port)
 		return (-1);
 	}
 
-	/* Remove iSCSI port. */
-	if (port->p_portal_group) {
+	/* Remove iSCSI or ioctl port. */
+	if (port->p_portal_group || port->p_ioctl_port) {
 		bzero(&req, sizeof(req));
-		strlcpy(req.driver, "iscsi", sizeof(req.driver));
+		strlcpy(req.driver, port->p_ioctl_port ? "ioctl" : "iscsi",
+		    sizeof(req.driver));
 		req.reqtype = CTL_REQ_REMOVE;
-		req.num_args = 2;
-		req.args = malloc(req.num_args * sizeof(*req.args));
-		if (req.args == NULL)
-			log_err(1, "malloc");
-		str_arg(&req.args[0], "cfiscsi_target", targ->t_name);
-		snprintf(tagstr, sizeof(tagstr), "%d", pg->pg_tag);
-		str_arg(&req.args[1], "cfiscsi_portal_group_tag", tagstr);
+		req.args_nvl = nvlist_create(0);
+		if (req.args_nvl == NULL)
+			log_err(1, "nvlist_create");
+
+		if (port->p_ioctl_port)
+			nvlist_add_stringf(req.args_nvl, "port_id", "%d",
+			    port->p_ctl_port);
+		else {
+			nvlist_add_string(req.args_nvl, "cfiscsi_target",
+			    targ->t_name);
+			nvlist_add_stringf(req.args_nvl,
+			    "cfiscsi_portal_group_tag", "%u", pg->pg_tag);
+		}
+
+		req.args = nvlist_pack(req.args_nvl, &req.args_len);
+		if (req.args == NULL) {
+			log_warn("error packing nvlist");
+			return (1);
+		}
+
 		error = ioctl(ctl_fd, CTL_PORT_REQ, &req);
-		free(req.args);
+		nvlist_destroy(req.args_nvl);
+
 		if (error != 0) {
 			log_warn("error issuing CTL_PORT_REQ ioctl");
 			return (1);
diff --git a/usr.sbin/ctld/parse.y b/usr.sbin/ctld/parse.y
index 3d8f2fc92313..c9478405599a 100644
--- a/usr.sbin/ctld/parse.y
+++ b/usr.sbin/ctld/parse.y
@@ -768,28 +768,41 @@ target_port:	PORT STR
 	{
 		struct pport *pp;
 		struct port *tp;
-
-		pp = pport_find(conf, $2);
-		if (pp == NULL) {
-			log_warnx("unknown port \"%s\" for target \"%s\"",
-			    $2, target->t_name);
-			free($2);
-			return (1);
-		}
-		if (!TAILQ_EMPTY(&pp->pp_ports)) {
-			log_warnx("can't link port \"%s\" to target \"%s\", "
-			    "port already linked to some target",
-			    $2, target->t_name);
-			free($2);
-			return (1);
-		}
-		tp = port_new_pp(conf, target, pp);
-		if (tp == NULL) {
-			log_warnx("can't link port \"%s\" to target \"%s\"",
-			    $2, target->t_name);
-			free($2);
-			return (1);
+		int ret, i_pp, i_vp = 0;
+
+		ret = sscanf($2, "ioctl/%d/%d", &i_pp, &i_vp);
+		if (ret > 0) {
+			tp = port_new_ioctl(conf, target, i_pp, i_vp);
+			if (tp == NULL) {
+				log_warnx("can't create new ioctl port for "
+				    "target \"%s\"", target->t_name);
+				free($2);
+				return (1);
+			}
+		} else {
+			pp = pport_find(conf, $2);
+			if (pp == NULL) {
+				log_warnx("unknown port \"%s\" for target \"%s\"",
+				    $2, target->t_name);
+				free($2);
+				return (1);
+			}
+			if (!TAILQ_EMPTY(&pp->pp_ports)) {
+				log_warnx("can't link port \"%s\" to target \"%s\", "
+				    "port already linked to some target",
+				    $2, target->t_name);
+				free($2);
+				return (1);
+			}
+			tp = port_new_pp(conf, target, pp);
+			if (tp == NULL) {
+				log_warnx("can't link port \"%s\" to target \"%s\"",
+				    $2, target->t_name);
+				free($2);
+				return (1);
+			}
 		}
+
 		free($2);
 	}
 	;
diff --git a/usr.sbin/ctld/uclparse.c b/usr.sbin/ctld/uclparse.c
index f26e84903e15..3d39da42569e 100644
--- a/usr.sbin/ctld/uclparse.c
+++ b/usr.sbin/ctld/uclparse.c
@@ -758,6 +758,19 @@ uclparse_target(const char *name, const ucl_object_t *top)
 			struct pport *pp;
 			struct port *tp;
 			const char *value = ucl_object_tostring(obj);
+			int ret, i_pp, i_vp = 0;
+
+			ret = sscanf(value, "ioctl/%d/%d", &i_pp, &i_vp);
+			if (ret > 0) {
+				tp = port_new_ioctl(conf, target, i_pp, i_vp);
+				if (tp == NULL) {
+					log_warnx("can't create new ioctl port "
+					    "for target \"%s\"", target->t_name);
+					return (1);
+				}
+
+				return (0);
+			}
 
 			pp = pport_find(conf, value);
 			if (pp == NULL) {