Initial work on the new jail(8). There are more features to add, and some

cleaning up to do on existing features, but this is pretty much what the
final product will look like.

1 files changed, 870 insertions, 385 deletions

diff --git a/usr.sbin/jail/jail.c b/usr.sbin/jail/jail.c
index 0722bfd22397..0da39d9a1cdd 100644
--- a/usr.sbin/jail/jail.c
+++ b/usr.sbin/jail/jail.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 1999 Poul-Henning Kamp.
- * Copyright (c) 2009 James Gritton
+ * Copyright (c) 2009-2010 James Gritton
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -28,44 +28,39 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/param.h>
-#include <sys/jail.h>
+#include <sys/types.h>
+#include <sys/stat.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
 
 #include <arpa/inet.h>
 #include <netinet/in.h>
 
-#include <ctype.h>
 #include <err.h>
 #include <errno.h>
-#include <grp.h>
-#include <jail.h>
-#include <login_cap.h>
-#include <netdb.h>
-#include <paths.h>
-#include <pwd.h>
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
-static struct jailparam *params;
-static char **param_values;
-static int nparams;
+#include "jailp.h"
 
-static char *ip4_addr;
-#ifdef INET6
-static char *ip6_addr;
-#endif
+#define JP_RDTUN(jp)	(((jp)->jp_ctltype & CTLFLAG_RDTUN) == CTLFLAG_RDTUN)
 
-static void add_ip_addr(char **addrp, char *newaddr);
-#ifdef INET6
-static void add_ip_addr46(char *newaddr);
-#endif
-static void add_ip_addrinfo(int ai_flags, char *value);
+const char *cfname;
+int verbose;
+
+static int create_jail(struct cfjail *j);
+static void clear_persist(struct cfjail *j);
+static int update_jail(struct cfjail *j);
+static int rdtun_params(struct cfjail *j, int dofail);
+static void running_jid(struct cfjail *j, int dflag);
+static int jailparam_set_note(const struct cfjail *j, struct jailparam *jp,
+    unsigned njp, int flags);
+static void print_jail(FILE *fp, struct cfjail *j, int oldcl);
+static void print_param(FILE *fp, const struct cfparam *p, int sep, int doname);
 static void quoted_print(FILE *fp, char *str);
-static void set_param(const char *name, char *value);
 static void usage(void);
 
 static const char *perm_sysctl[][3] = {
@@ -83,94 +78,93 @@ static const char *perm_sysctl[][3] = {
 	  "allow.socket_af", "allow.nosocket_af" },
 };
 
-extern char **environ;
-
-#define GET_USER_INFO do {						\
-	pwd = getpwnam(username);					\
-	if (pwd == NULL) {						\
-		if (errno)						\
-			err(1, "getpwnam: %s", username);		\
-		else							\
-			errx(1, "%s: no such user", username);		\
-	}								\
-	lcap = login_getpwclass(pwd);					\
-	if (lcap == NULL)						\
-		err(1, "getpwclass: %s", username);			\
-	ngroups = ngroups_max;						\
-	if (getgrouplist(username, pwd->pw_gid, groups, &ngroups) != 0)	\
-		err(1, "getgrouplist: %s", username);			\
-} while (0)
-
 int
 main(int argc, char **argv)
 {
-	login_cap_t *lcap = NULL;
-	struct passwd *pwd = NULL;
-	gid_t *groups;
+#ifdef INET6
+	struct in6_addr addr6;
+#endif
+	struct stat st;
+	FILE *jfp;
+	struct cfjail *j;
+	char *cs, *ncs, *JidFile;
 	size_t sysvallen;
-	int ch, cmdarg, i, jail_set_flags, jid, ngroups, sysval;
-	int hflag, iflag, Jflag, lflag, rflag, uflag, Uflag;
-	long ngroups_max;
-	unsigned pi;
-	char *jailname, *securelevel, *username, *JidFile;
+	unsigned op, pi;
+	int ch, docf, error, i, oldcl, sysval;
+	int dflag, iflag, plimit, Rflag;
 	char enforce_statfs[4];
-	static char *cleanenv;
-	const char *shell, *p = NULL;
-	FILE *fp;
-
-	hflag = iflag = Jflag = lflag = rflag = uflag = Uflag =
-	    jail_set_flags = 0;
-	cmdarg = jid = -1;
-	jailname = securelevel = username = JidFile = cleanenv = NULL;
-	fp = NULL;
-
-	ngroups_max = sysconf(_SC_NGROUPS_MAX) + 1;	
-	if ((groups = malloc(sizeof(gid_t) * ngroups_max)) == NULL)
-		err(1, "malloc");
 
-	while ((ch = getopt(argc, argv, "cdhilmn:r:s:u:U:J:")) != -1) {
+	op = 0;
+	plimit = -1;
+	dflag = iflag = Rflag = 0;
+	docf = 1;
+	cfname = CONF_FILE;
+	JidFile = NULL;
+
+	while ((ch = getopt(argc, argv, "cdf:hiJ:lmn:p:qrRs:U:v")) != -1) {
 		switch (ch) {
+		case 'c':
+			op |= JF_START;
+			break;
 		case 'd':
-			jail_set_flags |= JAIL_DYING;
+			dflag = 1;
+			break;
+		case 'f':
+			cfname = optarg;
 			break;
 		case 'h':
-			hflag = 1;
+			add_param(NULL, NULL, "ip_hostname", NULL);
+			docf = 0;
 			break;
 		case 'i':
 			iflag = 1;
+			verbose = -1;
 			break;
 		case 'J':
 			JidFile = optarg;
-			Jflag = 1;
+			break;
+		case 'l':
+			add_param(NULL, NULL, "exec.clean", NULL);
+			docf = 0;
+			break;
+		case 'm':
+			op |= JF_SET;
 			break;
 		case 'n':
-			jailname = optarg;
+			add_param(NULL, NULL, "name", optarg);
+			docf = 0;
 			break;
-		case 's':
-			securelevel = optarg;
+		case 'p':
+			plimit = strtol(optarg, NULL, 10);
+			if (plimit == 0)
+				plimit = -1;
 			break;
-		case 'u':
-			username = optarg;
-			uflag = 1;
+		case 'q':
+			verbose = -1;
 			break;
-		case 'U':
-			username = optarg;
-			Uflag = 1;
+		case 'r':
+			op |= JF_STOP;
 			break;
-		case 'l':
-			lflag = 1;
+		case 'R':
+			op |= JF_STOP;
+			Rflag = 1;
 			break;
-		case 'c':
-			jail_set_flags |= JAIL_CREATE;
+		case 's':
+			add_param(NULL, NULL, "securelevel", optarg);
+			docf = 0;
 			break;
-		case 'm':
-			jail_set_flags |= JAIL_UPDATE;
+		case 'u':
+			add_param(NULL, NULL, "exec.jail_user", optarg);
+			add_param(NULL, NULL, "exec.system_jail_user", NULL);
+			docf = 0;
 			break;
-		case 'r':
-			jid = jail_getid(optarg);
-			if (jid < 0)
-				errx(1, "%s", jail_errmsg);
-			rflag = 1;
+		case 'U':
+			add_param(NULL, NULL, "exec.jail_user", optarg);
+			add_param(NULL, NULL, "exec.nosystem_jail_user", NULL);
+			docf = 0;
+			break;
+		case 'v':
+			verbose = 1;
 			break;
 		default:
 			usage();
@@ -178,67 +172,36 @@ main(int argc, char **argv)
 	}
 	argc -= optind;
 	argv += optind;
-	if (rflag) {
-		if (argc > 0 || iflag || Jflag || lflag || uflag || Uflag)
+
+	/* Find out which of the four command line styles this is. */
+	oldcl = 0;
+	if (!op) {
+		/* Old-style command line with four fixed parameters */
+		if (argc < 4 || argv[0][0] != '/')
 			usage();
-		if (jail_remove(jid) < 0)
-			err(1, "jail_remove");
-		exit (0);
-	}
-	if (argc == 0)
-		usage();
-	if (uflag && Uflag)
-		usage();
-	if (lflag && username == NULL)
-		usage();
-	if (uflag)
-		GET_USER_INFO;
-
-	if (jailname)
-		set_param("name", jailname);
-	if (securelevel)
-		set_param("securelevel", securelevel);
-	if (jail_set_flags) {
-		for (i = 0; i < argc; i++) {
-			if (!strncmp(argv[i], "command=", 8)) {
-				cmdarg = i;
-				argv[cmdarg] += 8;
-				jail_set_flags |= JAIL_ATTACH;
-				break;
-			}
-			if (hflag) {
-				if (!strncmp(argv[i], "ip4.addr=", 9)) {
-					add_ip_addr(&ip4_addr, argv[i] + 9);
-					break;
-				}
+		op = JF_START;
+		docf = 0;
+		oldcl = 1;
+		add_param(NULL, NULL, "path", argv[0]);
+		add_param(NULL, NULL, "host.hostname", argv[1]);
+		if (argv[2][0] != '\0') {
+			for (cs = argv[2];; cs = ncs + 1) {
+				ncs = strchr(cs, ',');
+				if (ncs)
+					*ncs = '\0';
+				add_param(NULL, NULL,
 #ifdef INET6
-				if (!strncmp(argv[i], "ip6.addr=", 9)) {
-					add_ip_addr(&ip6_addr, argv[i] + 9);
-					break;
-				}
+				    inet_pton(AF_INET6, cs, &addr6) == 1
+				    ? "ip6.addr" :
 #endif
-				if (!strncmp(argv[i], "host.hostname=", 14))
-					add_ip_addrinfo(0, argv[i] + 14);
+				    "ip4.addr", cs);
+				if (!ncs)
+					break;
 			}
-			set_param(NULL, argv[i]);
 		}
-	} else {
-		if (argc < 4 || argv[0][0] != '/')
-			errx(1, "%s\n%s",
-			   "no -c or -m, so this must be an old-style command.",
-			   "But it doesn't look like one.");
-		set_param("path", argv[0]);
-		set_param("host.hostname", argv[1]);
-		if (hflag)
-			add_ip_addrinfo(0, argv[1]);
-		if (argv[2][0] != '\0')
-#ifdef INET6
-			add_ip_addr46(argv[2]);
-#else
-			add_ip_addr(&ip4_addr, argv[2]);
-#endif
-		cmdarg = 3;
-		/* Emulate the defaults from security.jail.* sysctls */
+		for (i = 3; i < argc; i++)
+			add_param(NULL, NULL, "command", argv[i]);
+		/* Emulate the defaults from security.jail.* sysctls. */
 		sysvallen = sizeof(sysval);
 		if (sysctlbyname("security.jail.jailed", &sysval, &sysvallen,
 		    NULL, 0) == 0 && sysval == 0) {
@@ -247,241 +210,812 @@ main(int argc, char **argv)
 				sysvallen = sizeof(sysval);
 				if (sysctlbyname(perm_sysctl[pi][0],
 				    &sysval, &sysvallen, NULL, 0) == 0)
-					set_param(perm_sysctl[pi]
-					    [sysval ? 2 : 1], NULL);
+					add_param(NULL, NULL,
+					    perm_sysctl[pi][sysval ? 2 : 1],
+					    NULL);
 			}
 			sysvallen = sizeof(sysval);
 			if (sysctlbyname("security.jail.enforce_statfs",
 			    &sysval, &sysvallen, NULL, 0) == 0) {
 				snprintf(enforce_statfs,
 				    sizeof(enforce_statfs), "%d", sysval);
-				set_param("enforce_statfs", enforce_statfs);
+				add_param(NULL, NULL, "enforce_statfs",
+				    enforce_statfs);
 			}
 		}
+	} else if (op == JF_STOP) {
+		/* Jail remove, perhaps using the config file */
+		if (!docf || argc == 0)
+			usage();
+		if (!Rflag)
+			for (i = 0; i < argc; i++)
+				if (strchr(argv[i], '='))
+					usage();
+		if ((docf = !Rflag &&
+		     (!strcmp(cfname, "-") || stat(cfname, &st) == 0)))
+			load_config();
+	} else if (argc > 1 || (argc == 1 && strchr(argv[0], '='))) {
+		/* Single jail specified on the command line */
+		if (Rflag)
+			usage();
+		docf = 0;
+		for (i = 0; i < argc; i++) {
+			if (!strncmp(argv[i], "command", 7) &&
+			    (argv[i][7] == '\0' || argv[i][7] == '=')) {
+				if (argv[i][7]  == '=')
+					add_param(NULL, NULL, "command",
+					    argv[i] + 8);
+				for (i++; i < argc; i++)
+					add_param(NULL, NULL, "command",
+					    argv[i]);
+				break;
+			}
+			if ((cs = strchr(argv[i], '=')))
+				*cs++ = '\0';
+			add_param(NULL, NULL, argv[i], cs);
+		}
+	} else {
+		/* From the config file, perhaps with a specified jail */
+		if (Rflag || !docf)
+			usage();
+		load_config();
 	}
-	if (ip4_addr != NULL)
-		set_param("ip4.addr", ip4_addr);
-#ifdef INET6
-	if (ip6_addr != NULL)
-		set_param("ip6.addr", ip6_addr);
-#endif
 
-	if (Jflag) {
-		fp = fopen(JidFile, "w");
-		if (fp == NULL)
-			errx(1, "Could not create JidFile: %s", JidFile);
+	/* Find out which jails will be run. */
+	find_intparams();
+	dep_setup(docf);
+	error = 0;
+	if (op == JF_STOP) {
+		for (i = 0; i < argc; i++)
+			if (start_state(argv[i], op, Rflag) < 0)
+				error = 1;
+	} else {
+		if (start_state(docf ? argv[0] : NULL, op, 0) < 0)
+			exit(1);
 	}
-	jid = jailparam_set(params, nparams, 
-	    jail_set_flags ? jail_set_flags : JAIL_CREATE | JAIL_ATTACH);
-	if (jid < 0)
-		errx(1, "%s", jail_errmsg);
-	if (iflag) {
-		printf("%d\n", jid);
-		fflush(stdout);
+
+	jfp = NULL;
+	if (JidFile != NULL) {
+		jfp = fopen(JidFile, "w");
+		if (jfp == NULL)
+			err(1, "open %s", JidFile);
+		setlinebuf(jfp);
 	}
-	if (Jflag) {
-		if (jail_set_flags) {
-			fprintf(fp, "jid=%d", jid);
-			for (i = 0; i < nparams; i++)
-				if (strcmp(params[i].jp_name, "jid")) {
-					fprintf(fp, " %s",
-					    (char *)params[i].jp_name);
-					if (param_values[i]) {
-						putc('=', fp);
-						quoted_print(fp,
-						    param_values[i]);
-					}
+	setlinebuf(stdout);
+
+	/*
+	 * The main loop: Get an available jail and perform the required
+	 * operation on it.  When that is done, the jail may be finished,
+	 * or it may go back for the next step.
+	 */
+	while ((j = next_jail()))
+	{
+		if (j->flags & JF_FAILED) {
+			clear_persist(j);
+			if (j->flags & JF_MOUNTED) {
+				(void)run_command(j, NULL, IP_MOUNT_DEVFS);
+				if (run_command(j, NULL, IP_MOUNT_FSTAB))
+					while (run_command(j, NULL, 0)) ;
+				if (run_command(j, NULL, IP_MOUNT))
+					while (run_command(j, NULL, 0)) ;
+			}
+			if (j->flags & JF_IFUP) {
+				if (run_command(j, NULL, IP__IP4_IFADDR))
+					while (run_command(j, NULL, 0)) ;
+#ifdef INET6
+				if (run_command(j, NULL, IP__IP6_IFADDR))
+					while (run_command(j, NULL, 0)) ;
+#endif
+			}
+			error = 1;
+			dep_done(j, 0);
+			continue;
+		}
+		if (!(j->flags & JF_CHECKINT))
+		{
+			j->flags |= JF_CHECKINT;
+			if (dflag)
+				add_param(j, NULL, "allow.dying", NULL);
+			if (check_intparams(j) < 0)
+				continue;
+		}
+		if (!(j->flags & JF_IPPARAMS) && (!JF_DO_STOP(j->flags) ||
+		    j->intparams[IP_INTERFACE] != NULL)) {
+			j->flags |= JF_IPPARAMS;
+			if (ip_params(j) < 0)
+				continue;
+		}
+		if (j->jp == NULL && (j->flags & (JF_START | JF_SET)) &&
+		    import_params(j) < 0)
+			continue;
+		if (!j->jid)
+			running_jid(j,
+			    (j->flags & (JF_SET | JF_DEPEND)) == JF_SET
+			    ? dflag || bool_param(j->intparams[IP_ALLOW_DYING])
+			    : 0);
+		if (j->comstring != NULL &&
+		    (finish_command(j, &plimit) || run_command(j, &plimit, 0)))
+			continue;
+
+		switch (j->flags & JF_OP_MASK) {
+			/*
+			 * These operations just turn into a different op
+			 * depending on the jail's current status.
+			 */
+		case JF_START_SET:
+			j->flags = j->jid < 0 ? JF_START : JF_SET;
+			break;
+		case JF_SET_RESTART:
+			if (j->jid < 0) {
+				warnx("\"%s\" not found", j->name);
+				failed(j);
+				continue;
+			}
+			j->flags = rdtun_params(j, 0) ? JF_RESTART : JF_SET;
+			if (j->flags == JF_RESTART)
+				dep_reset(j);
+			break;
+		case JF_START_SET_RESTART:
+			j->flags = j->jid < 0 ? JF_START
+			    : rdtun_params(j, 0) ? JF_RESTART : JF_SET;
+			if (j->flags == JF_RESTART)
+				dep_reset(j);
+		}
+
+		switch (j->flags & JF_OP_MASK) {
+		case JF_START:
+			/*
+			 * 1: check existence and dependencies
+			 * 2: configure IP addresses
+			 * 3: run any exec.prestart commands
+			 * 4: create the jail
+			 * 5: configure vnet interfaces
+			 * 6: run any exec.start or "command" commands
+			 * 7: run any exec.poststart commands
+			 */
+			switch (j->comparam) {
+			default:
+				if (j->jid > 0 &&
+				    !(j->flags & (JF_DEPEND | JF_WILD))) {
+					warnx("\"%s\" already exists", j->name);
+					failed(j);
+					continue;
 				}
-			fprintf(fp, "\n");
-		} else {
-			for (i = 0; i < nparams; i++)
-				if (!strcmp(params[i].jp_name, "path"))
-					break;
+				if (dep_check(j))
+					continue;
+				if (j->jid > 0)
+					goto jail_create_done;
+				if (run_command(j, &plimit, IP__IP4_IFADDR))
+					continue;
+				/* FALLTHROUGH */
+			case IP__IP4_IFADDR:
+#ifdef INET6
+				if (run_command(j, &plimit, IP__IP6_IFADDR))
+					continue;
+				/* FALLTHROUGH */
+			case IP__IP6_IFADDR:
+#endif
+				if (run_command(j, &plimit, IP_MOUNT))
+					continue;
+				/* FALLTHROUGH */
+			case IP_MOUNT:
+				if (run_command(j, &plimit, IP_MOUNT_FSTAB))
+					continue;
+				/* FALLTHROUGH */
+			case IP_MOUNT_FSTAB:
+				if (run_command(j, &plimit, IP_MOUNT_DEVFS))
+					continue;
+				/* FALLTHROUGH */
+			case IP_MOUNT_DEVFS:
+				if (run_command(j, &plimit, IP_EXEC_PRESTART))
+					continue;
+				/* FALLTHROUGH */
+			case IP_EXEC_PRESTART:
+				if (create_jail(j) < 0)
+					continue;
+				if (iflag)
+					printf("%d\n", j->jid);
+				if (jfp != NULL)
+					print_jail(jfp, j, oldcl);
+				if (verbose >= 0 && (j->name || verbose > 0))
+					jail_note(j, "created\n");
+				dep_done(j, DF_LIGHT);
+				if (bool_param(j->intparams[KP_VNET]) &&
+				    run_command(j, &plimit, IP_VNET_INTERFACE))
+					continue;
+				/* FALLTHROUGH */
+			case IP_VNET_INTERFACE:
+				if (run_command(j, &plimit, IP_EXEC_START))
+					continue;
+				/* FALLTHROUGH */
+			case IP_EXEC_START:
+				if (run_command(j, &plimit, IP_COMMAND))
+					continue;
+				/* FALLTHROUGH */
+			case IP_COMMAND:
+				if (run_command(j, &plimit, IP_EXEC_POSTSTART))
+					continue;
+				/* FALLTHROUGH */
+			case IP_EXEC_POSTSTART:
+			jail_create_done:
+				clear_persist(j);
+				dep_done(j, 0);
+			}
+			break;
+
+		case JF_SET:
+			/*
+			 * 1: check existence and dependencies
+			 * 2: update the jail
+			 */
+			if (j->jid < 0 && !(j->flags & JF_DEPEND)) {
+				warnx("\"%s\" not found", j->name);
+				failed(j);
+				continue;;
+			}
+			if (dep_check(j))
+				continue;
+			if (!(j->flags & JF_DEPEND)) {
+				if (rdtun_params(j, 1) < 0 ||
+				    update_jail(j) < 0)
+					continue;
+				if (verbose >= 0 && (j->name || verbose > 0))
+					jail_note(j, "updated\n");
+			}
+			dep_done(j, 0);
+			break;
+
+		case JF_STOP:
+		case JF_RESTART:
+			/*
+			 * 1: check dependencies and existence (note order)
+			 * 2: run any exec.prestop commands
+			 * 3: run any exec.stop commands
+			 * 4: send SIGTERM to all jail processes
+			 * 5: remove the jail
+			 * 6: run any exec.poststop commands
+			 * 7: take down IP addresses
+			 */
+			switch (j->comparam) {
+			default:
+				if (dep_check(j))
+					continue;
+				if (j->jid < 0) {
+					if (!(j->flags & (JF_DEPEND | JF_WILD))
+					    && verbose >= 0)
+						warnx("\"%s\" not found",
+						    j->name);
+					goto jail_remove_done;
+				}
+				if (run_command(j, &plimit, IP_EXEC_PRESTOP))
+					continue;
+				/* FALLTHROUGH */
+			case IP_EXEC_PRESTOP:
+				if (run_command(j, &plimit, IP_EXEC_STOP))
+					continue;
+				/* FALLTHROUGH */
+			case IP_EXEC_STOP:
+				j->comparam = IP_STOP_TIMEOUT;
+				if (term_procs(j))
+					continue;
+				/* FALLTHROUGH */
+			case IP_STOP_TIMEOUT:
+				(void)jail_remove(j->jid);
+				j->jid = -1;
+				if (verbose >= 0 &&
+				    (docf || argc > 1 ||
+				     wild_jail_name(argv[0]) || verbose > 0))
+					jail_note(j, "removed\n");
+				dep_done(j, DF_LIGHT);
+				if (run_command(j, &plimit, IP_EXEC_POSTSTOP))
+					continue;
+				/* FALLTHROUGH */
+			case IP_EXEC_POSTSTOP:
+				if (run_command(j, &plimit, IP_MOUNT_DEVFS))
+					continue;
+				/* FALLTHROUGH */
+			case IP_MOUNT_DEVFS:
+				if (run_command(j, &plimit, IP_MOUNT_FSTAB))
+					continue;
+				/* FALLTHROUGH */
+			case IP_MOUNT_FSTAB:
+				if (run_command(j, &plimit, IP_MOUNT))
+					continue;
+				/* FALLTHROUGH */
+			case IP_MOUNT:
+				if (run_command(j, &plimit, IP__IP4_IFADDR))
+					continue;
+				/* FALLTHROUGH */
+			case IP__IP4_IFADDR:
 #ifdef INET6
-			fprintf(fp, "%d\t%s\t%s\t%s%s%s\t%s\n",
-			    jid, i < nparams
-			    ? (char *)params[i].jp_value : argv[0],
-			    argv[1], ip4_addr ? ip4_addr : "",
-			    ip4_addr && ip4_addr[0] && ip6_addr && ip6_addr[0]
-			    ? "," : "", ip6_addr ? ip6_addr : "", argv[3]);
-#else
-			fprintf(fp, "%d\t%s\t%s\t%s\t%s\n",
-			    jid, i < nparams
-			    ? (char *)params[i].jp_value : argv[0],
-			    argv[1], ip4_addr ? ip4_addr : "", argv[3]);
+				if (run_command(j, &plimit, IP__IP6_IFADDR))
+					continue;
+				/* FALLTHROUGH */
+			case IP__IP6_IFADDR:
 #endif
+			jail_remove_done:
+				dep_done(j, 0);
+				if (j->flags & JF_START) {
+					j->comparam = 0;
+					j->flags &= ~JF_STOP;
+					dep_reset(j);
+					requeue(j,
+					    j->ndeps ? &waiting : &ready);
+				}
+			}
+			break;
+		}
+	}
+
+	if (jfp != NULL)
+		fclose(jfp);
+	exit(error);
+}
+
+/*
+ * Mark a jail's failure for future handling.
+ */
+void
+failed(struct cfjail *j)
+{
+	j->flags |= JF_FAILED;
+	TAILQ_REMOVE(j->queue, j, tq);
+	TAILQ_INSERT_HEAD(&ready, j, tq);
+	j->queue = &ready;
+}
+
+/*
+ * Exit slightly more gracefully when out of memory.
+ */
+void *
+emalloc(size_t size)
+{
+	void *p;
+
+	p = malloc(size);
+	if (!p)
+		err(1, "malloc");
+	return p;
+}
+
+void *
+erealloc(void *ptr, size_t size)
+{
+	void *p;
+
+	p = realloc(ptr, size);
+	if (!p)
+		err(1, "malloc");
+	return p;
+}
+
+char *
+estrdup(const char *str)
+{
+	char *ns;
+
+	ns = strdup(str);
+	if (!ns)
+		err(1, "malloc");
+	return ns;
+}
+
+/*
+ * Print a message including an optional jail name.
+ */
+void
+jail_note(const struct cfjail *j, const char *fmt, ...)
+{
+	va_list ap, tap;
+	char *cs;
+	size_t len;
+
+	va_start(ap, fmt);
+	va_copy(tap, ap);
+	len = vsnprintf(NULL, 0, fmt, tap);
+	va_end(tap);
+	cs = alloca(len + 1);
+	(void)vsnprintf(cs, len + 1, fmt, ap);
+	va_end(ap);
+	if (j->name)
+		printf("%s: %s", j->name, cs);
+	else
+		printf("%s", cs);
+}
+
+/*
+ * Print a warning message including an optional jail name.
+ */
+void
+jail_warnx(const struct cfjail *j, const char *fmt, ...)
+{
+	va_list ap, tap;
+	char *cs;
+	size_t len;
+
+	va_start(ap, fmt);
+	va_copy(tap, ap);
+	len = vsnprintf(NULL, 0, fmt, tap);
+	va_end(tap);
+	cs = alloca(len + 1);
+	(void)vsnprintf(cs, len + 1, fmt, ap);
+	va_end(ap);
+	if (j->name)
+		warnx("%s: %s", j->name, cs);
+	else
+		warnx("%s", cs);
+}
+
+/*
+ * Create a new jail.
+ */
+static int
+create_jail(struct cfjail *j)
+{
+	struct iovec jiov[4];
+	struct stat st;
+	struct jailparam *jp, *setparams, *setparams2, *sjp;
+	const char *path;
+	int dopersist, ns, jid, dying, didfail;
+
+	/*
+	 * Check the jail's path, with a better error message than jail_set
+	 * gives.
+	 */
+	if ((path = string_param(j->intparams[KP_PATH]))) {
+		if (stat(path, &st) < 0) {
+			jail_warnx(j, "path %s: %s", path, strerror(errno));
+			failed(j);
+			return -1;
+		}
+		if (!S_ISDIR(st.st_mode)) {
+			jail_warnx(j, "path %s: %s", path, strerror(ENOTDIR));
+			failed(j);
+			return -1;
 		}
-		(void)fclose(fp);
 	}
-	if (cmdarg < 0)
-		exit(0);
-	if (username != NULL) {
-		if (Uflag)
-			GET_USER_INFO;
-		if (lflag) {
-			p = getenv("TERM");
-			environ = &cleanenv;
+
+	/*
+	 * Copy all the parameters, except that "persist" is always set when
+	 * there are commands to run later.
+	 */
+	dopersist = !bool_param(j->intparams[KP_PERSIST]) &&
+	    (j->intparams[IP_EXEC_START] || j->intparams[IP_COMMAND] ||
+	     j->intparams[IP_EXEC_POSTSTART]);
+	sjp = setparams =
+	    alloca((j->njp + dopersist) * sizeof(struct jailparam));
+	if (dopersist && jailparam_init(sjp++, "persist") < 0) {
+		jail_warnx(j, "%s", jail_errmsg);
+		failed(j);
+		return -1;
+	}
+	for (jp = j->jp; jp < j->jp + j->njp; jp++)
+		if (!dopersist || !equalopts(jp->jp_name, "persist"))
+			*sjp++ = *jp;
+	ns = sjp - setparams;
+
+	didfail = 0;
+	j->jid = jailparam_set_note(j, setparams, ns, JAIL_CREATE);
+	if (j->jid < 0 && errno == EEXIST &&
+	    bool_param(j->intparams[IP_ALLOW_DYING]) &&
+	    int_param(j->intparams[KP_JID], &jid) && jid != 0) {
+		/*
+		 * The jail already exists, but may be dying.
+		 * Make sure it is, in which case an update is appropriate.
+		 */
+		*(const void **)&jiov[0].iov_base = "jid";
+		jiov[0].iov_len = sizeof("jid");
+		jiov[1].iov_base = &jid;
+		jiov[1].iov_len = sizeof(jid);
+		*(const void **)&jiov[2].iov_base = "dying";
+		jiov[2].iov_len = sizeof("dying");
+		jiov[3].iov_base = &dying;
+		jiov[3].iov_len = sizeof(dying);
+		if (jail_get(jiov, 4, JAIL_DYING) < 0) {
+			/*
+			 * It could be that the jail just barely finished
+			 * dying, or it could be that the jid never existed
+			 * but the name does.  In either case, another try
+			 * at creating the jail should do the right thing.
+			 */
+			if (errno == ENOENT)
+				j->jid = jailparam_set_note(j, setparams, ns,
+				    JAIL_CREATE);
+		} else if (dying) {
+			j->jid = jid;
+			if (rdtun_params(j, 1) < 0) {
+				j->jid = -1;
+				didfail = 1;
+			} else {
+				sjp = setparams2 = alloca((j->njp + dopersist) *
+				    sizeof(struct jailparam));
+				for (jp = setparams; jp < setparams + ns; jp++)
+					if (!JP_RDTUN(jp) ||
+					    !strcmp(jp->jp_name, "jid"))
+						*sjp++ = *jp;
+				j->jid = jailparam_set_note(j, setparams2,
+				    sjp - setparams2, JAIL_UPDATE | JAIL_DYING);
+				/*
+				 * Again, perhaps the jail just finished dying.
+				 */
+				if (j->jid < 0 && errno == ENOENT)
+					j->jid = jailparam_set_note(j,
+					    setparams, ns, JAIL_CREATE);
+			}
 		}
-		if (setgroups(ngroups, groups) != 0)
-			err(1, "setgroups");
-		if (setgid(pwd->pw_gid) != 0)
-			err(1, "setgid");
-		if (setusercontext(lcap, pwd, pwd->pw_uid,
-		    LOGIN_SETALL & ~LOGIN_SETGROUP & ~LOGIN_SETLOGIN) != 0)
-			err(1, "setusercontext");
-		login_close(lcap);
 	}
-	if (lflag) {
-		if (*pwd->pw_shell)
-			shell = pwd->pw_shell;
-		else
-			shell = _PATH_BSHELL;
-		if (chdir(pwd->pw_dir) < 0)
-			errx(1, "no home directory");
-		setenv("HOME", pwd->pw_dir, 1);
-		setenv("SHELL", shell, 1);
-		setenv("USER", pwd->pw_name, 1);
-		if (p)
-			setenv("TERM", p, 1);
+	if (j->jid < 0 && !didfail) {
+		jail_warnx(j, "%s", jail_errmsg);
+		failed(j);
+	}
+	if (dopersist) {
+		jailparam_free(setparams, 1);
+		if (j->jid > 0)
+			j->flags |= JF_PERSIST;
 	}
-	execvp(argv[cmdarg], argv + cmdarg);
-	err(1, "execvp: %s", argv[cmdarg]);
+	return j->jid;
 }
 
+/*
+ * Remove a temporarily set "persist" parameter.
+ */
 static void
-add_ip_addr(char **addrp, char *value)
+clear_persist(struct cfjail *j)
 {
-	int addrlen;
-	char *addr;
-
-	if (!*addrp) {
-		*addrp = strdup(value);
-		if (!*addrp)
-			err(1, "malloc");
-	} else if (value[0]) {
-		addrlen = strlen(*addrp) + strlen(value) + 2;
-		addr = malloc(addrlen);
-		if (!addr)
-			err(1, "malloc");
-		snprintf(addr, addrlen, "%s,%s", *addrp, value);
-		free(*addrp);
-		*addrp = addr;
+	struct iovec jiov[4];
+	int jid;
+
+	if (!(j->flags & JF_PERSIST))
+		return;
+	j->flags &= ~JF_PERSIST;
+	*(const void **)&jiov[0].iov_base = "jid";
+	jiov[0].iov_len = sizeof("jid");
+	jiov[1].iov_base = &j->jid;
+	jiov[1].iov_len = sizeof(j->jid);
+	*(const void **)&jiov[2].iov_base = "nopersist";
+	jiov[2].iov_len = sizeof("nopersist");
+	jiov[3].iov_base = NULL;
+	jiov[3].iov_len = 0;
+	jid = jail_set(jiov, 4, JAIL_UPDATE);
+	if (verbose > 0)
+		jail_note(j, "jail_set(JAIL_UPDATE) jid=%d nopersist%s%s\n",
+		    j->jid, jid < 0 ? ": " : "",
+		    jid < 0 ? strerror(errno) : "");
+}
+
+/*
+ * Set a jail's parameters.
+ */
+static int
+update_jail(struct cfjail *j)
+{
+	struct jailparam *jp, *setparams, *sjp;
+	int ns, jid;
+
+	ns = 0;
+	for (jp = j->jp; jp < j->jp + j->njp; jp++)
+		if (!JP_RDTUN(jp))
+			ns++;
+	if (ns == 0)
+		return 0;
+	sjp = setparams = alloca(++ns * sizeof(struct jailparam));
+	if (jailparam_init(sjp, "jid") < 0 ||
+	    jailparam_import_raw(sjp, &j->jid, sizeof j->jid) < 0) {
+		jail_warnx(j, "%s", jail_errmsg);
+		failed(j);
+		return -1;
 	}
+	for (jp = j->jp; jp < j->jp + j->njp; jp++)
+		if (!JP_RDTUN(jp))
+			*++sjp = *jp;
+
+	jid = jailparam_set_note(j, setparams, ns,
+	    bool_param(j->intparams[IP_ALLOW_DYING])
+	    ? JAIL_UPDATE | JAIL_DYING : JAIL_UPDATE);
+	if (jid < 0) {
+		jail_warnx(j, "%s", jail_errmsg);
+		failed(j);
+	}
+	jailparam_free(setparams, 1);
+	return jid;
 }
 
-#ifdef INET6
+/*
+ * Return if a jail set would change any create-only parameters.
+ */
+static int
+rdtun_params(struct cfjail *j, int dofail)
+{
+	struct jailparam *jp, *rtparams, *rtjp;
+	int nrt, rval;
+
+	if (j->flags & JF_RDTUN)
+		return 0;
+	j->flags |= JF_RDTUN;
+	nrt = 0;
+	for (jp = j->jp; jp < j->jp + j->njp; jp++)
+		if (JP_RDTUN(jp) && strcmp(jp->jp_name, "jid"))
+			nrt++;
+	if (nrt == 0)
+		return 0;
+	rtjp = rtparams = alloca(++nrt * sizeof(struct jailparam));
+	if (jailparam_init(rtjp, "jid") < 0 ||
+	    jailparam_import_raw(rtjp, &j->jid, sizeof j->jid) < 0) {
+		jail_warnx(j, "%s", jail_errmsg);
+		exit(1);
+	}
+	for (jp = j->jp; jp < j->jp + j->njp; jp++)
+		if (JP_RDTUN(jp) && strcmp(jp->jp_name, "jid"))
+			*++rtjp = *jp;
+	rval = 0;
+	if (jailparam_get(rtparams, nrt,
+	    bool_param(j->intparams[IP_ALLOW_DYING]) ? JAIL_DYING : 0) > 0) {
+		rtjp = rtparams + 1;
+		for (jp = j->jp, rtjp = rtparams + 1; rtjp < rtparams + nrt;
+		     jp++) {
+			if (JP_RDTUN(jp) && strcmp(jp->jp_name, "jid")) {
+				if (!((jp->jp_flags & (JP_BOOL | JP_NOBOOL)) &&
+				    jp->jp_valuelen == 0 &&
+				    *(int *)jp->jp_value) &&
+				    !(rtjp->jp_valuelen == jp->jp_valuelen &&
+				    !memcmp(rtjp->jp_value, jp->jp_value,
+				    jp->jp_valuelen))) {
+					if (dofail) {
+						jail_warnx(j, "%s cannot be "
+						    "changed after creation",
+						    jp->jp_name);
+						failed(j);
+						rval = -1;
+					} else
+						rval = 1;
+					break;
+				}
+				rtjp++;
+			}
+		}
+	}
+	for (rtjp = rtparams + 1; rtjp < rtparams + nrt; rtjp++)
+		rtjp->jp_name = NULL;
+	jailparam_free(rtparams, nrt);
+	return rval;
+}
+
+/*
+ * Get the jail's jid if it is running.
+ */
 static void
-add_ip_addr46(char *value)
+running_jid(struct cfjail *j, int dflag)
 {
-	char *p, *np;
+	struct iovec jiov[2];
+	const char *pval;
+	char *ep;
+	int jid;
 
-	for (p = value;; p = np + 1)
-	{
-		np = strchr(p, ',');
-		if (np)
-			*np = '\0';
-		add_ip_addrinfo(AI_NUMERICHOST, p);
-		if (!np)
-			break;
+	pval = string_param(j->intparams[KP_JID]);
+	if (pval != NULL) {
+		if (!(jid = strtol(pval, &ep, 10)) || *ep) {
+			j->jid = -1;
+			return;
+		}
+		*(const void **)&jiov[0].iov_base = "jid";
+		jiov[0].iov_len = sizeof("jid");
+		jiov[1].iov_base = &jid;
+		jiov[1].iov_len = sizeof(jid);
+	} else {
+		pval = string_param(j->intparams[KP_NAME]);
+		*(const void **)&jiov[0].iov_base = "name";
+		jiov[0].iov_len = sizeof("name");
+		jiov[1].iov_len = strlen(pval) + 1;
+		jiov[1].iov_base = alloca(jiov[1].iov_len);
+		strcpy(jiov[1].iov_base, pval);
 	}
+	j->jid = jail_get(jiov, 2, dflag ? JAIL_DYING : 0);
 }
-#endif
 
+/*
+ * Set jail parameters and possible print them out.
+ */
+static int
+jailparam_set_note(const struct cfjail *j, struct jailparam *jp, unsigned njp,
+    int flags)
+{
+	char *value;
+	int jid;
+	unsigned i;
+
+	jid = jailparam_set(jp, njp, flags);
+	if (verbose > 0) {
+		jail_note(j, "jail_set(%s%s)",
+		    (flags & (JAIL_CREATE | JAIL_UPDATE)) == JAIL_CREATE
+		    ? "JAIL_CREATE" : "JAIL_UPDATE",
+		    (flags & JAIL_DYING) ? " | JAIL_DYING" : "");
+		for (i = 0; i < njp; i++) {
+			printf(" %s", jp[i].jp_name);
+			if (jp[i].jp_value == NULL)
+				continue;
+			putchar('=');
+			value = jailparam_export(jp + i);
+			if (value == NULL)
+				err(1, "jailparam_export");
+			quoted_print(stdout, value);
+			free(value);
+		}
+		if (jid < 0)
+			printf(": %s", strerror(errno));
+		printf("\n");
+	}
+	return jid;
+}
+
+/*
+ * Print a jail record.
+ */
 static void
-add_ip_addrinfo(int ai_flags, char *value)
+print_jail(FILE *fp, struct cfjail *j, int oldcl)
 {
-	struct addrinfo hints, *ai0, *ai;
-	struct in_addr addr4;
-	size_t size;
-	int error, ip4ok;
-	int mib[4];
-	char avalue4[INET_ADDRSTRLEN];
-#ifdef INET6
-	struct in6_addr addr6;
-	int ip6ok;
-	char avalue6[INET6_ADDRSTRLEN];
-#endif
+	struct cfparam *p;
 
-	/* Look up the hostname (or get the address) */
-	memset(&hints, 0, sizeof(hints));
-	hints.ai_socktype = SOCK_STREAM;
+	if (oldcl) {
+		fprintf(fp, "%d\t", j->jid);
+		print_param(fp, j->intparams[KP_PATH], ',', 0);
+		putc('\t', fp);
+		print_param(fp, j->intparams[KP_HOSTNAME], ',', 0);
+		putc('\t', fp);
+		print_param(fp, j->intparams[KP_IP4_ADDR], ',', 0);
 #ifdef INET6
-	hints.ai_family = PF_UNSPEC;
-#else
-	hints.ai_family = PF_INET;
+		if (j->intparams[KP_IP6_ADDR] &&
+		    !STAILQ_EMPTY(&j->intparams[KP_IP6_ADDR]->val)) {
+			if (j->intparams[KP_IP4_ADDR] &&
+			    !STAILQ_EMPTY(&j->intparams[KP_IP4_ADDR]->val))
+				putc(',', fp);
+			print_param(fp, j->intparams[KP_IP6_ADDR], ',', 0);
+		}
 #endif
-	hints.ai_flags = ai_flags;
-	error = getaddrinfo(value, NULL, &hints, &ai0);
-	if (error != 0)
-		errx(1, "hostname %s: %s", value, gai_strerror(error));
+		putc('\t', fp);
+		print_param(fp, j->intparams[IP_COMMAND], ' ', 0);
+	} else {
+		fprintf(fp, "jid=%d", j->jid);
+		TAILQ_FOREACH(p, &j->params, tq)
+			if (strcmp(p->name, "jid")) {
+				putc(' ', fp);
+				print_param(fp, p, ',', 1);
+			}
+	}
+	putc('\n', fp);
+}
 
-	/*
-	 * Silently ignore unsupported address families from DNS lookups.
-	 * But if this is a numeric address, let the kernel give the error.
-	 */
-	if (ai_flags & AI_NUMERICHOST)
-		ip4ok =
-#ifdef INET6
-		    ip6ok =
-#endif
-		    1;
-	else {
-		size = 4;
-		ip4ok = (sysctlnametomib("security.jail.param.ip4", mib,
-		    &size) == 0);
-#ifdef INET6
-		size = 4;
-		ip6ok = (sysctlnametomib("security.jail.param.ip6", mib,
-		    &size) == 0);
-#endif
+/*
+ * Print a parameter value, or a name=value pair.
+ */
+static void
+print_param(FILE *fp, const struct cfparam *p, int sep, int doname)
+{
+	const struct cfstring *s, *ts;
+
+	if (doname)
+		fputs(p->name, fp);
+	if (p == NULL || STAILQ_EMPTY(&p->val))
+		return;
+	if (doname)
+		putc('=', fp);
+	STAILQ_FOREACH_SAFE(s, &p->val, tq, ts) {
+		quoted_print(fp, s->s);
+		if (ts != NULL)
+			putc(sep, fp);
 	}
-	
-	/* Convert the addresses to ASCII so set_param can convert them back. */
-	for (ai = ai0; ai; ai = ai->ai_next)
-		switch (ai->ai_family) {
-		case AF_INET:
-			if (!ip4ok)
-				break;
-			memcpy(&addr4, &((struct sockaddr_in *)
-			    (void *)ai->ai_addr)->sin_addr, sizeof(addr4));
-			if (inet_ntop(AF_INET, &addr4, avalue4,
-			    INET_ADDRSTRLEN) == NULL)
-				err(1, "inet_ntop");
-			add_ip_addr(&ip4_addr, avalue4);
-			break;
-#ifdef INET6
-		case AF_INET6:
-			if (!ip6ok)
-				break;
-			memcpy(&addr6, &((struct sockaddr_in6 *)
-			    (void *)ai->ai_addr)->sin6_addr, sizeof(addr6));
-			if (inet_ntop(AF_INET6, &addr6, avalue6,
-			    INET6_ADDRSTRLEN) == NULL)
-				err(1, "inet_ntop");
-			add_ip_addr(&ip6_addr, avalue6);
-			break;
-#endif
-		}
-	freeaddrinfo(ai0);
 }
 
+/*
+ * Print a string with quotes around spaces.
+ */
 static void
 quoted_print(FILE *fp, char *str)
 {
 	int c, qc;
 	char *p = str;
 
-	/* An empty string needs quoting. */
-	if (!*p) {
-		fputs("\"\"", fp);
-		return;
-	}
-
-	/*
-	 * The value will be surrounded by quotes if it contains spaces
-	 * or quotes.
-	 */
-	qc = strchr(p, '\'') ? '"'
+	qc = !*p ? '"'
+	    : strchr(p, '\'') ? '"'
 	    : strchr(p, '"') ? '\''
 	    : strchr(p, ' ') || strchr(p, '\t') ? '"'
 	    : 0;
@@ -497,65 +1031,16 @@ quoted_print(FILE *fp, char *str)
 }
 
 static void
-set_param(const char *name, char *value)
-{
-	struct jailparam *param;
-	int i;
-
-	static int paramlistsize;
-
-	/* Separate the name from the value, if not done already. */
-	if (name == NULL) {
-		name = value;
-		if ((value = strchr(value, '=')))
-			*value++ = '\0';
-	}
-
-	/* jail_set won't chdir along with its chroot, so do it here. */
-	if (!strcmp(name, "path") && chdir(value) < 0)
-		err(1, "chdir: %s", value);
-
-	/* Check for repeat parameters */
-	for (i = 0; i < nparams; i++)
-		if (!strcmp(name, params[i].jp_name)) {
-			jailparam_free(params + i, 1);
-			memcpy(params + i, params + i + 1,
-			    (--nparams - i) * sizeof(struct jailparam));
-			break;
-		}
-
-	/* Make sure there is room for the new param record. */
-	if (!nparams) {
-		paramlistsize = 32;
-		params = malloc(paramlistsize * sizeof(*params));
-		param_values = malloc(paramlistsize * sizeof(*param_values));
-		if (params == NULL || param_values == NULL)
-			err(1, "malloc");
-	} else if (nparams >= paramlistsize) {
-		paramlistsize *= 2;
-		params = realloc(params, paramlistsize * sizeof(*params));
-		param_values = realloc(param_values,
-		    paramlistsize * sizeof(*param_values));
-		if (params == NULL)
-			err(1, "realloc");
-	}
-
-	/* Look up the paramter. */
-	param_values[nparams] = value;
-	param = params + nparams++;
-	if (jailparam_init(param, name) < 0 ||
-	    jailparam_import(param, value) < 0)
-		errx(1, "%s", jail_errmsg);
-}
-
-static void
 usage(void)
 {
 
 	(void)fprintf(stderr,
-	    "usage: jail [-d] [-h] [-i] [-J jid_file] "
-			"[-l -u username | -U username]\n"
-	    "            [-c | -m] param=value ... [command=command ...]\n"
-	    "       jail [-r jail]\n");
+	    "usage: jail [-dhilqv] [-J jid_file] [-u username] [-U username]\n"
+	    "            -[cmr] param=value ... [command=command ...]\n"
+	    "       jail [-dqv] [-f file] -[cmr] [jail]\n"
+	    "       jail [-qv] [-f file] -[rR] ['*' | jail ...]\n"
+	    "       jail [-dhilqv] [-J jid_file] [-u username] [-U username]\n"
+	    "            [-n jailname] [-s securelevel]\n"
+	    "            path hostname [ip[,...]] command ...\n");
 	exit(1);
 }