After r290196, the kernel won't wait for stuff like gmirror nodes

if they are not required for mounting rootfs.  However, it's possible
that some setups try to mount them in mountcritlocal (ie from fstab).

Export the list of current root mount holds using a new sysctl,
vfs.root_mount_hold, and make mountcritlocal retry if "mount -a" fails
and the list is not empty.

MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3709

3 files changed, 67 insertions, 2 deletions

diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf
index fbe18db386af..4c0e899f268c 100644
--- a/etc/defaults/rc.conf
+++ b/etc/defaults/rc.conf
@@ -84,6 +84,7 @@ geli_autodetach="YES"	# Automatically detach on last close.
 #geli_mirror_home_flags="-k /etc/geli/home.keys"
 
 root_rw_mount="YES"	# Set to NO to inhibit remounting root read-write.
+root_hold_delay="30"	# Time to wait for root mount hold release.
 fsck_y_enable="NO"	# Set to YES to do fsck -y if the initial preen fails.
 fsck_y_flags=""		# Additional flags for fsck -y
 background_fsck="YES"	# Attempt to run fsck in the background where possible.
diff --git a/etc/rc.d/mountcritlocal b/etc/rc.d/mountcritlocal
index 06bf464f239f..1513ec7397ac 100755
--- a/etc/rc.d/mountcritlocal
+++ b/etc/rc.d/mountcritlocal
@@ -15,7 +15,7 @@ stop_cmd=sync
 
 mountcritlocal_start()
 {
-	local err
+	local err holders waited
 
 	# Set up the list of network filesystem types for which mounting
 	# should be delayed until after network initialization.
@@ -35,8 +35,42 @@ mountcritlocal_start()
 		mount_excludes="${mount_excludes}${fstype},"
 	done
 	mount_excludes=${mount_excludes%,}
+
+	# Originally, root mount hold had to be released before mounting the root
+	# filesystem.  This delayed the boot, so it was changed to only wait if
+	# the root device isn't readily available.  This can result in this script
+	# executing before all the devices - such as graid(8) - are available.
+	# Thus, should the mount fail, we will wait for the root mount hold release
+	# and retry.
 	mount -a -t ${mount_excludes}
 	err=$?
+	if [ $? -ne 0 ]; then
+		echo
+		echo 'Mounting /etc/fstab filesystems failed,' \
+		    'will retry after root mount hold release'
+
+		waited=0
+		while [ ${waited} -lt ${root_hold_delay} ]; do
+			holders="$(sysctl -n vfs.root_mount_hold)"
+			if [ -z "${holders}" ]; then
+				break;
+			fi
+			if [ ${waited} -eq 0 ]; then
+				echo -n "Waiting ${root_hold_delay}s" \
+				"for the root mount holders: ${holders}"
+			else
+				echo -n .
+			fi
+			if [ ${waited} -eq ${root_hold_delay} ]; then
+				break 2
+			fi
+			sleep 1
+			waited=$(($waited + 1))
+		done
+		mount -a -t ${mount_excludes}
+		err=$?
+	fi
+
 	check_startmsgs && echo '.'
 
 	case ${err} in
@@ -44,7 +78,7 @@ mountcritlocal_start()
 		;;
 	*)
 		echo 'Mounting /etc/fstab filesystems failed,' \
-		    ' startup aborted'
+		    'startup aborted'
 		stop_boot true
 		;;
 	esac
diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c
index 2691a6ee1ef9..cf2425306dd6 100644
--- a/sys/kern/vfs_mountroot.c
+++ b/sys/kern/vfs_mountroot.c
@@ -88,6 +88,7 @@ __FBSDID("$FreeBSD$");
 
 static int parse_mount(char **);
 static struct mntarg *parse_mountroot_options(struct mntarg *, const char *);
+static int sysctl_vfs_root_mount_hold(SYSCTL_HANDLER_ARGS);
 static int vfs_mountroot_wait_if_neccessary(const char *fs, const char *dev);
 
 /*
@@ -130,6 +131,35 @@ static int root_mount_complete;
 static int root_mount_timeout = 3;
 TUNABLE_INT("vfs.mountroot.timeout", &root_mount_timeout);
 
+SYSCTL_PROC(_vfs, OID_AUTO, root_mount_hold,
+    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
+    NULL, 0, sysctl_vfs_root_mount_hold, "A",
+    "List of root mount hold tokens");
+
+static int
+sysctl_vfs_root_mount_hold(SYSCTL_HANDLER_ARGS)
+{
+	struct sbuf sb;
+	struct root_hold_token *h;
+	int error;
+
+	sbuf_new(&sb, NULL, 256, SBUF_AUTOEXTEND | SBUF_INCLUDENUL);
+
+	mtx_lock(&root_holds_mtx);
+	LIST_FOREACH(h, &root_holds, list) {
+		if (h != LIST_FIRST(&root_holds))
+			sbuf_putc(&sb, ' ');
+		sbuf_printf(&sb, "%s", h->who);
+	}
+	mtx_unlock(&root_holds_mtx);
+
+	error = sbuf_finish(&sb);
+	if (error == 0)
+		error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
+	sbuf_delete(&sb);
+	return (error);
+}
+
 struct root_hold_token *
 root_mount_hold(const char *identifier)
 {