Make CAM use root_mount_hold_token() to delay boot.

Before this change CAM used config_intrhook_establish() for this purpose,
but that approach does not allow to delay it again after releasing once.

USB stack uses root_mount_hold() to delay boot until bus scan is complete.
But once it is, CAM had no time to scan SCSI bus, registered by umass(4),
if it already done other scans and called config_intrhook_disestablish().
The new approach makes it work smooth, assuming the USB device is found
during the initial bus scan.  Devices appearing on USB bus later may still
require setting kern.cam.boot_delay, but hopefully those are minority.

MFC after:	2 weeks
Sponsored by:	iXsystems, Inc.

1 files changed, 52 insertions, 40 deletions

diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index 4736c7212a2c..f879979bd8a0 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -99,13 +99,6 @@ MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
 
-/* Object for defering XPT actions to a taskqueue */
-struct xpt_task {
-	struct task	task;
-	void		*data1;
-	uintptr_t	data2;
-};
-
 struct xpt_softc {
 	uint32_t		xpt_generation;
 
@@ -129,10 +122,10 @@ struct xpt_softc {
 	TAILQ_HEAD(,cam_eb)	xpt_busses;
 	u_int			bus_generation;
 
-	struct intr_config_hook	xpt_config_hook;
-
 	int			boot_delay;
 	struct callout 		boot_callout;
+	struct task		boot_task;
+	struct root_hold_token	xpt_rootmount;
 
 	struct mtx		xpt_topo_lock;
 	struct mtx		xpt_lock;
@@ -273,6 +266,7 @@ static struct cam_et*
 static struct cam_ed*
 		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
 static void	 xpt_config(void *arg);
+static void	 xpt_hold_boot_locked(void);
 static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 				 u_int32_t new_priority);
 static xpt_devicefunc_t xptpassannouncefunc;
@@ -881,7 +875,7 @@ xpt_rescan(union ccb *ccb)
 		}
 	}
 	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
-	xsoftc.buses_to_config++;
+	xpt_hold_boot_locked();
 	wakeup(&xsoftc.ccb_scanq);
 	xpt_unlock_buses();
 }
@@ -913,6 +907,7 @@ xpt_init(void *dummy)
 	 */
 	xsoftc.boot_delay = CAM_BOOT_DELAY;
 #endif
+
 	/*
 	 * The xpt layer is, itself, the equivalent of a SIM.
 	 * Allow 16 ccbs in the ccb pool for it.  This should
@@ -979,14 +974,11 @@ xpt_init(void *dummy)
 		       "- failing attach\n");
 		return (ENOMEM);
 	}
+
 	/*
 	 * Register a callback for when interrupts are enabled.
 	 */
-	xsoftc.xpt_config_hook.ich_func = xpt_config;
-	if (config_intrhook_establish(&xsoftc.xpt_config_hook) != 0) {
-		printf("xpt_init: config_intrhook_establish failed "
-		       "- failing attach\n");
-	}
+	config_intrhook_oneshot(xpt_config, NULL);
 
 	return (0);
 }
@@ -5148,6 +5140,10 @@ xpt_stop_tags(struct cam_path *path)
 	xpt_action((union ccb *)&crs);
 }
 
+/*
+ * Assume all possible buses are detected by this time, so allow boot
+ * as soon as they all are scanned.
+ */
 static void
 xpt_boot_delay(void *arg)
 {
@@ -5155,12 +5151,26 @@ xpt_boot_delay(void *arg)
 	xpt_release_boot();
 }
 
+/*
+ * Now that all config hooks have completed, start boot_delay timer,
+ * waiting for possibly still undetected buses (USB) to appear.
+ */
+static void
+xpt_ch_done(void *arg)
+{
+
+	callout_init(&xsoftc.boot_callout, 1);
+	callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
+	    xpt_boot_delay, NULL, 0);
+}
+SYSINIT(xpt_hw_delay, SI_SUB_INT_CONFIG_HOOKS, SI_ORDER_ANY, xpt_ch_done, NULL);
+
+/*
+ * Now that interrupts are enabled, go find our devices
+ */
 static void
 xpt_config(void *arg)
 {
-	/*
-	 * Now that interrupts are enabled, go find our devices
-	 */
 	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
 		printf("xpt_config: failed to create taskqueue thread.\n");
 
@@ -5179,9 +5189,7 @@ xpt_config(void *arg)
 
 	periphdriver_init(1);
 	xpt_hold_boot();
-	callout_init(&xsoftc.boot_callout, 1);
-	callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
-	    xpt_boot_delay, NULL, 0);
+
 	/* Fire up rescan thread. */
 	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
 	    "cam", "scanner")) {
@@ -5190,31 +5198,38 @@ xpt_config(void *arg)
 }
 
 void
+xpt_hold_boot_locked(void)
+{
+
+	if (xsoftc.buses_to_config++ == 0)
+		root_mount_hold_token("CAM", &xsoftc.xpt_rootmount);
+}
+
+void
 xpt_hold_boot(void)
 {
+
 	xpt_lock_buses();
-	xsoftc.buses_to_config++;
+	xpt_hold_boot_locked();
 	xpt_unlock_buses();
 }
 
 void
 xpt_release_boot(void)
 {
-	xpt_lock_buses();
-	xsoftc.buses_to_config--;
-	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
-		struct	xpt_task *task;
 
-		xsoftc.buses_config_done = 1;
-		xpt_unlock_buses();
-		/* Call manually because we don't have any buses */
-		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
-		if (task != NULL) {
-			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
-			taskqueue_enqueue(taskqueue_thread, &task->task);
-		}
-	} else
-		xpt_unlock_buses();
+	xpt_lock_buses();
+	if (--xsoftc.buses_to_config == 0) {
+		if (xsoftc.buses_config_done == 0) {
+			xsoftc.buses_config_done = 1;
+			xsoftc.buses_to_config++;
+			TASK_INIT(&xsoftc.boot_task, 0, xpt_finishconfig_task,
+			    NULL);
+			taskqueue_enqueue(taskqueue_thread, &xsoftc.boot_task);
+		} else
+			root_mount_rel(&xsoftc.xpt_rootmount);
+	}
+	xpt_unlock_buses();
 }
 
 /*
@@ -5252,10 +5267,7 @@ xpt_finishconfig_task(void *context, int pending)
 	if (!bootverbose)
 		xpt_for_all_devices(xptpassannouncefunc, NULL);
 
-	/* Release our hook so that the boot can continue. */
-	config_intrhook_disestablish(&xsoftc.xpt_config_hook);
-
-	free(context, M_CAMXPT);
+	xpt_release_boot();
 }
 
 cam_status