MFV r274272 and diff reduction with upstream.

Illumos issue:
    5244 zio pipeline callers should explicitly invoke next stage

Tested with:	ztest plus ZFS over GELI configuration
MFC after:	1 month

9 files changed, 59 insertions, 50 deletions

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev_impl.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev_impl.h
index 642a4c05d64e..e7515d5ae773 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev_impl.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev_impl.h
@@ -60,7 +60,7 @@ typedef int	vdev_open_func_t(vdev_t *vd, uint64_t *size, uint64_t *max_size,
     uint64_t *logical_ashift, uint64_t *physical_ashift);
 typedef void	vdev_close_func_t(vdev_t *vd);
 typedef uint64_t vdev_asize_func_t(vdev_t *vd, uint64_t psize);
-typedef int	vdev_io_start_func_t(zio_t *zio);
+typedef void	vdev_io_start_func_t(zio_t *zio);
 typedef void	vdev_io_done_func_t(zio_t *zio);
 typedef void	vdev_state_change_func_t(vdev_t *vd, int, int);
 typedef void	vdev_hold_func_t(vdev_t *vd);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
index df7edf4c1c65..17ca65b7520f 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
@@ -152,9 +152,6 @@ typedef enum zio_priority {
 	ZIO_PRIORITY_NOW		/* non-queued I/Os (e.g. ioctl) */
 } zio_priority_t;
 
-#define	ZIO_PIPELINE_CONTINUE		0x100
-#define	ZIO_PIPELINE_STOP		0x101
-
 enum zio_flag {
 	/*
 	 * Flags inherited by gang, ddt, and vdev children,
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_disk.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_disk.c
index 1caff2c554c6..eefa1dc9f01d 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_disk.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_disk.c
@@ -715,7 +715,7 @@ vdev_disk_ioctl_done(void *zio_arg, int error)
 	zio_interrupt(zio);
 }
 
-static int
+static void
 vdev_disk_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
@@ -732,7 +732,7 @@ vdev_disk_io_start(zio_t *zio)
 	if (dvd == NULL || (dvd->vd_ldi_offline && dvd->vd_lh == NULL)) {
 		zio->io_error = SET_ERROR(ENXIO);
 		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		return;
 	}
 
 	if (zio->io_type == ZIO_TYPE_IOCTL) {
@@ -740,7 +740,7 @@ vdev_disk_io_start(zio_t *zio)
 		if (!vdev_readable(vd)) {
 			zio->io_error = SET_ERROR(ENXIO);
 			zio_interrupt(zio);
-			return (ZIO_PIPELINE_STOP);
+			return;
 		}
 
 		switch (zio->io_cmd) {
@@ -771,7 +771,7 @@ vdev_disk_io_start(zio_t *zio)
 				 * and will call vdev_disk_ioctl_done()
 				 * upon completion.
 				 */
-				return (ZIO_PIPELINE_STOP);
+				return;
 			}
 
 			if (error == ENOTSUP || error == ENOTTY) {
@@ -792,8 +792,8 @@ vdev_disk_io_start(zio_t *zio)
 			zio->io_error = SET_ERROR(ENOTSUP);
 		}
 
-		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		zio_execute(zio);
+		return;
 	}
 
 	vb = kmem_alloc(sizeof (vdev_buf_t), KM_SLEEP);
@@ -814,8 +814,6 @@ vdev_disk_io_start(zio_t *zio)
 
 	/* ldi_strategy() will return non-zero only on programming errors */
 	VERIFY(ldi_strategy(dvd->vd_lh, bp) == 0);
-
-	return (ZIO_PIPELINE_STOP);
 }
 
 static void
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c
index ab492d80640a..3616273129c2 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -154,7 +154,7 @@ vdev_file_close(vdev_t *vd)
 	vd->vdev_tsd = NULL;
 }
 
-static int
+static void
 vdev_file_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
@@ -165,7 +165,7 @@ vdev_file_io_start(zio_t *zio)
 	if (!vdev_readable(vd)) {
 		zio->io_error = SET_ERROR(ENXIO);
 		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		return;
 	}
 
 	vf = vd->vdev_tsd;
@@ -181,8 +181,8 @@ vdev_file_io_start(zio_t *zio)
 			zio->io_error = SET_ERROR(ENOTSUP);
 		}
 
-		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		zio_execute(zio);
+		return;
 	}
 
 	zio->io_error = vn_rdwr(zio->io_type == ZIO_TYPE_READ ?
@@ -194,7 +194,10 @@ vdev_file_io_start(zio_t *zio)
 
 	zio_interrupt(zio);
 
-	return (ZIO_PIPELINE_STOP);
+#ifdef illumos
+	VERIFY3U(taskq_dispatch(system_taskq, vdev_file_io_strategy, bp,
+	    TQ_SLEEP), !=, 0);
+#endif
 }
 
 /* ARGSUSED */
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c
index b28125ef60b0..8e8b4272b298 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c
@@ -788,7 +788,7 @@ vdev_geom_io_intr(struct bio *bp)
 	zio_interrupt(zio);
 }
 
-static int
+static void
 vdev_geom_io_start(zio_t *zio)
 {
 	vdev_t *vd;
@@ -803,6 +803,8 @@ vdev_geom_io_start(zio_t *zio)
 		/* XXPOLICY */
 		if (!vdev_readable(vd)) {
 			zio->io_error = SET_ERROR(ENXIO);
+			zio_interrupt(zio);
+			return;
 		} else {
 			switch (zio->io_cmd) {
 			case DKIOCFLUSHWRITECACHE:
@@ -818,23 +820,23 @@ vdev_geom_io_start(zio_t *zio)
 			}
 		}
 
-		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		zio_execute(zio);
+		return;
 	case ZIO_TYPE_FREE:
 		if (vd->vdev_notrim) {
 			zio->io_error = SET_ERROR(ENOTSUP);
 		} else if (!vdev_geom_bio_delete_disable) {
 			goto sendreq;
 		}
-		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		zio_execute(zio);
+		return;
 	}
 sendreq:
 	cp = vd->vdev_tsd;
 	if (cp == NULL) {
 		zio->io_error = SET_ERROR(ENXIO);
 		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		return;
 	}
 	bp = g_alloc_bio();
 	bp->bio_caller1 = zio;
@@ -863,8 +865,6 @@ sendreq:
 	bp->bio_done = vdev_geom_io_intr;
 
 	g_io_request(bp, cp);
-
-	return (ZIO_PIPELINE_STOP);
 }
 
 static void
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c
index 2fefa609ac0c..60d221a08de8 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -425,7 +425,7 @@ vdev_mirror_child_select(zio_t *zio)
 	return (-1);
 }
 
-static int
+static void
 vdev_mirror_io_start(zio_t *zio)
 {
 	mirror_map_t *mm;
@@ -450,8 +450,8 @@ vdev_mirror_io_start(zio_t *zio)
 				    zio->io_type, zio->io_priority, 0,
 				    vdev_mirror_scrub_done, mc));
 			}
-			zio_interrupt(zio);
-			return (ZIO_PIPELINE_STOP);
+			zio_execute(zio);
+			return;
 		}
 		/*
 		 * For normal reads just pick one child.
@@ -478,8 +478,7 @@ vdev_mirror_io_start(zio_t *zio)
 		c++;
 	}
 
-	zio_interrupt(zio);
-	return (ZIO_PIPELINE_STOP);
+	zio_execute(zio);
 }
 
 static int
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c
index fec0037b2b1c..24a44a697769 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c
@@ -24,7 +24,7 @@
  */
 
 /*
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  */
 
 /*
@@ -67,12 +67,11 @@ vdev_missing_close(vdev_t *vd)
 }
 
 /* ARGSUSED */
-static int
+static void
 vdev_missing_io_start(zio_t *zio)
 {
 	zio->io_error = SET_ERROR(ENOTSUP);
-	zio_interrupt(zio);
-	return (ZIO_PIPELINE_STOP);
+	zio_execute(zio);
 }
 
 /* ARGSUSED */
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c
index 925aaea79f35..00045e1d528a 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
 
@@ -1726,7 +1726,7 @@ vdev_raidz_child_done(zio_t *zio)
  *      vdevs have had errors, then create zio read operations to the parity
  *      columns' VDevs as well.
  */
-static int
+static void
 vdev_raidz_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
@@ -1756,8 +1756,8 @@ vdev_raidz_io_start(zio_t *zio)
 			    vdev_raidz_child_done, rc));
 		}
 
-		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		zio_execute(zio);
+		return;
 	}
 
 	if (zio->io_type == ZIO_TYPE_WRITE) {
@@ -1789,8 +1789,8 @@ vdev_raidz_io_start(zio_t *zio)
 			    ZIO_FLAG_NODATA | ZIO_FLAG_OPTIONAL, NULL, NULL));
 		}
 
-		zio_interrupt(zio);
-		return (ZIO_PIPELINE_STOP);
+		zio_execute(zio);
+		return;
 	}
 
 	ASSERT(zio->io_type == ZIO_TYPE_READ);
@@ -1830,8 +1830,7 @@ vdev_raidz_io_start(zio_t *zio)
 		}
 	}
 
-	zio_interrupt(zio);
-	return (ZIO_PIPELINE_STOP);
+	zio_execute(zio);
 }
 
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
index 3586f8723bd7..45efb22f1a3f 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
@@ -90,6 +90,9 @@ kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
 extern vmem_t *zio_alloc_arena;
 #endif
 
+#define	ZIO_PIPELINE_CONTINUE		0x100
+#define	ZIO_PIPELINE_STOP		0x101
+
 /*
  * The following actions directly effect the spa's sync-to-convergence logic.
  * The values below define the sync pass when we start performing the action.
@@ -2557,6 +2560,18 @@ zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp)
  * Read, write and delete to physical devices
  * ==========================================================================
  */
+
+
+/*
+ * Issue an I/O to the underlying vdev. Typically the issue pipeline
+ * stops after this stage and will resume upon I/O completion.
+ * However, there are instances where the vdev layer may need to
+ * continue the pipeline when an I/O was not issued. Since the I/O
+ * that was sent to the vdev layer might be different than the one
+ * currently active in the pipeline (see vdev_queue_io()), we explicitly
+ * force the underlying vdev layers to call either zio_execute() or
+ * zio_interrupt() to ensure that the pipeline continues with the correct I/O.
+ */
 static int
 zio_vdev_io_start(zio_t *zio)
 {
@@ -2575,7 +2590,8 @@ zio_vdev_io_start(zio_t *zio)
 		/*
 		 * The mirror_ops handle multiple DVAs in a single BP.
 		 */
-		return (vdev_mirror_ops.vdev_op_io_start(zio));
+		vdev_mirror_ops.vdev_op_io_start(zio);
+		return (ZIO_PIPELINE_STOP);
 	}
 
 	if (vd->vdev_ops->vdev_op_leaf && zio->io_type == ZIO_TYPE_FREE &&
@@ -2589,7 +2605,7 @@ zio_vdev_io_start(zio_t *zio)
 	 * can quickly react to certain workloads.  In particular, we care
 	 * about non-scrubbing, top-level reads and writes with the following
 	 * characteristics:
-	 * 	- synchronous writes of user data to non-slog devices
+	 *	- synchronous writes of user data to non-slog devices
 	 *	- any reads of user data
 	 * When these conditions are met, adjust the timestamp of spa_last_io
 	 * which allows the scan thread to adjust its workload accordingly.
@@ -2693,10 +2709,8 @@ zio_vdev_io_start(zio_t *zio)
 			return (ZIO_PIPELINE_STOP);
 	}
 
-	ret = vd->vdev_ops->vdev_op_io_start(zio);
-	ASSERT(ret == ZIO_PIPELINE_STOP);
-
-	return (ret);
+	vd->vdev_ops->vdev_op_io_start(zio);
+	return (ZIO_PIPELINE_STOP);
 }
 
 static int