Add aio_writev and aio_readv

POSIX AIO is great, but it lacks vectored I/O functions. This commit
fixes that shortcoming by adding aio_writev and aio_readv. They aren't
part of the standard, but they're an obvious extension. They work just
like their synchronous equivalents pwritev and preadv.

It isn't yet possible to use vectored aiocbs with lio_listio, but that
could be added in the future.

Reviewed by:    jhb, kib, bcr
Relnotes:       yes
Differential Revision: https://reviews.freebsd.org/D27743

1 files changed, 672 insertions, 67 deletions

diff --git a/tests/sys/aio/aio_test.c b/tests/sys/aio/aio_test.c
index a9216335d768..891892e5e757 100644
--- a/tests/sys/aio/aio_test.c
+++ b/tests/sys/aio/aio_test.c
@@ -282,6 +282,47 @@ aio_write_test(struct aio_context *ac, completion comp, struct sigevent *sev)
 }
 
 /*
+ * Perform a vectored I/O test of our initialized data buffer to the provided
+ * file descriptor.
+ *
+ * To vectorize the linear buffer, chop it up into two pieces of dissimilar
+ * size, and swap their offsets.
+ */
+static void
+aio_writev_test(struct aio_context *ac, completion comp, struct sigevent *sev)
+{
+	struct aiocb aio;
+	struct iovec iov[2];
+	size_t len0, len1;
+	ssize_t len;
+
+	bzero(&aio, sizeof(aio));
+
+	aio.aio_fildes = ac->ac_write_fd;
+	aio.aio_offset = 0;
+	len0 = ac->ac_buflen * 3 / 4;
+	len1 = ac->ac_buflen / 4;
+	iov[0].iov_base = ac->ac_buffer + len1;
+	iov[0].iov_len = len0;
+	iov[1].iov_base = ac->ac_buffer;
+	iov[1].iov_len = len1;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = 2;
+	if (sev)
+		aio.aio_sigevent = *sev;
+
+	if (aio_writev(&aio) < 0)
+		atf_tc_fail("aio_writev failed: %s", strerror(errno));
+
+	len = comp(&aio);
+	if (len < 0)
+		atf_tc_fail("aio failed: %s", strerror(errno));
+
+	if (len != ac->ac_buflen)
+		atf_tc_fail("aio short write (%jd)", (intmax_t)len);
+}
+
+/*
  * Perform a simple read test of our initialized data buffer from the
  * provided file descriptor.
  */
@@ -314,6 +355,43 @@ aio_read_test(struct aio_context *ac, completion comp, struct sigevent *sev)
 		atf_tc_fail("buffer mismatched");
 }
 
+static void
+aio_readv_test(struct aio_context *ac, completion comp, struct sigevent *sev)
+{
+	struct aiocb aio;
+	struct iovec iov[2];
+	size_t len0, len1;
+	ssize_t len;
+
+	bzero(ac->ac_buffer, ac->ac_buflen);
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = ac->ac_read_fd;
+	aio.aio_offset = 0;
+	len0 = ac->ac_buflen * 3 / 4;
+	len1 = ac->ac_buflen / 4;
+	iov[0].iov_base = ac->ac_buffer + len1;
+	iov[0].iov_len = len0;
+	iov[1].iov_base = ac->ac_buffer;
+	iov[1].iov_len = len1;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = 2;
+	if (sev)
+		aio.aio_sigevent = *sev;
+
+	if (aio_readv(&aio) < 0)
+		atf_tc_fail("aio_read failed: %s", strerror(errno));
+
+	len = comp(&aio);
+	if (len < 0)
+		atf_tc_fail("aio failed: %s", strerror(errno));
+
+	ATF_REQUIRE_EQ_MSG(len, ac->ac_buflen,
+	    "aio short read (%jd)", (intmax_t)len);
+
+	if (aio_test_buffer(ac->ac_buffer, ac->ac_buflen, ac->ac_seed) == 0)
+		atf_tc_fail("buffer mismatched");
+}
+
 /*
  * Series of type-specific tests for AIO.  For now, we just make sure we can
  * issue a write and then a read to each type.  We assume that once a write
@@ -328,7 +406,7 @@ aio_read_test(struct aio_context *ac, completion comp, struct sigevent *sev)
 #define	FILE_PATHNAME	"testfile"
 
 static void
-aio_file_test(completion comp, struct sigevent *sev)
+aio_file_test(completion comp, struct sigevent *sev, bool vectored)
 {
 	struct aio_context ac;
 	int fd;
@@ -340,39 +418,44 @@ aio_file_test(completion comp, struct sigevent *sev)
 	ATF_REQUIRE_MSG(fd != -1, "open failed: %s", strerror(errno));
 
 	aio_context_init(&ac, fd, fd, FILE_LEN);
-	aio_write_test(&ac, comp, sev);
-	aio_read_test(&ac, comp, sev);
+	if (vectored) {
+		aio_writev_test(&ac, comp, sev);
+		aio_readv_test(&ac, comp, sev);
+	} else {
+		aio_write_test(&ac, comp, sev);
+		aio_read_test(&ac, comp, sev);
+	}
 	close(fd);
 }
 
 ATF_TC_WITHOUT_HEAD(file_poll);
 ATF_TC_BODY(file_poll, tc)
 {
-	aio_file_test(poll, NULL);
+	aio_file_test(poll, NULL, false);
 }
 
 ATF_TC_WITHOUT_HEAD(file_signal);
 ATF_TC_BODY(file_signal, tc)
 {
-	aio_file_test(poll_signaled, setup_signal());
+	aio_file_test(poll_signaled, setup_signal(), false);
 }
 
 ATF_TC_WITHOUT_HEAD(file_suspend);
 ATF_TC_BODY(file_suspend, tc)
 {
-	aio_file_test(suspend, NULL);
+	aio_file_test(suspend, NULL, false);
 }
 
 ATF_TC_WITHOUT_HEAD(file_thread);
 ATF_TC_BODY(file_thread, tc)
 {
-	aio_file_test(poll_signaled, setup_thread());
+	aio_file_test(poll_signaled, setup_thread(), false);
 }
 
 ATF_TC_WITHOUT_HEAD(file_waitcomplete);
 ATF_TC_BODY(file_waitcomplete, tc)
 {
-	aio_file_test(waitcomplete, NULL);
+	aio_file_test(waitcomplete, NULL, false);
 }
 
 #define	FIFO_LEN	256
@@ -446,7 +529,7 @@ ATF_TC_BODY(fifo_waitcomplete, tc)
 
 #define	UNIX_SOCKETPAIR_LEN	256
 static void
-aio_unix_socketpair_test(completion comp, struct sigevent *sev)
+aio_unix_socketpair_test(completion comp, struct sigevent *sev, bool vectored)
 {
 	struct aio_context ac;
 	struct rusage ru_before, ru_after;
@@ -460,14 +543,16 @@ aio_unix_socketpair_test(completion comp, struct sigevent *sev)
 	aio_context_init(&ac, sockets[0], sockets[1], UNIX_SOCKETPAIR_LEN);
 	ATF_REQUIRE_MSG(getrusage(RUSAGE_SELF, &ru_before) != -1,
 	    "getrusage failed: %s", strerror(errno));
-	aio_write_test(&ac, comp, sev);
+	if (vectored) {
+		aio_writev_test(&ac, comp, sev);
+		aio_readv_test(&ac, comp, sev);
+	} else {
+		aio_write_test(&ac, comp, sev);
+		aio_read_test(&ac, comp, sev);
+	}
 	ATF_REQUIRE_MSG(getrusage(RUSAGE_SELF, &ru_after) != -1,
 	    "getrusage failed: %s", strerror(errno));
 	ATF_REQUIRE(ru_after.ru_msgsnd == ru_before.ru_msgsnd + 1);
-	ru_before = ru_after;
-	aio_read_test(&ac, comp, sev);
-	ATF_REQUIRE_MSG(getrusage(RUSAGE_SELF, &ru_after) != -1,
-	    "getrusage failed: %s", strerror(errno));
 	ATF_REQUIRE(ru_after.ru_msgrcv == ru_before.ru_msgrcv + 1);
 
 	close(sockets[0]);
@@ -477,31 +562,31 @@ aio_unix_socketpair_test(completion comp, struct sigevent *sev)
 ATF_TC_WITHOUT_HEAD(socket_poll);
 ATF_TC_BODY(socket_poll, tc)
 {
-	aio_unix_socketpair_test(poll, NULL);
+	aio_unix_socketpair_test(poll, NULL, false);
 }
 
 ATF_TC_WITHOUT_HEAD(socket_signal);
 ATF_TC_BODY(socket_signal, tc)
 {
-	aio_unix_socketpair_test(poll_signaled, setup_signal());
+	aio_unix_socketpair_test(poll_signaled, setup_signal(), false);
 }
 
 ATF_TC_WITHOUT_HEAD(socket_suspend);
 ATF_TC_BODY(socket_suspend, tc)
 {
-	aio_unix_socketpair_test(suspend, NULL);
+	aio_unix_socketpair_test(suspend, NULL, false);
 }
 
 ATF_TC_WITHOUT_HEAD(socket_thread);
 ATF_TC_BODY(socket_thread, tc)
 {
-	aio_unix_socketpair_test(poll_signaled, setup_thread());
+	aio_unix_socketpair_test(poll_signaled, setup_thread(), false);
 }
 
 ATF_TC_WITHOUT_HEAD(socket_waitcomplete);
 ATF_TC_BODY(socket_waitcomplete, tc)
 {
-	aio_unix_socketpair_test(waitcomplete, NULL);
+	aio_unix_socketpair_test(waitcomplete, NULL, false);
 }
 
 struct aio_pty_arg {
@@ -629,40 +714,11 @@ ATF_TC_BODY(pipe_waitcomplete, tc)
 #define	MD_LEN		GLOBAL_MAX
 #define	MDUNIT_LINK	"mdunit_link"
 
-static void
-aio_md_cleanup(void)
-{
-	struct md_ioctl mdio;
-	int mdctl_fd, error, n, unit;
-	char buf[80];
-
-	mdctl_fd = open("/dev/" MDCTL_NAME, O_RDWR, 0);
-	ATF_REQUIRE(mdctl_fd >= 0);
-	n = readlink(MDUNIT_LINK, buf, sizeof(buf));
-	if (n > 0) {
-		if (sscanf(buf, "%d", &unit) == 1 && unit >= 0) {
-			bzero(&mdio, sizeof(mdio));
-			mdio.md_version = MDIOVERSION;
-			mdio.md_unit = unit;
-			if (ioctl(mdctl_fd, MDIOCDETACH, &mdio) == -1) {
-				error = errno;
-				close(mdctl_fd);
-				errno = error;
-				atf_tc_fail("ioctl MDIOCDETACH failed: %s",
-				    strerror(errno));
-			}
-		}
-	}
-		
-	close(mdctl_fd);
-}
-
-static void
-aio_md_test(completion comp, struct sigevent *sev)
+static int
+aio_md_setup(void)
 {
 	int error, fd, mdctl_fd, unit;
 	char pathname[PATH_MAX];
-	struct aio_context ac;
 	struct md_ioctl mdio;
 	char buf[80];
 
@@ -695,9 +751,52 @@ aio_md_test(completion comp, struct sigevent *sev)
 	ATF_REQUIRE_MSG(fd != -1,
 	    "opening %s failed: %s", pathname, strerror(errno));
 
+	return (fd);
+}
+
+static void
+aio_md_cleanup(void)
+{
+	struct md_ioctl mdio;
+	int mdctl_fd, error, n, unit;
+	char buf[80];
+
+	mdctl_fd = open("/dev/" MDCTL_NAME, O_RDWR, 0);
+	ATF_REQUIRE(mdctl_fd >= 0);
+	n = readlink(MDUNIT_LINK, buf, sizeof(buf));
+	if (n > 0) {
+		if (sscanf(buf, "%d", &unit) == 1 && unit >= 0) {
+			bzero(&mdio, sizeof(mdio));
+			mdio.md_version = MDIOVERSION;
+			mdio.md_unit = unit;
+			if (ioctl(mdctl_fd, MDIOCDETACH, &mdio) == -1) {
+				error = errno;
+				close(mdctl_fd);
+				errno = error;
+				atf_tc_fail("ioctl MDIOCDETACH failed: %s",
+				    strerror(errno));
+			}
+		}
+	}
+
+	close(mdctl_fd);
+}
+
+static void
+aio_md_test(completion comp, struct sigevent *sev, bool vectored)
+{
+	struct aio_context ac;
+	int fd;
+
+	fd = aio_md_setup();
 	aio_context_init(&ac, fd, fd, MD_LEN);
-	aio_write_test(&ac, comp, sev);
-	aio_read_test(&ac, comp, sev);
+	if (vectored) {
+		aio_writev_test(&ac, comp, sev);
+		aio_readv_test(&ac, comp, sev);
+	} else {
+		aio_write_test(&ac, comp, sev);
+		aio_read_test(&ac, comp, sev);
+	}
 	
 	close(fd);
 }
@@ -710,7 +809,7 @@ ATF_TC_HEAD(md_poll, tc)
 }
 ATF_TC_BODY(md_poll, tc)
 {
-	aio_md_test(poll, NULL);
+	aio_md_test(poll, NULL, false);
 }
 ATF_TC_CLEANUP(md_poll, tc)
 {
@@ -725,7 +824,7 @@ ATF_TC_HEAD(md_signal, tc)
 }
 ATF_TC_BODY(md_signal, tc)
 {
-	aio_md_test(poll_signaled, setup_signal());
+	aio_md_test(poll_signaled, setup_signal(), false);
 }
 ATF_TC_CLEANUP(md_signal, tc)
 {
@@ -740,7 +839,7 @@ ATF_TC_HEAD(md_suspend, tc)
 }
 ATF_TC_BODY(md_suspend, tc)
 {
-	aio_md_test(suspend, NULL);
+	aio_md_test(suspend, NULL, false);
 }
 ATF_TC_CLEANUP(md_suspend, tc)
 {
@@ -755,7 +854,7 @@ ATF_TC_HEAD(md_thread, tc)
 }
 ATF_TC_BODY(md_thread, tc)
 {
-	aio_md_test(poll_signaled, setup_thread());
+	aio_md_test(poll_signaled, setup_thread(), false);
 }
 ATF_TC_CLEANUP(md_thread, tc)
 {
@@ -770,13 +869,89 @@ ATF_TC_HEAD(md_waitcomplete, tc)
 }
 ATF_TC_BODY(md_waitcomplete, tc)
 {
-	aio_md_test(waitcomplete, NULL);
+	aio_md_test(waitcomplete, NULL, false);
 }
 ATF_TC_CLEANUP(md_waitcomplete, tc)
 {
 	aio_md_cleanup();
 }
 
+#define	ZVOL_VDEV_PATHNAME	"test_vdev"
+#define POOL_SIZE		(1 << 28)	/* 256 MB */
+#define ZVOL_SIZE		"64m"
+#define POOL_NAME		"aio_testpool"
+#define ZVOL_NAME		"aio_testvol"
+
+static int
+aio_zvol_setup(void)
+{
+	FILE *pidfile;
+	int fd;
+	pid_t pid;
+	char pool_name[80];
+	char cmd[160];
+	char zvol_name[160];
+	char devname[160];
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_KERNEL_MODULE("zfs");
+
+	fd = open(ZVOL_VDEV_PATHNAME, O_RDWR | O_CREAT, 0600);
+	ATF_REQUIRE_MSG(fd != -1, "open failed: %s", strerror(errno));
+	ATF_REQUIRE_EQ_MSG(0,
+	    ftruncate(fd, POOL_SIZE), "ftruncate failed: %s", strerror(errno));
+	close(fd);
+
+	pid = getpid();
+	pidfile = fopen("pidfile", "w");
+	ATF_REQUIRE_MSG(NULL != pidfile, "fopen: %s", strerror(errno));
+	fprintf(pidfile, "%d", pid);
+	fclose(pidfile);
+
+	snprintf(pool_name, sizeof(pool_name), POOL_NAME ".%d", pid);
+	snprintf(zvol_name, sizeof(zvol_name), "%s/" ZVOL_NAME, pool_name);
+	snprintf(cmd, sizeof(cmd), "zpool create %s $PWD/" ZVOL_VDEV_PATHNAME,
+	    pool_name);
+	ATF_REQUIRE_EQ_MSG(0, system(cmd),
+	    "zpool create failed: %s", strerror(errno));
+	snprintf(cmd, sizeof(cmd),
+	    "zfs create -o volblocksize=8192 -o volmode=dev -V "
+		ZVOL_SIZE " %s", zvol_name);
+	ATF_REQUIRE_EQ_MSG(0, system(cmd),
+	    "zfs create failed: %s", strerror(errno));
+	/*
+	 * XXX Due to bug 251828, we need an extra "zfs set" here
+	 * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251828
+	 */
+	snprintf(cmd, sizeof(cmd), "zfs set volmode=dev %s", zvol_name);
+	ATF_REQUIRE_EQ_MSG(0, system(cmd),
+	    "zfs set failed: %s", strerror(errno));
+
+	snprintf(devname, sizeof(devname), "/dev/zvol/%s", zvol_name);
+	do {
+		fd = open(devname, O_RDWR);
+	} while (fd == -1 && errno == EINTR) ;
+	ATF_REQUIRE_MSG(fd != -1, "open failed: %s", strerror(errno));
+	return (fd);
+}
+
+static void
+aio_zvol_cleanup(void)
+{
+	FILE *pidfile;
+	pid_t testpid;
+	char cmd[160];
+
+	pidfile = fopen("pidfile", "r");
+	ATF_REQUIRE_MSG(NULL != pidfile, "fopen: %s", strerror(errno));
+	ATF_REQUIRE_EQ(1, fscanf(pidfile, "%d", &testpid));
+	fclose(pidfile);
+
+	snprintf(cmd, sizeof(cmd), "zpool destroy " POOL_NAME ".%d", testpid);
+	system(cmd);
+}
+
+
 ATF_TC_WITHOUT_HEAD(aio_large_read_test);
 ATF_TC_BODY(aio_large_read_test, tc)
 {
@@ -907,16 +1082,13 @@ ATF_TC_BODY(aio_socket_two_reads, tc)
 	close(s[0]);
 }
 
-/*
- * This test ensures that aio_write() on a blocking socket of a "large"
- * buffer does not return a short completion.
- */
-ATF_TC_WITHOUT_HEAD(aio_socket_blocking_short_write);
-ATF_TC_BODY(aio_socket_blocking_short_write, tc)
+static void
+aio_socket_blocking_short_write_test(bool vectored)
 {
 	struct aiocb iocb, *iocbp;
+	struct iovec iov[2];
 	char *buffer[2];
-	ssize_t done;
+	ssize_t done, r;
 	int buffer_size, sb_size;
 	socklen_t len;
 	int s[2];
@@ -954,9 +1126,21 @@ ATF_TC_BODY(aio_socket_blocking_short_write, tc)
 
 	memset(&iocb, 0, sizeof(iocb));
 	iocb.aio_fildes = s[1];
-	iocb.aio_buf = buffer[1];
-	iocb.aio_nbytes = buffer_size;
-	ATF_REQUIRE(aio_write(&iocb) == 0);
+	if (vectored) {
+		iov[0].iov_base = buffer[1];
+		iov[0].iov_len = buffer_size / 2 + 1;
+		iov[1].iov_base = buffer[1] + buffer_size / 2 + 1;
+		iov[1].iov_len = buffer_size / 2 - 1;
+		iocb.aio_iov = iov;
+		iocb.aio_iovcnt = 2;
+		r = aio_writev(&iocb);
+		ATF_CHECK_EQ_MSG(0, r, "aio_writev returned %zd", r);
+	} else {
+		iocb.aio_buf = buffer[1];
+		iocb.aio_nbytes = buffer_size;
+		r = aio_write(&iocb);
+		ATF_CHECK_EQ_MSG(0, r, "aio_writev returned %zd", r);
+	}
 
 	done = recv(s[0], buffer[0], buffer_size, MSG_WAITALL);
 	ATF_REQUIRE(done == buffer_size);
@@ -972,6 +1156,26 @@ ATF_TC_BODY(aio_socket_blocking_short_write, tc)
 }
 
 /*
+ * This test ensures that aio_write() on a blocking socket of a "large"
+ * buffer does not return a short completion.
+ */
+ATF_TC_WITHOUT_HEAD(aio_socket_blocking_short_write);
+ATF_TC_BODY(aio_socket_blocking_short_write, tc)
+{
+	aio_socket_blocking_short_write_test(false);
+}
+
+/*
+ * Like aio_socket_blocking_short_write, but also tests that partially
+ * completed vectored sends can be retried correctly.
+ */
+ATF_TC_WITHOUT_HEAD(aio_socket_blocking_short_write_vectored);
+ATF_TC_BODY(aio_socket_blocking_short_write_vectored, tc)
+{
+	aio_socket_blocking_short_write_test(true);
+}
+
+/*
  * This test verifies that cancelling a partially completed socket write
  * returns a short write rather than ECANCELED.
  */
@@ -1155,6 +1359,395 @@ ATF_TC_BODY(aio_fsync_test, tc)
 	close(fd);
 }
 
+/*
+ * We shouldn't be able to DoS the system by setting iov_len to an insane
+ * value
+ */
+ATF_TC_WITHOUT_HEAD(aio_writev_dos_iov_len);
+ATF_TC_BODY(aio_writev_dos_iov_len, tc)
+{
+	struct aiocb aio;
+	const struct aiocb *const iocbs[] = {&aio};
+	const char *wbuf = "Hello, world!";
+	struct iovec iov[1];
+	ssize_t len, r;
+	int fd;
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_UNSAFE_AIO();
+
+	fd = open("testfile", O_RDWR | O_CREAT, 0600);
+	ATF_REQUIRE_MSG(fd != -1, "open failed: %s", strerror(errno));
+
+	len = strlen(wbuf);
+	iov[0].iov_base = __DECONST(void*, wbuf);
+	iov[0].iov_len = 1 << 30;
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = fd;
+	aio.aio_offset = 0;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = 1;
+
+	r = aio_writev(&aio);
+	ATF_CHECK_EQ_MSG(0, r, "aio_writev returned %zd", r);
+	ATF_REQUIRE_EQ(0, aio_suspend(iocbs, 1, NULL));
+	r = aio_return(&aio);
+	ATF_CHECK_EQ_MSG(-1, r, "aio_return returned %zd", r);
+	ATF_CHECK_MSG(errno == EFAULT || errno == EINVAL,
+	    "aio_writev: %s", strerror(errno));
+
+	close(fd);
+}
+
+/*
+ * We shouldn't be able to DoS the system by setting aio_iovcnt to an insane
+ * value
+ */
+ATF_TC_WITHOUT_HEAD(aio_writev_dos_iovcnt);
+ATF_TC_BODY(aio_writev_dos_iovcnt, tc)
+{
+	struct aiocb aio;
+	const char *wbuf = "Hello, world!";
+	struct iovec iov[1];
+	ssize_t len;
+	int fd;
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_UNSAFE_AIO();
+
+	fd = open("testfile", O_RDWR | O_CREAT, 0600);
+	ATF_REQUIRE_MSG(fd != -1, "open failed: %s", strerror(errno));
+
+	len = strlen(wbuf);
+	iov[0].iov_base = __DECONST(void*, wbuf);
+	iov[0].iov_len = len;
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = fd;
+	aio.aio_offset = 0;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = 1 << 30;
+
+	ATF_REQUIRE_EQ(-1, aio_writev(&aio));
+	ATF_CHECK_EQ(EINVAL, errno);
+
+	close(fd);
+}
+
+ATF_TC_WITH_CLEANUP(aio_writev_efault);
+ATF_TC_HEAD(aio_writev_efault, tc)
+{
+	atf_tc_set_md_var(tc, "descr",
+	    "Vectored AIO should gracefully handle invalid addresses");
+	atf_tc_set_md_var(tc, "require.user", "root");
+}
+ATF_TC_BODY(aio_writev_efault, tc)
+{
+	struct aiocb aio;
+	ssize_t buflen;
+	char *buffer;
+	struct iovec iov[2];
+	long seed;
+	int fd;
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_UNSAFE_AIO();
+
+	fd = aio_md_setup();
+
+	seed = random();
+	buflen = 4096;
+	buffer = malloc(buflen);
+	aio_fill_buffer(buffer, buflen, seed);
+	iov[0].iov_base = buffer;
+	iov[0].iov_len = buflen;
+	iov[1].iov_base = (void*)-1;	/* Invalid! */
+	iov[1].iov_len = buflen;
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = fd;
+	aio.aio_offset = 0;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = nitems(iov);
+
+	ATF_REQUIRE_EQ(-1, aio_writev(&aio));
+	ATF_CHECK_EQ(EFAULT, errno);
+
+	close(fd);
+}
+ATF_TC_CLEANUP(aio_writev_efault, tc)
+{
+	aio_md_cleanup();
+}
+
+ATF_TC_WITHOUT_HEAD(aio_writev_empty_file_poll);
+ATF_TC_BODY(aio_writev_empty_file_poll, tc)
+{
+	struct aiocb aio;
+	int fd;
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_UNSAFE_AIO();
+
+	fd = open("testfile", O_RDWR | O_CREAT, 0600);
+	ATF_REQUIRE_MSG(fd != -1, "open failed: %s", strerror(errno));
+
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = fd;
+	aio.aio_offset = 0;
+	aio.aio_iovcnt = 0;
+
+	ATF_REQUIRE_EQ(0, aio_writev(&aio));
+	ATF_REQUIRE_EQ(0, suspend(&aio));
+
+	close(fd);
+}
+
+ATF_TC_WITHOUT_HEAD(aio_writev_empty_file_signal);
+ATF_TC_BODY(aio_writev_empty_file_signal, tc)
+{
+	struct aiocb aio;
+	int fd;
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_UNSAFE_AIO();
+
+	fd = open("testfile", O_RDWR | O_CREAT, 0600);
+	ATF_REQUIRE_MSG(fd != -1, "open failed: %s", strerror(errno));
+
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = fd;
+	aio.aio_offset = 0;
+	aio.aio_iovcnt = 0;
+	aio.aio_sigevent = *setup_signal();
+
+	ATF_REQUIRE_EQ(0, aio_writev(&aio));
+	ATF_REQUIRE_EQ(0, poll_signaled(&aio));
+
+	close(fd);
+}
+
+// aio_writev and aio_readv should still work even if the iovcnt is greater
+// than the number of buffered AIO operations permitted per process.
+ATF_TC_WITH_CLEANUP(vectored_big_iovcnt);
+ATF_TC_HEAD(vectored_big_iovcnt, tc)
+{
+	atf_tc_set_md_var(tc, "descr",
+	    "Vectored AIO should still work even if the iovcnt is greater than "
+	    "the number of buffered AIO operations permitted by the process");
+	atf_tc_set_md_var(tc, "require.user", "root");
+}
+ATF_TC_BODY(vectored_big_iovcnt, tc)
+{
+	struct aiocb aio;
+	struct iovec *iov;
+	ssize_t len, buflen;
+	char *buffer;
+	const char *oid = "vfs.aio.max_buf_aio";
+	long seed;
+	int max_buf_aio;
+	int fd, i;
+	ssize_t sysctl_len = sizeof(max_buf_aio);
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_UNSAFE_AIO();
+
+	if (sysctlbyname(oid, &max_buf_aio, &sysctl_len, NULL, 0) == -1)
+		atf_libc_error(errno, "Failed to read %s", oid);
+
+	seed = random();
+	buflen = 512 * (max_buf_aio + 1);
+	buffer = malloc(buflen);
+	aio_fill_buffer(buffer, buflen, seed);
+	iov = calloc(max_buf_aio + 1, sizeof(struct iovec));
+
+	fd = aio_md_setup();
+
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = fd;
+	aio.aio_offset = 0;
+	for (i = 0; i < max_buf_aio + 1; i++) {
+		iov[i].iov_base = &buffer[i * 512];
+		iov[i].iov_len = 512;
+	}
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = max_buf_aio + 1;
+
+	if (aio_writev(&aio) < 0)
+		atf_tc_fail("aio_writev failed: %s", strerror(errno));
+
+	len = poll(&aio);
+	if (len < 0)
+		atf_tc_fail("aio failed: %s", strerror(errno));
+
+	if (len != buflen)
+		atf_tc_fail("aio short write (%jd)", (intmax_t)len);
+
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = fd;
+	aio.aio_offset = 0;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = max_buf_aio + 1;
+
+	if (aio_readv(&aio) < 0)
+		atf_tc_fail("aio_readv failed: %s", strerror(errno));
+
+	len = poll(&aio);
+	if (len < 0)
+		atf_tc_fail("aio failed: %s", strerror(errno));
+
+	if (len != buflen)
+		atf_tc_fail("aio short read (%jd)", (intmax_t)len);
+
+	if (aio_test_buffer(buffer, buflen, seed) == 0)
+		atf_tc_fail("buffer mismatched");
+
+	close(fd);
+}
+ATF_TC_CLEANUP(vectored_big_iovcnt, tc)
+{
+	aio_md_cleanup();
+}
+
+ATF_TC_WITHOUT_HEAD(vectored_file_poll);
+ATF_TC_BODY(vectored_file_poll, tc)
+{
+	aio_file_test(poll, NULL, true);
+}
+
+ATF_TC_WITH_CLEANUP(vectored_md_poll);
+ATF_TC_HEAD(vectored_md_poll, tc)
+{
+	atf_tc_set_md_var(tc, "require.user", "root");
+}
+ATF_TC_BODY(vectored_md_poll, tc)
+{
+	aio_md_test(poll, NULL, true);
+}
+ATF_TC_CLEANUP(vectored_md_poll, tc)
+{
+	aio_md_cleanup();
+}
+
+ATF_TC_WITHOUT_HEAD(vectored_socket_poll);
+ATF_TC_BODY(vectored_socket_poll, tc)
+{
+	aio_unix_socketpair_test(poll, NULL, true);
+}
+
+// aio_writev and aio_readv should still work even if the iov contains elements
+// that aren't a multiple of the device's sector size, and even if the total
+// amount if I/O _is_ a multiple of the device's sector size.
+ATF_TC_WITH_CLEANUP(vectored_unaligned);
+ATF_TC_HEAD(vectored_unaligned, tc)
+{
+	atf_tc_set_md_var(tc, "descr",
+	    "Vectored AIO should still work even if the iov contains elements "
+	    "that aren't a multiple of the sector size.");
+	atf_tc_set_md_var(tc, "require.user", "root");
+}
+ATF_TC_BODY(vectored_unaligned, tc)
+{
+	struct aio_context ac;
+	struct aiocb aio;
+	struct iovec iov[3];
+	ssize_t len, total_len;
+	int fd;
+
+	ATF_REQUIRE_KERNEL_MODULE("aio");
+	ATF_REQUIRE_UNSAFE_AIO();
+
+	/* 
+	 * Use a zvol with volmode=dev, so it will allow .d_write with
+	 * unaligned uio.  geom devices use physio, which doesn't allow that.
+	 */
+	fd = aio_zvol_setup();
+	aio_context_init(&ac, fd, fd, FILE_LEN);
+
+	/* Break the buffer into 3 parts:
+	 * * A 4kB part, aligned to 4kB
+	 * * Two other parts that add up to 4kB:
+	 *   - 256B
+	 *   - 4kB - 256B
+	 */
+	iov[0].iov_base = ac.ac_buffer;
+	iov[0].iov_len = 4096;
+	iov[1].iov_base = (void*)((uintptr_t)iov[0].iov_base + iov[0].iov_len);
+	iov[1].iov_len = 256;
+	iov[2].iov_base = (void*)((uintptr_t)iov[1].iov_base + iov[1].iov_len);
+	iov[2].iov_len = 4096 - iov[1].iov_len;
+	total_len = iov[0].iov_len + iov[1].iov_len + iov[2].iov_len;
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = ac.ac_write_fd;
+	aio.aio_offset = 0;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = 3;
+
+	if (aio_writev(&aio) < 0)
+		atf_tc_fail("aio_writev failed: %s", strerror(errno));
+
+	len = poll(&aio);
+	if (len < 0)
+		atf_tc_fail("aio failed: %s", strerror(errno));
+
+	if (len != total_len)
+		atf_tc_fail("aio short write (%jd)", (intmax_t)len);
+
+	bzero(&aio, sizeof(aio));
+	aio.aio_fildes = ac.ac_read_fd;
+	aio.aio_offset = 0;
+	aio.aio_iov = iov;
+	aio.aio_iovcnt = 3;
+
+	if (aio_readv(&aio) < 0)
+		atf_tc_fail("aio_readv failed: %s", strerror(errno));
+	len = poll(&aio);
+
+	ATF_REQUIRE_MSG(aio_test_buffer(ac.ac_buffer, total_len,
+	    ac.ac_seed) != 0, "aio_test_buffer: internal error");
+
+	close(fd);
+}
+ATF_TC_CLEANUP(vectored_unaligned, tc)
+{
+	aio_zvol_cleanup();
+}
+
+static void
+aio_zvol_test(completion comp, struct sigevent *sev, bool vectored)
+{
+	struct aio_context ac;
+	int fd;
+
+	fd = aio_zvol_setup();
+	aio_context_init(&ac, fd, fd, MD_LEN);
+	if (vectored) {
+		aio_writev_test(&ac, comp, sev);
+		aio_readv_test(&ac, comp, sev);
+	} else {
+		aio_write_test(&ac, comp, sev);
+		aio_read_test(&ac, comp, sev);
+	}
+
+	close(fd);
+}
+
+/*
+ * Note that unlike md, the zvol is not a geom device, does not allow unmapped
+ * buffers, and does not use physio.
+ */
+ATF_TC_WITH_CLEANUP(vectored_zvol_poll);
+ATF_TC_HEAD(vectored_zvol_poll, tc)
+{
+	atf_tc_set_md_var(tc, "require.user", "root");
+}
+ATF_TC_BODY(vectored_zvol_poll, tc)
+{
+	aio_zvol_test(poll, NULL, true);
+}
+ATF_TC_CLEANUP(vectored_zvol_poll, tc)
+{
+	aio_zvol_cleanup();
+}
+
 ATF_TP_ADD_TCS(tp)
 {
 
@@ -1193,7 +1786,19 @@ ATF_TP_ADD_TCS(tp)
 	ATF_TP_ADD_TC(tp, aio_large_read_test);
 	ATF_TP_ADD_TC(tp, aio_socket_two_reads);
 	ATF_TP_ADD_TC(tp, aio_socket_blocking_short_write);
+	ATF_TP_ADD_TC(tp, aio_socket_blocking_short_write_vectored);
 	ATF_TP_ADD_TC(tp, aio_socket_short_write_cancel);
+	ATF_TP_ADD_TC(tp, aio_writev_dos_iov_len);
+	ATF_TP_ADD_TC(tp, aio_writev_dos_iovcnt);
+	ATF_TP_ADD_TC(tp, aio_writev_efault);
+	ATF_TP_ADD_TC(tp, aio_writev_empty_file_poll);
+	ATF_TP_ADD_TC(tp, aio_writev_empty_file_signal);
+	ATF_TP_ADD_TC(tp, vectored_big_iovcnt);
+	ATF_TP_ADD_TC(tp, vectored_file_poll);
+	ATF_TP_ADD_TC(tp, vectored_md_poll);
+	ATF_TP_ADD_TC(tp, vectored_zvol_poll);
+	ATF_TP_ADD_TC(tp, vectored_unaligned);
+	ATF_TP_ADD_TC(tp, vectored_socket_poll);
 
 	return (atf_no_error());
 }