Mfp4 CH=177274,177280,177284-177285,177297,177324-177325

  VNET socket push back:
  try to minimize the number of places where we have to switch vnets
  and narrow down the time we stay switched.  Add assertions to the
  socket code to catch possibly unset vnets as seen in r204147.

  While this reduces the number of vnet recursion in some places like
  NFS, POSIX local sockets and some netgraph, .. recursions are
  impossible to fix.

  The current expectations are documented at the beginning of
  uipc_socket.c along with the other information there.

  Sponsored by: The FreeBSD Foundation
  Sponsored by: CK Software GmbH
  Reviewed by:  jhb
  Tested by:    zec

Tested by:	Mikolaj Golub (to.my.trociny gmail.com)
MFC after:	2 weeks

23 files changed, 179 insertions, 103 deletions

diff --git a/sys/fs/nfsclient/nfs_clvfsops.c b/sys/fs/nfsclient/nfs_clvfsops.c
index d6db84fc338c..b2d4864c6c55 100644
--- a/sys/fs/nfsclient/nfs_clvfsops.c
+++ b/sys/fs/nfsclient/nfs_clvfsops.c
@@ -452,10 +452,12 @@ ncl_mountroot(struct mount *mp)
 		sin.sin_family = AF_INET;
 		sin.sin_len = sizeof(sin);
                 /* XXX MRT use table 0 for this sort of thing */
+		CURVNET_SET(TD_TO_VNET(td));
 		error = rtrequest(RTM_ADD, (struct sockaddr *)&sin,
 		    (struct sockaddr *)&nd->mygateway,
 		    (struct sockaddr *)&mask,
 		    RTF_UP | RTF_GATEWAY, NULL);
+		CURVNET_RESTORE();
 		if (error)
 			panic("nfs_mountroot: RTM_ADD: %d", error);
 	}
diff --git a/sys/fs/portalfs/portal_vnops.c b/sys/fs/portalfs/portal_vnops.c
index 64e5afda33a9..6fcc1ce64219 100644
--- a/sys/fs/portalfs/portal_vnops.c
+++ b/sys/fs/portalfs/portal_vnops.c
@@ -60,6 +60,8 @@
 
 #include <fs/portalfs/portal.h>
 
+#include <net/vnet.h>
+
 static int portal_fileid = PORTAL_ROOTFILEID+1;
 
 static void	portal_closefd(struct thread *td, int fd);
@@ -185,8 +187,12 @@ portal_connect(so, so2)
 	if ((so2->so_options & SO_ACCEPTCONN) == 0)
 		return (ECONNREFUSED);
 
-	if ((so3 = sonewconn(so2, 0)) == 0)
+	CURVNET_SET(so2->so_vnet);
+	if ((so3 = sonewconn(so2, 0)) == 0) {
+		CURVNET_RESTORE();
 		return (ECONNREFUSED);
+	}
+	CURVNET_RESTORE();
 
 	unp2 = sotounpcb(so2);
 	unp3 = sotounpcb(so3);
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index d59bcb400b79..bca83b561717 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -78,6 +78,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/ktrace.h>
 #endif
 
+#include <net/vnet.h>
+
 #include <security/audit/audit.h>
 
 #include <vm/uma.h>
@@ -2330,7 +2332,9 @@ fputsock(struct socket *so)
 
 	ACCEPT_LOCK();
 	SOCK_LOCK(so);
+	CURVNET_SET(so->so_vnet);
 	sorele(so);
+	CURVNET_RESTORE();
 }
 
 /*
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index 2766ca429305..c9b053443f45 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -80,9 +80,7 @@ soo_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
 	if (error)
 		return (error);
 #endif
-	CURVNET_SET(so->so_vnet);
 	error = soreceive(so, 0, uio, 0, 0, 0);
-	CURVNET_RESTORE();
 	return (error);
 }
 
@@ -123,7 +121,6 @@ soo_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
 	struct socket *so = fp->f_data;
 	int error = 0;
 
-	CURVNET_SET(so->so_vnet);
 	switch (cmd) {
 	case FIONBIO:
 		SOCK_LOCK(so);
@@ -210,14 +207,18 @@ soo_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
 		 */
 		if (IOCGROUP(cmd) == 'i')
 			error = ifioctl(so, cmd, data, td);
-		else if (IOCGROUP(cmd) == 'r')
+		else if (IOCGROUP(cmd) == 'r') {
+			CURVNET_SET(so->so_vnet);
 			error = rtioctl_fib(cmd, data, so->so_fibnum);
-		else
+			CURVNET_RESTORE();
+		} else {
+			CURVNET_SET(so->so_vnet);
 			error = ((*so->so_proto->pr_usrreqs->pru_control)
 			    (so, cmd, data, 0, td));
+			CURVNET_RESTORE();
+		}
 		break;
 	}
-	CURVNET_RESTORE();
 	return (error);
 }
 
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 738bef941060..54a050f2a39a 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -92,6 +92,12 @@
  * from a listen queue to a file descriptor, in order to prevent garbage
  * collection of the socket at an untimely moment.  For a number of reasons,
  * these interfaces are not preferred, and should be avoided.
+ * 
+ * NOTE: With regard to VNETs the general rule is that callers do not set
+ * curvnet. Exceptions to this rule include soabort(), sodisconnect(),
+ * sofree() (and with that sorele(), sotryfree()), as well as sonewconn()
+ * and sorflush(), which are usually called from a pre-set VNET context.
+ * sopoll() currently does not need a VNET context to be set.
  */
 
 #include <sys/cdefs.h>
@@ -175,6 +181,10 @@ int	maxsockets;
 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block");
 
+#define	VNET_SO_ASSERT(so)						\
+	VNET_ASSERT(curvnet != NULL,					\
+	    ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));
+
 static int somaxconn = SOMAXCONN;
 static int sysctl_somaxconn(SYSCTL_HANDLER_ARGS);
 /* XXX: we dont have SYSCTL_USHORT */
@@ -295,6 +305,8 @@ soalloc(struct vnet *vnet)
 	so->so_gencnt = ++so_gencnt;
 	++numopensockets;
 #ifdef VIMAGE
+	VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p",
+	    __func__, __LINE__, so));
 	vnet->vnet_sockcnt++;
 	so->so_vnet = vnet;
 #endif
@@ -318,6 +330,8 @@ sodealloc(struct socket *so)
 	so->so_gencnt = ++so_gencnt;
 	--numopensockets;	/* Could be below, but faster here. */
 #ifdef VIMAGE
+	VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p",
+	    __func__, __LINE__, so));
 	so->so_vnet->vnet_sockcnt--;
 #endif
 	mtx_unlock(&so_global_mtx);
@@ -457,6 +471,7 @@ sonewconn(struct socket *head, int connstatus)
 #endif
 	knlist_init_mtx(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv));
 	knlist_init_mtx(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd));
+	VNET_SO_ASSERT(head);
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
 		sodealloc(so);
@@ -531,8 +546,12 @@ sobind(struct socket *so, struct sockaddr *nam, struct thread *td)
 int
 solisten(struct socket *so, int backlog, struct thread *td)
 {
+	int error;
 
-	return ((*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td));
+	CURVNET_SET(so->so_vnet);
+	error = (*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td);
+	CURVNET_RESTORE();
+	return error;
 }
 
 int
@@ -620,6 +639,7 @@ sofree(struct socket *so)
 	SOCK_UNLOCK(so);
 	ACCEPT_UNLOCK();
 
+	VNET_SO_ASSERT(so);
 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
 		(*pr->pr_domain->dom_dispose)(so->so_rcv.sb_mb);
 	if (pr->pr_usrreqs->pru_detach != NULL)
@@ -749,6 +769,7 @@ soabort(struct socket *so)
 	KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF"));
 	KASSERT((so->so_state & SQ_COMP) == 0, ("soabort: SQ_COMP"));
 	KASSERT((so->so_state & SQ_INCOMP) == 0, ("soabort: SQ_INCOMP"));
+	VNET_SO_ASSERT(so);
 
 	if (so->so_proto->pr_usrreqs->pru_abort != NULL)
 		(*so->so_proto->pr_usrreqs->pru_abort)(so);
@@ -766,7 +787,10 @@ soaccept(struct socket *so, struct sockaddr **nam)
 	KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF"));
 	so->so_state &= ~SS_NOFDREF;
 	SOCK_UNLOCK(so);
+
+	CURVNET_SET(so->so_vnet);
 	error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam);
+	CURVNET_RESTORE();
 	return (error);
 }
 
@@ -804,8 +828,12 @@ soconnect(struct socket *so, struct sockaddr *nam, struct thread *td)
 int
 soconnect2(struct socket *so1, struct socket *so2)
 {
+	int error;
 
-	return ((*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2));
+	CURVNET_SET(so1->so_vnet);
+	error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2);
+	CURVNET_RESTORE();
+	return (error);
 }
 
 int
@@ -817,6 +845,7 @@ sodisconnect(struct socket *so)
 		return (ENOTCONN);
 	if (so->so_state & SS_ISDISCONNECTING)
 		return (EALREADY);
+	VNET_SO_ASSERT(so);
 	error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so);
 	return (error);
 }
@@ -1080,6 +1109,7 @@ sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio,
 	 * there are probably other places that this also happens.  We must
 	 * rethink this.
 	 */
+	VNET_SO_ASSERT(so);
 	error = (*so->so_proto->pr_usrreqs->pru_send)(so,
 	    (flags & MSG_OOB) ? PRUS_OOB :
 	/*
@@ -1267,6 +1297,7 @@ restart:
 			 * places that this also happens.  We must rethink
 			 * this.
 			 */
+			VNET_SO_ASSERT(so);
 			error = (*so->so_proto->pr_usrreqs->pru_send)(so,
 			    (flags & MSG_OOB) ? PRUS_OOB :
 			/*
@@ -1333,6 +1364,7 @@ soreceive_rcvoob(struct socket *so, struct uio *uio, int flags)
 	int error;
 
 	KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0"));
+	VNET_SO_ASSERT(so);
 
 	m = m_get(M_WAIT, MT_DATA);
 	error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
@@ -1441,8 +1473,10 @@ soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio,
 	if (mp != NULL)
 		*mp = NULL;
 	if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING)
-	    && uio->uio_resid)
+	    && uio->uio_resid) {
+		VNET_SO_ASSERT(so);
 		(*pr->pr_usrreqs->pru_rcvd)(so, 0);
+	}
 
 	error = sblock(&so->so_rcv, SBLOCKWAIT(flags));
 	if (error)
@@ -1589,6 +1623,7 @@ dontblock:
 			cm->m_next = NULL;
 			if (pr->pr_domain->dom_externalize != NULL) {
 				SOCKBUF_UNLOCK(&so->so_rcv);
+				VNET_SO_ASSERT(so);
 				error = (*pr->pr_domain->dom_externalize)
 				    (cm, controlp);
 				SOCKBUF_LOCK(&so->so_rcv);
@@ -1804,6 +1839,7 @@ dontblock:
 			 */
 			if (pr->pr_flags & PR_WANTRCVD) {
 				SOCKBUF_UNLOCK(&so->so_rcv);
+				VNET_SO_ASSERT(so);
 				(*pr->pr_usrreqs->pru_rcvd)(so, flags);
 				SOCKBUF_LOCK(&so->so_rcv);
 			}
@@ -1850,6 +1886,7 @@ dontblock:
 		if (!(flags & MSG_SOCALLBCK) &&
 		    (pr->pr_flags & PR_WANTRCVD)) {
 			SOCKBUF_UNLOCK(&so->so_rcv);
+			VNET_SO_ASSERT(so);
 			(*pr->pr_usrreqs->pru_rcvd)(so, flags);
 			SOCKBUF_LOCK(&so->so_rcv);
 		}
@@ -2046,6 +2083,7 @@ deliver:
 		    (((flags & MSG_WAITALL) && uio->uio_resid > 0) ||
 		     !(flags & MSG_SOCALLBCK))) {
 			SOCKBUF_UNLOCK(sb);
+			VNET_SO_ASSERT(so);
 			(*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags);
 			SOCKBUF_LOCK(sb);
 		}
@@ -2256,9 +2294,13 @@ int
 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
     struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
 {
+	int error;
 
-	return (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, mp0,
+	CURVNET_SET(so->so_vnet);
+	error = (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, mp0,
 	    controlp, flagsp));
+	CURVNET_RESTORE();
+	return (error);
 }
 
 int
@@ -2269,17 +2311,19 @@ soshutdown(struct socket *so, int how)
 
 	if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
 		return (EINVAL);
+
+	CURVNET_SET(so->so_vnet);
 	if (pr->pr_usrreqs->pru_flush != NULL) {
 	        (*pr->pr_usrreqs->pru_flush)(so, how);
 	}
 	if (how != SHUT_WR)
 		sorflush(so);
 	if (how != SHUT_RD) {
-		CURVNET_SET(so->so_vnet);
 		error = (*pr->pr_usrreqs->pru_shutdown)(so);
 		CURVNET_RESTORE();
 		return (error);
 	}
+	CURVNET_RESTORE();
 	return (0);
 }
 
@@ -2290,6 +2334,8 @@ sorflush(struct socket *so)
 	struct protosw *pr = so->so_proto;
 	struct sockbuf asb;
 
+	VNET_SO_ASSERT(so);
+
 	/*
 	 * In order to avoid calling dom_dispose with the socket buffer mutex
 	 * held, and in order to generally avoid holding the lock for a long
@@ -2303,7 +2349,6 @@ sorflush(struct socket *so)
 	 * socket buffer.  Don't let our acquire be interrupted by a signal
 	 * despite any existing socket disposition on interruptable waiting.
 	 */
-	CURVNET_SET(so->so_vnet);
 	socantrcvmore(so);
 	(void) sblock(sb, SBL_WAIT | SBL_NOINTR);
 
@@ -2327,7 +2372,6 @@ sorflush(struct socket *so)
 	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL)
 		(*pr->pr_domain->dom_dispose)(asb.sb_mb);
 	sbrelease_internal(&asb, so);
-	CURVNET_RESTORE();
 }
 
 /*
@@ -2392,11 +2436,14 @@ sosetopt(struct socket *so, struct sockopt *sopt)
 	struct mac extmac;
 #endif
 
+	CURVNET_SET(so->so_vnet);
 	error = 0;
 	if (sopt->sopt_level != SOL_SOCKET) {
-		if (so->so_proto && so->so_proto->pr_ctloutput)
-			return ((*so->so_proto->pr_ctloutput)
-				  (so, sopt));
+		if (so->so_proto && so->so_proto->pr_ctloutput) {
+			error = (*so->so_proto->pr_ctloutput)(so, sopt);
+			CURVNET_RESTORE();
+			return (error);
+		}
 		error = ENOPROTOOPT;
 	} else {
 		switch (sopt->sopt_name) {
@@ -2592,6 +2639,7 @@ sosetopt(struct socket *so, struct sockopt *sopt)
 		}
 	}
 bad:
+	CURVNET_RESTORE();
 	return (error);
 }
 
@@ -2635,13 +2683,15 @@ sogetopt(struct socket *so, struct sockopt *sopt)
 	struct mac extmac;
 #endif
 
+	CURVNET_SET(so->so_vnet);
 	error = 0;
 	if (sopt->sopt_level != SOL_SOCKET) {
-		if (so->so_proto && so->so_proto->pr_ctloutput) {
-			return ((*so->so_proto->pr_ctloutput)
-				  (so, sopt));
-		} else
-			return (ENOPROTOOPT);
+		if (so->so_proto && so->so_proto->pr_ctloutput)
+			error = (*so->so_proto->pr_ctloutput)(so, sopt);
+		else
+			error = ENOPROTOOPT;
+		CURVNET_RESTORE();
+		return (error);
 	} else {
 		switch (sopt->sopt_name) {
 #ifdef INET
@@ -2725,11 +2775,11 @@ integer:
 			error = sooptcopyin(sopt, &extmac, sizeof(extmac),
 			    sizeof(extmac));
 			if (error)
-				return (error);
+				goto bad;
 			error = mac_getsockopt_label(sopt->sopt_td->td_ucred,
 			    so, &extmac);
 			if (error)
-				return (error);
+				goto bad;
 			error = sooptcopyout(sopt, &extmac, sizeof extmac);
 #else
 			error = EOPNOTSUPP;
@@ -2741,11 +2791,11 @@ integer:
 			error = sooptcopyin(sopt, &extmac, sizeof(extmac),
 			    sizeof(extmac));
 			if (error)
-				return (error);
+				goto bad;
 			error = mac_getsockopt_peerlabel(
 			    sopt->sopt_td->td_ucred, so, &extmac);
 			if (error)
-				return (error);
+				goto bad;
 			error = sooptcopyout(sopt, &extmac, sizeof extmac);
 #else
 			error = EOPNOTSUPP;
@@ -2768,8 +2818,12 @@ integer:
 			error = ENOPROTOOPT;
 			break;
 		}
-		return (error);
 	}
+#ifdef MAC
+bad:
+#endif
+	CURVNET_RESTORE();
+	return (error);
 }
 
 /* XXX; prepare mbuf for (__FreeBSD__ < 3) routines. */
@@ -2903,6 +2957,10 @@ sopoll(struct socket *so, int events, struct ucred *active_cred,
     struct thread *td)
 {
 
+	/*
+	 * We do not need to set or assert curvnet as long as everyone uses
+	 * sopoll_generic().
+	 */
 	return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred,
 	    td));
 }
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 958c8079caf9..d3326d45a709 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -260,14 +260,9 @@ listen(td, uap)
 		so = fp->f_data;
 #ifdef MAC
 		error = mac_socket_check_listen(td->td_ucred, so);
-		if (error == 0) {
+		if (error == 0)
 #endif
-			CURVNET_SET(so->so_vnet);
 			error = solisten(so, uap->backlog, td);
-			CURVNET_RESTORE();
-#ifdef MAC
-		}
-#endif
 		fdrop(fp, td);
 	}
 	return(error);
@@ -428,9 +423,7 @@ kern_accept(struct thread *td, int s, struct sockaddr **name,
 	tmp = fflag & FASYNC;
 	(void) fo_ioctl(nfp, FIOASYNC, &tmp, td->td_ucred, td);
 	sa = 0;
-	CURVNET_SET(so->so_vnet);
 	error = soaccept(so, &sa);
-	CURVNET_RESTORE();
 	if (error) {
 		/*
 		 * return a namelen of zero for older code which might
@@ -981,11 +974,9 @@ kern_recvit(td, s, mp, fromseg, controlp)
 		ktruio = cloneuio(&auio);
 #endif
 	len = auio.uio_resid;
-	CURVNET_SET(so->so_vnet);
 	error = soreceive(so, &fromsa, &auio, (struct mbuf **)0,
 	    (mp->msg_control || controlp) ? &control : (struct mbuf **)0,
 	    &mp->msg_flags);
-	CURVNET_RESTORE();
 	if (error) {
 		if (auio.uio_resid != (int)len && (error == ERESTART ||
 		    error == EINTR || error == EWOULDBLOCK))
@@ -1331,9 +1322,7 @@ kern_setsockopt(td, s, level, name, val, valseg, valsize)
 	error = getsock(td->td_proc->p_fd, s, &fp, NULL);
 	if (error == 0) {
 		so = fp->f_data;
-		CURVNET_SET(so->so_vnet);
 		error = sosetopt(so, &sopt);
-		CURVNET_RESTORE();
 		fdrop(fp, td);
 	}
 	return(error);
@@ -1412,9 +1401,7 @@ kern_getsockopt(td, s, level, name, val, valseg, valsize)
 	error = getsock(td->td_proc->p_fd, s, &fp, NULL);
 	if (error == 0) {
 		so = fp->f_data;
-		CURVNET_SET(so->so_vnet);
 		error = sogetopt(so, &sopt);
-		CURVNET_RESTORE();
 		*valsize = sopt.sopt_valsize;
 		fdrop(fp, td);
 	}
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index d049688ae54c..4bad3869fe85 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -1311,7 +1311,9 @@ unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	}
 	if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
 		if (so2->so_options & SO_ACCEPTCONN) {
+			CURVNET_SET(so2->so_vnet);
 			so3 = sonewconn(so2, 0);
+			CURVNET_RESTORE();
 		} else
 			so3 = NULL;
 		if (so3 == NULL) {
@@ -2215,8 +2217,14 @@ unp_gc(__unused void *arg, int pending)
 	 * struct files associated with these sockets but leave each socket
 	 * with one remaining ref.
 	 */
-	for (i = 0; i < total; i++)
-		sorflush(unref[i]->f_data);
+	for (i = 0; i < total; i++) {
+		struct socket *so;
+
+		so = unref[i]->f_data;
+		CURVNET_SET(so->so_vnet);
+		sorflush(so);
+		CURVNET_RESTORE();
+	}
 
 	/*
 	 * And finally release the sockets so they can be reclaimed.
diff --git a/sys/net/if.c b/sys/net/if.c
index ccd4689e9a45..2e21ae9b185d 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -2440,10 +2440,13 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 	int error;
 	int oif_flags;
 
+	CURVNET_SET(so->so_vnet);
 	switch (cmd) {
 	case SIOCGIFCONF:
 	case OSIOCGIFCONF:
-		return (ifconf(cmd, data));
+		error = ifconf(cmd, data);
+		CURVNET_RESTORE();
+		return (error);
 
 #ifdef COMPAT_FREEBSD32
 	case SIOCGIFCONF32:
@@ -2455,7 +2458,9 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 			ifc.ifc_len = ifc32->ifc_len;
 			ifc.ifc_buf = PTRIN(ifc32->ifc_buf);
 
-			return (ifconf(SIOCGIFCONF, (void *)&ifc));
+			error = ifconf(SIOCGIFCONF, (void *)&ifc);
+			CURVNET_RESTORE();
+			return (error);
 		}
 #endif
 	}
@@ -2465,42 +2470,55 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 #ifdef VIMAGE
 	case SIOCSIFRVNET:
 		error = priv_check(td, PRIV_NET_SETIFVNET);
-		if (error)
-			return (error);
-		return (if_vmove_reclaim(td, ifr->ifr_name, ifr->ifr_jid));
+		if (error == 0)
+			error = if_vmove_reclaim(td, ifr->ifr_name,
+			    ifr->ifr_jid);
+		CURVNET_RESTORE();
+		return (error);
 #endif
 	case SIOCIFCREATE:
 	case SIOCIFCREATE2:
 		error = priv_check(td, PRIV_NET_IFCREATE);
-		if (error)
-			return (error);
-		return (if_clone_create(ifr->ifr_name, sizeof(ifr->ifr_name),
-			cmd == SIOCIFCREATE2 ? ifr->ifr_data : NULL));
+		if (error == 0)
+			error = if_clone_create(ifr->ifr_name,
+			    sizeof(ifr->ifr_name),
+			    cmd == SIOCIFCREATE2 ? ifr->ifr_data : NULL);
+		CURVNET_RESTORE();
+		return (error);
 	case SIOCIFDESTROY:
 		error = priv_check(td, PRIV_NET_IFDESTROY);
-		if (error)
-			return (error);
-		return if_clone_destroy(ifr->ifr_name);
+		if (error == 0)
+			error = if_clone_destroy(ifr->ifr_name);
+		CURVNET_RESTORE();
+		return (error);
 
 	case SIOCIFGCLONERS:
-		return (if_clone_list((struct if_clonereq *)data));
+		error = if_clone_list((struct if_clonereq *)data);
+		CURVNET_RESTORE();
+		return (error);
 	case SIOCGIFGMEMB:
-		return (if_getgroupmembers((struct ifgroupreq *)data));
+		error = if_getgroupmembers((struct ifgroupreq *)data);
+		CURVNET_RESTORE();
+		return (error);
 	}
 
 	ifp = ifunit_ref(ifr->ifr_name);
-	if (ifp == NULL)
+	if (ifp == NULL) {
+		CURVNET_RESTORE();
 		return (ENXIO);
+	}
 
 	error = ifhwioctl(cmd, ifp, data, td);
 	if (error != ENOIOCTL) {
 		if_rele(ifp);
+		CURVNET_RESTORE();
 		return (error);
 	}
 
 	oif_flags = ifp->if_flags;
 	if (so->so_proto == NULL) {
 		if_rele(ifp);
+		CURVNET_RESTORE();
 		return (EOPNOTSUPP);
 	}
 #ifndef COMPAT_43
@@ -2575,6 +2593,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 #endif
 	}
 	if_rele(ifp);
+	CURVNET_RESTORE();
 	return (error);
 }
 
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
index 0ac24e54961a..efca538993c4 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
@@ -50,6 +50,9 @@
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
@@ -592,8 +595,11 @@ ng_btsocket_l2cap_process_l2ca_con_ind(struct ng_mesg *msg,
 		 * space then create new socket and set proper source address.
 		 */
 
-		if (pcb->so->so_qlen <= pcb->so->so_qlimit)
+		if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
+			CURVNET_SET(pcb->so->so_vnet);
 			so1 = sonewconn(pcb->so, 0);
+			CURVNET_RESTORE();
+		}
 
 		if (so1 == NULL) {
 			result = NG_L2CAP_NO_RESOURCES;
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
index 1fd9eb3b83dc..f07da3a5527e 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
@@ -52,6 +52,9 @@
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <sys/uio.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
@@ -1163,8 +1166,11 @@ ng_btsocket_rfcomm_connect_ind(ng_btsocket_rfcomm_session_p s, int channel)
 
 	mtx_lock(&pcb->pcb_mtx);
 
-	if (pcb->so->so_qlen <= pcb->so->so_qlimit)
+	if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
+		CURVNET_SET(pcb->so->so_vnet);
 		so1 = sonewconn(pcb->so, 0);
+		CURVNET_RESTORE();
+	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
diff --git a/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c b/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
index e1bff5767083..85b68dcb3f6f 100644
--- a/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
+++ b/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
@@ -50,6 +50,9 @@
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
@@ -477,8 +480,11 @@ ng_btsocket_sco_process_lp_con_ind(struct ng_mesg *msg,
 		 * space then create new socket and set proper source address.
 		 */
 
-		if (pcb->so->so_qlen <= pcb->so->so_qlimit)
+		if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
+			CURVNET_SET(pcb->so->so_vnet);
 			so1 = sonewconn(pcb->so, 0);
+			CURVNET_RESTORE();
+		}
 
 		if (so1 == NULL) {
 			status = 0x0d; /* Rejected due to limited resources */
diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c
index 39c9aa787872..1329c40e52d0 100644
--- a/sys/netinet/sctp_input.c
+++ b/sys/netinet/sctp_input.c
@@ -2675,8 +2675,10 @@ sctp_handle_cookie_echo(struct mbuf *m, int iphlen, int offset,
 			oso = (*inp_p)->sctp_socket;
 			atomic_add_int(&(*stcb)->asoc.refcnt, 1);
 			SCTP_TCB_UNLOCK((*stcb));
+			CURVNET_SET(oso->so_vnet);
 			so = sonewconn(oso, 0
 			    );
+			CURVNET_RESTORE();
 			SCTP_TCB_LOCK((*stcb));
 			atomic_subtract_int(&(*stcb)->asoc.refcnt, 1);
 
diff --git a/sys/netinet/sctp_peeloff.c b/sys/netinet/sctp_peeloff.c
index 748e23f2477c..548966d85110 100644
--- a/sys/netinet/sctp_peeloff.c
+++ b/sys/netinet/sctp_peeloff.c
@@ -167,8 +167,10 @@ sctp_get_peeloff(struct socket *head, sctp_assoc_t assoc_id, int *error)
 	}
 	atomic_add_int(&stcb->asoc.refcnt, 1);
 	SCTP_TCB_UNLOCK(stcb);
+	CURVNET_SET(head->so_vnet);
 	newso = sonewconn(head, SS_ISCONNECTED
 	    );
+	CURVNET_RESTORE();
 	if (newso == NULL) {
 		SCTPDBG(SCTP_DEBUG_PEEL1, "sctp_peeloff:sonewconn failed\n");
 		SCTP_LTRACE_ERR_RET(NULL, stcb, NULL, SCTP_FROM_SCTP_PEELOFF, ENOMEM);
diff --git a/sys/nfsclient/bootp_subr.c b/sys/nfsclient/bootp_subr.c
index cff72c7f2bc4..9fcd03f1fb19 100644
--- a/sys/nfsclient/bootp_subr.c
+++ b/sys/nfsclient/bootp_subr.c
@@ -584,8 +584,6 @@ bootpc_call(struct bootpc_globalcontext *gctx, struct thread *td)
 	int retry;
 	const char *s;
 
-	CURVNET_SET(TD_TO_VNET(td));
-
 	/*
 	 * Create socket and set its recieve timeout.
 	 */
@@ -962,7 +960,6 @@ gotreply:
 out:
 	soclose(so);
 out0:
-	CURVNET_RESTORE();
 	return error;
 }
 
@@ -977,8 +974,6 @@ bootpc_fakeup_interface(struct bootpc_ifcontext *ifctx,
 	struct ifaddr *ifa;
 	struct sockaddr_dl *sdl;
 
-	CURVNET_SET(TD_TO_VNET(td));
-
 	error = socreate(AF_INET, &ifctx->so, SOCK_DGRAM, 0, td->td_ucred, td);
 	if (error != 0)
 		panic("nfs_boot: socreate, error=%d", error);
@@ -1053,8 +1048,6 @@ bootpc_fakeup_interface(struct bootpc_ifcontext *ifctx,
 		      ifctx->ireq.ifr_name);
 	ifctx->sdl = sdl;
 
-	CURVNET_RESTORE();
-
 	return error;
 }
 
@@ -1630,6 +1623,7 @@ bootpc_init(void)
 	/*
 	 * Find a network interface.
 	 */
+	CURVNET_SET(TD_TO_VNET(td));
 #ifdef BOOTP_WIRED_TO
 	printf("bootpc_init: wired to interface '%s'\n",
 	       __XSTRING(BOOTP_WIRED_TO));
@@ -1676,6 +1670,7 @@ bootpc_init(void)
 		ifctx = ifctx->next;
 	}
 	IFNET_RUNLOCK();
+	CURVNET_RESTORE();
 
 	if (gctx->interfaces == NULL || gctx->interfaces->ifp == NULL) {
 #ifdef BOOTP_WIRED_TO
diff --git a/sys/nfsclient/krpc_subr.c b/sys/nfsclient/krpc_subr.c
index c2a84506390b..0639b6c79e05 100644
--- a/sys/nfsclient/krpc_subr.c
+++ b/sys/nfsclient/krpc_subr.c
@@ -216,8 +216,6 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 	nam = mhead = NULL;
 	from = NULL;
 
-	CURVNET_SET(TD_TO_VNET(td));
-
 	/*
 	 * Create socket and set its recieve timeout.
 	 */
@@ -430,7 +428,6 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 	if (mhead) m_freem(mhead);
 	if (from) free(from, M_SONAME);
 	soclose(so);
-	CURVNET_RESTORE();
 	return error;
 }
 
diff --git a/sys/nfsclient/nfs_diskless.c b/sys/nfsclient/nfs_diskless.c
index aacecfff961b..b574945e7d16 100644
--- a/sys/nfsclient/nfs_diskless.c
+++ b/sys/nfsclient/nfs_diskless.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/jail.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mount.h>
@@ -202,6 +203,7 @@ nfs_setup_diskless(void)
 		return;
 	}
 	ifa = NULL;
+	CURVNET_SET(TD_TO_VNET(curthread));
 	IFNET_RLOCK();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -213,12 +215,14 @@ nfs_setup_diskless(void)
 					  LLADDR(&ourdl),
 					  sdl->sdl_alen)) {
 				    IFNET_RUNLOCK();
+				    CURVNET_RESTORE();
 				    goto match_done;
 				}
 			}
 		}
 	}
 	IFNET_RUNLOCK();
+	CURVNET_RESTORE();
 	printf("nfs_diskless: no interface\n");
 	return;	/* no matching interface */
 match_done:
diff --git a/sys/nfsclient/nfs_vfsops.c b/sys/nfsclient/nfs_vfsops.c
index fc54ff69df0f..3700708a28f5 100644
--- a/sys/nfsclient/nfs_vfsops.c
+++ b/sys/nfsclient/nfs_vfsops.c
@@ -428,7 +428,6 @@ nfs_mountroot(struct mount *mp)
 	char buf[128];
 	char *cp;
 
-	CURVNET_SET(TD_TO_VNET(td));
 
 #if defined(BOOTP_NFSROOT) && defined(BOOTP)
 	bootpc_init();		/* use bootp to get nfs_diskless filled in */
@@ -437,7 +436,6 @@ nfs_mountroot(struct mount *mp)
 #endif
 
 	if (nfs_diskless_valid == 0) {
-		CURVNET_RESTORE();
 		return (-1);
 	}
 	if (nfs_diskless_valid == 1)
@@ -504,10 +502,12 @@ nfs_mountroot(struct mount *mp)
 		sin.sin_family = AF_INET;
 		sin.sin_len = sizeof(sin);
                 /* XXX MRT use table 0 for this sort of thing */
+		CURVNET_SET(TD_TO_VNET(td));
 		error = rtrequest(RTM_ADD, (struct sockaddr *)&sin,
 		    (struct sockaddr *)&nd->mygateway,
 		    (struct sockaddr *)&mask,
 		    RTF_UP | RTF_GATEWAY, NULL);
+		CURVNET_RESTORE();
 		if (error)
 			panic("nfs_mountroot: RTM_ADD: %d", error);
 	}
@@ -525,7 +525,6 @@ nfs_mountroot(struct mount *mp)
 	nd->root_args.hostname = buf;
 	if ((error = nfs_mountdiskless(buf,
 	    &nd->root_saddr, &nd->root_args, td, &vp, mp)) != 0) {
-		CURVNET_RESTORE();
 		return (error);
 	}
 
@@ -539,7 +538,6 @@ nfs_mountroot(struct mount *mp)
 	    sizeof (prison0.pr_hostname));
 	mtx_unlock(&prison0.pr_mtx);
 	inittodr(ntohl(nd->root_time));
-	CURVNET_RESTORE();
 	return (0);
 }
 
diff --git a/sys/rpc/clnt_dg.c b/sys/rpc/clnt_dg.c
index d35a6a944861..58d55ffe2531 100644
--- a/sys/rpc/clnt_dg.c
+++ b/sys/rpc/clnt_dg.c
@@ -200,14 +200,11 @@ clnt_dg_create(
 		return (NULL);
 	}
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
 		rpc_createerr.cf_stat = RPC_TLIERROR;
 		rpc_createerr.cf_error.re_errno = 0;
-		CURVNET_RESTORE();
 		return (NULL);
 	}
-	CURVNET_RESTORE();
 
 	/*
 	 * Find the receive and the send size
diff --git a/sys/rpc/clnt_vc.c b/sys/rpc/clnt_vc.c
index 98824070b9cc..9f36bba93978 100644
--- a/sys/rpc/clnt_vc.c
+++ b/sys/rpc/clnt_vc.c
@@ -220,9 +220,7 @@ clnt_vc_create(
 		}
 	}
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
-		CURVNET_RESTORE();
 		goto err;
 	}
 
@@ -245,7 +243,6 @@ clnt_vc_create(
 		sopt.sopt_valsize = sizeof(one);
 		sosetopt(so, &sopt);
 	}
-	CURVNET_RESTORE();
 
 	ct->ct_closeit = FALSE;
 
diff --git a/sys/rpc/rpc_generic.c b/sys/rpc/rpc_generic.c
index 1c6aa821123a..fd39350ae4dc 100644
--- a/sys/rpc/rpc_generic.c
+++ b/sys/rpc/rpc_generic.c
@@ -187,7 +187,9 @@ __rpc_socket2sockinfo(struct socket *so, struct __rpc_sockinfo *sip)
 	struct sockopt opt;
 	int error;
 
+	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
+	CURVNET_RESTORE();
 	if (error)
 		return 0;
 
@@ -825,7 +827,6 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 	sa->sa_len = salen;
 
 	if (*portp == 0) {
-		CURVNET_SET(so->so_vnet);
 		bzero(&opt, sizeof(opt));
 		opt.sopt_dir = SOPT_GET;
 		opt.sopt_level = proto;
@@ -834,14 +835,12 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 		opt.sopt_valsize = sizeof(old);
 		error = sogetopt(so, &opt);
 		if (error) {
-			CURVNET_RESTORE();
 			goto out;
 		}
 
 		opt.sopt_dir = SOPT_SET;
 		opt.sopt_val = &portlow;
 		error = sosetopt(so, &opt);
-		CURVNET_RESTORE();
 		if (error)
 			goto out;
 	}
@@ -852,9 +851,7 @@ bindresvport(struct socket *so, struct sockaddr *sa)
 		if (error) {
 			opt.sopt_dir = SOPT_SET;
 			opt.sopt_val = &old;
-			CURVNET_SET(so->so_vnet);
 			sosetopt(so, &opt);
-			CURVNET_RESTORE();
 		}
 	}
 out:
diff --git a/sys/rpc/svc_dg.c b/sys/rpc/svc_dg.c
index 9d7696b0145c..adfe2b51f87c 100644
--- a/sys/rpc/svc_dg.c
+++ b/sys/rpc/svc_dg.c
@@ -103,10 +103,8 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	struct sockaddr* sa;
 	int error;
 
-	CURVNET_SET(so->so_vnet);
 	if (!__rpc_socket2sockinfo(so, &si)) {
 		printf(svc_dg_str, svc_dg_err1);
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 	/*
@@ -116,7 +114,6 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	recvsize = __rpc_get_t_size(si.si_af, si.si_proto, (int)recvsize);
 	if ((sendsize == 0) || (recvsize == 0)) {
 		printf(svc_dg_str, svc_dg_err2);
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 
@@ -128,6 +125,7 @@ svc_dg_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt->xp_p2 = NULL;
 	xprt->xp_ops = &svc_dg_ops;
 
+	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
 	CURVNET_RESTORE();
 	if (error)
diff --git a/sys/rpc/svc_generic.c b/sys/rpc/svc_generic.c
index e6e8acdb1d93..f8b70de00da8 100644
--- a/sys/rpc/svc_generic.c
+++ b/sys/rpc/svc_generic.c
@@ -230,14 +230,11 @@ svc_tli_create(
 		/*
 		 * It is an open socket. Get the transport info.
 		 */
-		CURVNET_SET(so->so_vnet);
 		if (!__rpc_socket2sockinfo(so, &si)) {
 			printf(
 		"svc_tli_create: could not get transport information\n");
-			CURVNET_RESTORE();
 			return (NULL);
 		}
-		CURVNET_RESTORE();
 	}
 
 	/*
@@ -264,9 +261,7 @@ svc_tli_create(
 		"svc_tli_create: could not bind to requested address\n");
 				goto freedata;
 			}
-			CURVNET_SET(so->so_vnet);
 			solisten(so, (int)bindaddr->qlen, curthread);
-			CURVNET_RESTORE();
 		}
 			
 	}
diff --git a/sys/rpc/svc_vc.c b/sys/rpc/svc_vc.c
index 999cad107fa1..fddcf586a08e 100644
--- a/sys/rpc/svc_vc.c
+++ b/sys/rpc/svc_vc.c
@@ -154,10 +154,8 @@ svc_vc_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt->xp_p2 = NULL;
 	xprt->xp_ops = &svc_vc_rendezvous_ops;
 
-	CURVNET_SET(so->so_vnet);
 	error = so->so_proto->pr_usrreqs->pru_sockaddr(so, &sa);
 	if (error) {
-		CURVNET_RESTORE();
 		goto cleanup_svc_vc_create;
 	}
 
@@ -167,7 +165,6 @@ svc_vc_create(SVCPOOL *pool, struct socket *so, size_t sendsize,
 	xprt_register(xprt);
 
 	solisten(so, SOMAXCONN, curthread);
-	CURVNET_RESTORE();
 
 	SOCKBUF_LOCK(&so->so_rcv);
 	xprt->xp_upcallset = 1;
@@ -200,10 +197,8 @@ svc_vc_create_conn(SVCPOOL *pool, struct socket *so, struct sockaddr *raddr)
 	opt.sopt_name = SO_KEEPALIVE;
 	opt.sopt_val = &one;
 	opt.sopt_valsize = sizeof(one);
-	CURVNET_SET(so->so_vnet);
 	error = sosetopt(so, &opt);
 	if (error) {
-		CURVNET_RESTORE();
 		return (NULL);
 	}
 
@@ -216,11 +211,9 @@ svc_vc_create_conn(SVCPOOL *pool, struct socket *so, struct sockaddr *raddr)
 		opt.sopt_valsize = sizeof(one);
 		error = sosetopt(so, &opt);
 		if (error) {
-			CURVNET_RESTORE();
 			return (NULL);
 		}
 	}
-	CURVNET_RESTORE();
 
 	cd = mem_alloc(sizeof(*cd));
 	cd->strm_stat = XPRT_IDLE;
@@ -635,10 +628,8 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
 		uio.uio_td = curthread;
 		m = NULL;
 		rcvflag = MSG_DONTWAIT;
-		CURVNET_SET(xprt->xp_socket->so_vnet);
 		error = soreceive(xprt->xp_socket, NULL, &uio, &m, NULL,
 		    &rcvflag);
-		CURVNET_RESTORE();
 
 		if (error == EWOULDBLOCK) {
 			/*