bhyve: support for enabling/disabling the net backend

Extend the net backend interface with two functions, namely netbe_rx_disable()
and netbe_rx_enable(), which can be used by the net device emulators to stop
the backend from invoking the receive callback. This is useful for device
emulators, i.e., on hardware resets or to implement receive backpressure.
The mevent module has been extendede to support the addition of a disabled
event. To prevent race conditions, the net backends will start with receive
operation disabled. A follow-up patch will use the new functionalities in
the virtio-net device.

Reviewed by:	jhb, markj
MFC after:	1 week
Differential Revision: https://reviews.freebsd.org/D20973

3 files changed, 87 insertions, 8 deletions

diff --git a/usr.sbin/bhyve/mevent.c b/usr.sbin/bhyve/mevent.c
index c40def6176d4..60289df6c9a2 100644
--- a/usr.sbin/bhyve/mevent.c
+++ b/usr.sbin/bhyve/mevent.c
@@ -62,10 +62,11 @@ __FBSDID("$FreeBSD$");
 
 #define	MEVENT_MAX	64
 
-#define	MEV_ADD		1
-#define	MEV_ENABLE	2
-#define	MEV_DISABLE	3
-#define	MEV_DEL_PENDING	4
+#define	MEV_ADD			1
+#define	MEV_ENABLE		2
+#define	MEV_DISABLE		3
+#define	MEV_DEL_PENDING		4
+#define	MEV_ADD_DISABLED	5
 
 extern char *vmname;
 
@@ -161,6 +162,9 @@ mevent_kq_flags(struct mevent *mevp)
 	case MEV_ADD:
 		ret = EV_ADD;		/* implicitly enabled */
 		break;
+	case MEV_ADD_DISABLED:
+		ret = EV_ADD | EV_DISABLE;
+		break;
 	case MEV_ENABLE:
 		ret = EV_ENABLE;
 		break;
@@ -249,9 +253,10 @@ mevent_handle(struct kevent *kev, int numev)
 	}
 }
 
-struct mevent *
-mevent_add(int tfd, enum ev_type type,
-	   void (*func)(int, enum ev_type, void *), void *param)
+static struct mevent *
+mevent_add_state(int tfd, enum ev_type type,
+	   void (*func)(int, enum ev_type, void *), void *param,
+	   int state)
 {
 	struct mevent *lp, *mevp;
 
@@ -299,7 +304,7 @@ mevent_add(int tfd, enum ev_type type,
 
 	LIST_INSERT_HEAD(&change_head, mevp, me_list);
 	mevp->me_cq = 1;
-	mevp->me_state = MEV_ADD;
+	mevp->me_state = state;
 	mevent_notify();
 
 exit:
@@ -308,6 +313,14 @@ exit:
 	return (mevp);
 }
 
+struct mevent *
+mevent_add(int tfd, enum ev_type type,
+	   void (*func)(int, enum ev_type, void *), void *param)
+{
+
+	return mevent_add_state(tfd, type, func, param, MEV_ADD);
+}
+
 static int
 mevent_update(struct mevent *evp, int newstate)
 {
diff --git a/usr.sbin/bhyve/net_backends.c b/usr.sbin/bhyve/net_backends.c
index 88afaca4b1c4..a94dfca54fb7 100644
--- a/usr.sbin/bhyve/net_backends.c
+++ b/usr.sbin/bhyve/net_backends.c
@@ -110,6 +110,17 @@ struct net_backend {
 	ssize_t (*recv)(struct net_backend *be, struct iovec *iov, int iovcnt);
 
 	/*
+	 * Ask the backend to enable or disable receive operation in the
+	 * backend. On return from a disable operation, it is guaranteed
+	 * that the receive callback won't be called until receive is
+	 * enabled again. Note however that it is up to the caller to make
+	 * sure that netbe_recv() is not currently being executed by another
+	 * thread.
+	 */
+	void (*recv_enable)(struct net_backend *be);
+	void (*recv_disable)(struct net_backend *be);
+
+	/*
 	 * Ask the backend for the virtio-net features it is able to
 	 * support. Possible features are TSO, UFO and checksum offloading
 	 * in both rx and tx direction and for both IPv4 and IPv6.
@@ -248,6 +259,22 @@ tap_recv(struct net_backend *be, struct iovec *iov, int iovcnt)
 	return (ret);
 }
 
+static void
+tap_recv_enable(struct net_backend *be)
+{
+	struct tap_priv *priv = (struct tap_priv *)be->opaque;
+
+	mevent_enable(priv->mevp);
+}
+
+static void
+tap_recv_disable(struct net_backend *be)
+{
+	struct tap_priv *priv = (struct tap_priv *)be->opaque;
+
+	mevent_disable(priv->mevp);
+}
+
 static uint64_t
 tap_get_cap(struct net_backend *be)
 {
@@ -270,6 +297,8 @@ static struct net_backend tap_backend = {
 	.cleanup = tap_cleanup,
 	.send = tap_send,
 	.recv = tap_recv,
+	.recv_enable = tap_recv_enable,
+	.recv_disable = tap_recv_disable,
 	.get_cap = tap_get_cap,
 	.set_cap = tap_set_cap,
 };
@@ -282,6 +311,8 @@ static struct net_backend vmnet_backend = {
 	.cleanup = tap_cleanup,
 	.send = tap_send,
 	.recv = tap_recv,
+	.recv_enable = tap_recv_enable,
+	.recv_disable = tap_recv_disable,
 	.get_cap = tap_get_cap,
 	.set_cap = tap_set_cap,
 };
@@ -571,6 +602,22 @@ netmap_recv(struct net_backend *be, struct iovec *iov, int iovcnt)
 	return (totlen);
 }
 
+static void
+netmap_recv_enable(struct net_backend *be)
+{
+	struct netmap_priv *priv = (struct netmap_priv *)be->opaque;
+
+	mevent_enable(priv->mevp);
+}
+
+static void
+netmap_recv_disable(struct net_backend *be)
+{
+	struct netmap_priv *priv = (struct netmap_priv *)be->opaque;
+
+	mevent_disable(priv->mevp);
+}
+
 static struct net_backend netmap_backend = {
 	.prefix = "netmap",
 	.priv_size = sizeof(struct netmap_priv),
@@ -578,6 +625,8 @@ static struct net_backend netmap_backend = {
 	.cleanup = netmap_cleanup,
 	.send = netmap_send,
 	.recv = netmap_recv,
+	.recv_enable = netmap_recv_enable,
+	.recv_disable = netmap_recv_disable,
 	.get_cap = netmap_get_cap,
 	.set_cap = netmap_set_cap,
 };
@@ -590,6 +639,8 @@ static struct net_backend vale_backend = {
 	.cleanup = netmap_cleanup,
 	.send = netmap_send,
 	.recv = netmap_recv,
+	.recv_enable = netmap_recv_enable,
+	.recv_disable = netmap_recv_disable,
 	.get_cap = netmap_get_cap,
 	.set_cap = netmap_set_cap,
 };
@@ -805,3 +856,16 @@ netbe_rx_discard(struct net_backend *be)
 	return netbe_recv(be, &iov, 1);
 }
 
+void
+netbe_rx_disable(struct net_backend *be)
+{
+
+	return be->recv_enable(be);
+}
+
+void
+netbe_rx_enable(struct net_backend *be)
+{
+
+	return be->recv_disable(be);
+}
diff --git a/usr.sbin/bhyve/net_backends.h b/usr.sbin/bhyve/net_backends.h
index bba39db59b83..b27f715164a9 100644
--- a/usr.sbin/bhyve/net_backends.h
+++ b/usr.sbin/bhyve/net_backends.h
@@ -46,6 +46,8 @@ int	 netbe_set_cap(net_backend_t *be, uint64_t cap,
 ssize_t	netbe_send(net_backend_t *be, struct iovec *iov, int iovcnt);
 ssize_t	netbe_recv(net_backend_t *be, struct iovec *iov, int iovcnt);
 ssize_t	netbe_rx_discard(net_backend_t *be);
+void	netbe_rx_disable(net_backend_t *be);
+void	netbe_rx_enable(net_backend_t *be);
 
 
 /*