Rework locking in BPF code to remove rwlock from fast path.

On high packets rate the contention on rwlock in bpf_*tap*() functions
can lead to packets dropping. To avoid this, migrate this code to use
epoch(9) KPI and ConcurrencyKit's lists.

* all lists changed to use CK_LIST;
* reference counting added to bpf_if and bpf_d;
* now bpf_if references ifnet and releases this reference on destroy;
* each bpf_d descriptor references bpf_if when it is attached;
* new struct bpf_program_buffer introduced to keep BPF filter programs;
* bpf_program_buffer, bpf_d and bpf_if structures are freed by
  epoch_call();
* bpf_freelist and ifnet_departure event are no longer needed, thus
  both are removed;

Reviewed by:	melifaro
Sponsored by:	Yandex LLC
Differential Revision:	https://reviews.freebsd.org/D20224

1 files changed, 6 insertions, 2 deletions

diff --git a/sys/net/bpfdesc.h b/sys/net/bpfdesc.h
index 7d23e06ec3a3..be978280311d 100644
--- a/sys/net/bpfdesc.h
+++ b/sys/net/bpfdesc.h
@@ -43,9 +43,10 @@
 
 #include <sys/callout.h>
 #include <sys/selinfo.h>
-#include <sys/queue.h>
+#include <sys/ck.h>
 #include <sys/conf.h>
 #include <sys/counter.h>
+#include <sys/epoch.h>
 #include <net/if.h>
 
 /*
@@ -53,7 +54,7 @@
  */
 struct zbuf;
 struct bpf_d {
-	LIST_ENTRY(bpf_d) bd_next;	/* Linked list of descriptors */
+	CK_LIST_ENTRY(bpf_d) bd_next;	/* Linked list of descriptors */
 	/*
 	 * Buffer slots: two memory buffers store the incoming packets.
 	 *   The model has three slots.  Sbuf is always occupied.
@@ -104,6 +105,9 @@ struct bpf_d {
 	counter_u64_t	bd_wdcount;	/* number of packets dropped during a write */
 	counter_u64_t	bd_zcopy;	/* number of zero copy operations */
 	u_char		bd_compat32;	/* 32-bit stream on LP64 system */
+
+	volatile u_int	bd_refcnt;
+	struct epoch_context epoch_ctx;
 };
 
 /* Values for bd_state */