import unbound 1.5.0

8 files changed, 280 insertions, 120 deletions

diff --git a/services/cache/dns.c b/services/cache/dns.c
index f2a04a227cb9..c663b8e8b9a2 100644
--- a/services/cache/dns.c
+++ b/services/cache/dns.c
@@ -795,3 +795,22 @@ dns_cache_store(struct module_env* env, struct query_info* msgqinf,
 	}
 	return 1;
 }
+
+int 
+dns_cache_prefetch_adjust(struct module_env* env, struct query_info* qinfo,
+        time_t adjust)
+{
+	struct msgreply_entry* msg;
+	msg = msg_cache_lookup(env, qinfo->qname, qinfo->qname_len,
+		qinfo->qtype, qinfo->qclass, *env->now, 1);
+	if(msg) {
+		struct reply_info* rep = (struct reply_info*)msg->entry.data;
+		if(rep) {
+			rep->prefetch_ttl += adjust;
+			lock_rw_unlock(&msg->entry.lock);
+			return 1;
+		}
+		lock_rw_unlock(&msg->entry.lock);
+	}
+	return 0;
+}
diff --git a/services/cache/dns.h b/services/cache/dns.h
index a7a6190cffba..05a3e6296543 100644
--- a/services/cache/dns.h
+++ b/services/cache/dns.h
@@ -179,4 +179,16 @@ struct dns_msg* dns_msg_create(uint8_t* qname, size_t qnamelen, uint16_t qtype,
 int dns_msg_authadd(struct dns_msg* msg, struct regional* region, 
 	struct ub_packed_rrset_key* rrset, time_t now);
 
+/**
+ * Adjust the prefetch_ttl for a cached message.  This adds a value to the
+ * prefetch ttl - postponing the time when it will be prefetched for future
+ * incoming queries.
+ * @param env: module environment with caches and time.
+ * @param qinfo: query info for the query that needs adjustment.
+ * @param adjust: time in seconds to add to the prefetch_leeway.
+ * @return false if not in cache. true if added.
+ */
+int dns_cache_prefetch_adjust(struct module_env* env, struct query_info* qinfo,
+        time_t adjust);
+
 #endif /* SERVICES_CACHE_DNS_H */
diff --git a/services/listen_dnsport.c b/services/listen_dnsport.c
index 8b1d62e3a209..b7ffb6d3fad3 100644
--- a/services/listen_dnsport.c
+++ b/services/listen_dnsport.c
@@ -57,7 +57,7 @@
 #include <fcntl.h>
 
 /** number of queued TCP connections for listen() */
-#define TCP_BACKLOG 5 
+#define TCP_BACKLOG 256 
 
 /**
  * Debug print of the getaddrinfo returned address.
@@ -153,8 +153,8 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 #endif
 		}
 #endif /* SO_REUSEADDR */
-#if defined(__linux__) && defined(SO_REUSEPORT)
-		/* Linux specific: try to set SO_REUSEPORT so that incoming
+#ifdef SO_REUSEPORT
+		/* try to set SO_REUSEPORT so that incoming
 		 * queries are distributed evenly among the receiving threads.
 		 * Each thread must have its own socket bound to the same port,
 		 * with SO_REUSEPORT set on each socket.
@@ -172,7 +172,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		}
 #else
 		(void)reuseport;
-#endif /* defined(__linux__) && defined(SO_REUSEPORT) */
+#endif /* defined(SO_REUSEPORT) */
 	}
 	if(rcv) {
 #ifdef SO_RCVBUF
@@ -362,11 +362,26 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 # endif /* IPv6 MTU */
 	} else if(family == AF_INET) {
 #  if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT)
+/* linux 3.15 has IP_PMTUDISC_OMIT, Hannes Frederic Sowa made it so that
+ * PMTU information is not accepted, but fragmentation is allowed
+ * if and only if the packet size exceeds the outgoing interface MTU
+ * (and also uses the interface mtu to determine the size of the packets).
+ * So there won't be any EMSGSIZE error.  Against DNS fragmentation attacks.
+ * FreeBSD already has same semantics without setting the option. */
+#    if defined(IP_PMTUDISC_OMIT)
+		int action = IP_PMTUDISC_OMIT;
+#    else
 		int action = IP_PMTUDISC_DONT;
+#    endif
 		if (setsockopt(s, IPPROTO_IP, IP_MTU_DISCOVER, 
 			&action, (socklen_t)sizeof(action)) < 0) {
 			log_err("setsockopt(..., IP_MTU_DISCOVER, "
-				"IP_PMTUDISC_DONT...) failed: %s",
+#    if defined(IP_PMTUDISC_OMIT)
+				"IP_PMTUDISC_OMIT"
+#    else
+				"IP_PMTUDISC_DONT"
+#    endif
+				"...) failed: %s",
 				strerror(errno));
 #    ifndef USE_WINSOCK
 			close(s);
@@ -404,8 +419,7 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		if(family==AF_INET6 && errno==EINVAL)
 			*noproto = 1;
 		else if(errno != EADDRINUSE) {
-			log_err("can't bind socket: %s", strerror(errno));
-			log_addr(0, "failed address",
+			log_err_addr("can't bind socket", strerror(errno),
 				(struct sockaddr_storage*)addr, addrlen);
 		}
 #endif /* EADDRINUSE */
@@ -413,9 +427,8 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 #else /* USE_WINSOCK */
 		if(WSAGetLastError() != WSAEADDRINUSE &&
 			WSAGetLastError() != WSAEADDRNOTAVAIL) {
-			log_err("can't bind socket: %s", 
-				wsa_strerror(WSAGetLastError()));
-			log_addr(0, "failed address",
+			log_err_addr("can't bind socket", 
+				wsa_strerror(WSAGetLastError()),
 				(struct sockaddr_storage*)addr, addrlen);
 		}
 		closesocket(s);
@@ -478,8 +491,8 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
 		return -1;
 	}
 #endif /* SO_REUSEADDR */
-#if defined(__linux__) && defined(SO_REUSEPORT)
-	/* Linux specific: try to set SO_REUSEPORT so that incoming
+#ifdef SO_REUSEPORT
+	/* try to set SO_REUSEPORT so that incoming
 	 * connections are distributed evenly among the receiving threads.
 	 * Each thread must have its own socket bound to the same port,
 	 * with SO_REUSEPORT set on each socket.
@@ -497,7 +510,7 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
 	}
 #else
 	(void)reuseport;
-#endif /* defined(__linux__) && defined(SO_REUSEPORT) */
+#endif /* defined(SO_REUSEPORT) */
 #if defined(IPV6_V6ONLY)
 	if(addr->ai_family == AF_INET6 && v6only) {
 		if(setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, 
@@ -523,16 +536,14 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto,
 		if(addr->ai_family==AF_INET6 && errno==EINVAL)
 			*noproto = 1;
 		else {
-			log_err("can't bind socket: %s", strerror(errno));
-			log_addr(0, "failed address",
+			log_err_addr("can't bind socket", strerror(errno),
 				(struct sockaddr_storage*)addr->ai_addr,
 				addr->ai_addrlen);
 		}
 		close(s);
 #else
-		log_err("can't bind socket: %s", 
-			wsa_strerror(WSAGetLastError()));
-		log_addr(0, "failed address",
+		log_err_addr("can't bind socket", 
+			wsa_strerror(WSAGetLastError()),
 			(struct sockaddr_storage*)addr->ai_addr,
 			addr->ai_addrlen);
 		closesocket(s);
@@ -837,7 +848,7 @@ listen_cp_insert(struct comm_point* c, struct listen_dnsport* front)
 struct listen_dnsport* 
 listen_create(struct comm_base* base, struct listen_port* ports,
 	size_t bufsize, int tcp_accept_count, void* sslctx,
-	comm_point_callback_t* cb, void *cb_arg)
+	struct dt_env* dtenv, comm_point_callback_t* cb, void *cb_arg)
 {
 	struct listen_dnsport* front = (struct listen_dnsport*)
 		malloc(sizeof(struct listen_dnsport));
@@ -871,6 +882,7 @@ listen_create(struct comm_base* base, struct listen_port* ports,
 			listen_delete(front);
 			return NULL;
 		}
+		cp->dtenv = dtenv;
 		cp->do_not_close = 1;
 		if(!listen_cp_insert(cp, front)) {
 			log_err("malloc failed");
diff --git a/services/listen_dnsport.h b/services/listen_dnsport.h
index 61fb9a0b44e8..075f6d281d5d 100644
--- a/services/listen_dnsport.h
+++ b/services/listen_dnsport.h
@@ -129,6 +129,7 @@ void listening_ports_free(struct listen_port* list);
  * @param tcp_accept_count: max number of simultaneous TCP connections 
  * 	from clients.
  * @param sslctx: nonNULL if ssl context.
+ * @param dtenv: nonNULL if dnstap enabled.
  * @param cb: callback function when a request arrives. It is passed
  *	  the packet and user argument. Return true to send a reply.
  * @param cb_arg: user data argument for callback function.
@@ -136,7 +137,8 @@ void listening_ports_free(struct listen_port* list);
  */
 struct listen_dnsport* listen_create(struct comm_base* base,
 	struct listen_port* ports, size_t bufsize, int tcp_accept_count,
-	void* sslctx, comm_point_callback_t* cb, void* cb_arg);
+	void* sslctx, struct dt_env *dtenv, comm_point_callback_t* cb,
+	void* cb_arg);
 
 /**
  * delete the listening structure
diff --git a/services/localzone.c b/services/localzone.c
index ac889799b430..d285a127cbbf 100644
--- a/services/localzone.c
+++ b/services/localzone.c
@@ -594,6 +594,8 @@ lz_enter_defaults(struct local_zones* zones, struct config_file* cfg)
 
 	/* this list of zones is from RFC 6303 */
 
+	/* block localhost level zones, first, later the LAN zones */
+
 	/* localhost. zone */
 	if(!lz_exists(zones, "localhost.") &&
 		!lz_nodefault(cfg, "localhost.")) {
@@ -650,6 +652,14 @@ lz_enter_defaults(struct local_zones* zones, struct config_file* cfg)
 		}
 		lock_rw_unlock(&z->lock);
 	}
+
+	/* if unblock lan-zones, then do not add the zones below.
+	 * we do add the zones above, about 127.0.0.1, because localhost is
+	 * not on the lan. */
+	if(cfg->unblock_lan_zones)
+		return 1;
+
+	/* block LAN level zones */
 	if (	!add_as112_default(zones, cfg, "10.in-addr.arpa.") ||
 		!add_as112_default(zones, cfg, "16.172.in-addr.arpa.") ||
 		!add_as112_default(zones, cfg, "17.172.in-addr.arpa.") ||
@@ -669,6 +679,70 @@ lz_enter_defaults(struct local_zones* zones, struct config_file* cfg)
 		!add_as112_default(zones, cfg, "31.172.in-addr.arpa.") ||
 		!add_as112_default(zones, cfg, "168.192.in-addr.arpa.") ||
 		!add_as112_default(zones, cfg, "0.in-addr.arpa.") ||
+		!add_as112_default(zones, cfg, "64.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "65.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "66.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "67.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "68.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "69.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "70.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "71.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "72.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "73.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "74.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "75.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "76.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "77.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "78.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "79.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "80.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "81.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "82.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "83.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "84.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "85.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "86.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "87.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "88.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "89.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "90.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "91.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "92.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "93.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "94.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "95.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "96.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "97.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "98.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "99.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "100.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "101.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "102.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "103.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "104.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "105.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "106.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "107.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "108.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "109.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "110.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "111.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "112.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "113.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "114.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "115.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "116.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "117.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "118.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "119.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "120.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "121.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "122.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "123.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "124.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "125.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "126.100.in-addr.arpa.") ||
+      		!add_as112_default(zones, cfg, "127.100.in-addr.arpa.") ||
 		!add_as112_default(zones, cfg, "254.169.in-addr.arpa.") ||
 		!add_as112_default(zones, cfg, "2.0.192.in-addr.arpa.") ||
 		!add_as112_default(zones, cfg, "100.51.198.in-addr.arpa.") ||
diff --git a/services/modstack.c b/services/modstack.c
index 56515a61f369..49bb2fd15adf 100644
--- a/services/modstack.c
+++ b/services/modstack.c
@@ -43,6 +43,7 @@
 #include "services/modstack.h"
 #include "util/module.h"
 #include "util/fptr_wlist.h"
+#include "dns64/dns64.h"
 #include "iterator/iterator.h"
 #include "validator/validator.h"
 
@@ -59,12 +60,12 @@ count_modules(const char* s)
                 return 0;
         while(*s) {
                 /* skip whitespace */
-                while(*s && isspace((int)*s))
+                while(*s && isspace((unsigned char)*s))
                         s++;
-                if(*s && !isspace((int)*s)) {
+                if(*s && !isspace((unsigned char)*s)) {
                         /* skip identifier */
                         num++;
-                        while(*s && !isspace((int)*s))
+                        while(*s && !isspace((unsigned char)*s))
                                 s++;
                 }
         }
@@ -116,6 +117,7 @@ module_list_avail(void)
 {
         /* these are the modules available */
         static const char* names[] = {
+		"dns64",
 #ifdef WITH_PYTHONMODULE
 		"python", 
 #endif
@@ -133,6 +135,7 @@ static fbgetfunctype*
 module_funcs_avail(void)
 {
         static struct module_func_block* (*fb[])(void) = {
+		&dns64_get_funcblock,
 #ifdef WITH_PYTHONMODULE
 		&pythonmod_get_funcblock, 
 #endif
@@ -149,7 +152,7 @@ module_func_block* module_factory(const char** str)
         const char* s = *str;
 	const char** names = module_list_avail();
 	fbgetfunctype* fb = module_funcs_avail();
-        while(*s && isspace((int)*s))
+        while(*s && isspace((unsigned char)*s))
                 s++;
 	while(names[i]) {
                 if(strncmp(names[i], s, strlen(names[i])) == 0) {
diff --git a/services/outside_network.c b/services/outside_network.c
index fedbd0fa8c8e..5bb52ff9fe44 100644
--- a/services/outside_network.c
+++ b/services/outside_network.c
@@ -58,6 +58,7 @@
 #include "util/random.h"
 #include "util/fptr_wlist.h"
 #include "ldns/sbuffer.h"
+#include "dnstap/dnstap.h"
 #ifdef HAVE_OPENSSL_SSL_H
 #include <openssl/ssl.h>
 #endif
@@ -75,11 +76,14 @@
 #define OUTBOUND_UDP_RETRY 1
 
 /** initiate TCP transaction for serviced query */
-static void serviced_tcp_initiate(struct outside_network* outnet, 
-	struct serviced_query* sq, sldns_buffer* buff);
+static void serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff);
 /** with a fd available, randomize and send UDP */
-static int randomize_and_send_udp(struct outside_network* outnet, 
-	struct pending* pend, sldns_buffer* packet, int timeout);
+static int randomize_and_send_udp(struct pending* pend, sldns_buffer* packet,
+	int timeout);
+
+/** remove waiting tcp from the outnet waiting list */
+static void waiting_list_remove(struct outside_network* outnet,
+	struct waiting_tcp* w);
 
 int 
 pending_cmp(const void* key1, const void* key2)
@@ -210,12 +214,12 @@ outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
 		s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 	if(s == -1) {
 #ifndef USE_WINSOCK
-		log_err("outgoing tcp: socket: %s", strerror(errno));
+		log_err_addr("outgoing tcp: socket", strerror(errno),
+			&w->addr, w->addrlen);
 #else
-		log_err("outgoing tcp: socket: %s", 
-			wsa_strerror(WSAGetLastError()));
+		log_err_addr("outgoing tcp: socket", 
+			wsa_strerror(WSAGetLastError()), &w->addr, w->addrlen);
 #endif
-		log_addr(0, "failed address", &w->addr, w->addrlen);
 		return 0;
 	}
 	if(!pick_outgoing_tcp(w, s))
@@ -231,15 +235,14 @@ outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
 #endif
 			if(tcp_connect_errno_needs_log(
 				(struct sockaddr*)&w->addr, w->addrlen))
-				log_err("outgoing tcp: connect: %s",
-					strerror(errno));
+				log_err_addr("outgoing tcp: connect",
+					strerror(errno), &w->addr, w->addrlen);
 			close(s);
 #else /* USE_WINSOCK */
 		if(WSAGetLastError() != WSAEINPROGRESS &&
 			WSAGetLastError() != WSAEWOULDBLOCK) {
 			closesocket(s);
 #endif
-			log_addr(0, "failed address", &w->addr, w->addrlen);
 			return 0;
 		}
 	}
@@ -258,6 +261,7 @@ outnet_tcp_take_into_use(struct waiting_tcp* w, uint8_t* pkt, size_t pkt_len)
 	w->pkt = NULL;
 	w->next_waiting = (void*)pend;
 	pend->id = LDNS_ID_WIRE(pkt);
+	w->outnet->num_tcp_outgoing++;
 	w->outnet->tcp_free = pend->next_free;
 	pend->next_free = NULL;
 	pend->query = w;
@@ -378,7 +382,7 @@ outnet_send_wait_udp(struct outside_network* outnet)
 		free(pend->pkt); /* freeing now makes get_mem correct */
 		pend->pkt = NULL; 
 		pend->pkt_len = 0;
-		if(!randomize_and_send_udp(outnet, pend, outnet->udp_buff, 
+		if(!randomize_and_send_udp(pend, outnet->udp_buff,
 			pend->timeout)) {
 			/* callback error on pending */
 			if(pend->cb) {
@@ -588,7 +592,7 @@ outside_network_create(struct comm_base *base, size_t bufsize,
 	struct ub_randstate* rnd, int use_caps_for_id, int* availports, 
 	int numavailports, size_t unwanted_threshold,
 	void (*unwanted_action)(void*), void* unwanted_param, int do_udp,
-	void* sslctx, int delayclose)
+	void* sslctx, int delayclose, struct dt_env* dtenv)
 {
 	struct outside_network* outnet = (struct outside_network*)
 		calloc(1, sizeof(struct outside_network));
@@ -600,9 +604,15 @@ outside_network_create(struct comm_base *base, size_t bufsize,
 	comm_base_timept(base, &outnet->now_secs, &outnet->now_tv);
 	outnet->base = base;
 	outnet->num_tcp = num_tcp;
+	outnet->num_tcp_outgoing = 0;
 	outnet->infra = infra;
 	outnet->rnd = rnd;
 	outnet->sslctx = sslctx;
+#ifdef USE_DNSTAP
+	outnet->dtenv = dtenv;
+#else
+	(void)dtenv;
+#endif
 	outnet->svcd_overhead = 0;
 	outnet->want_to_quit = 0;
 	outnet->unwanted_threshold = unwanted_threshold;
@@ -991,10 +1001,10 @@ select_ifport(struct outside_network* outnet, struct pending* pend,
 }
 
 static int
-randomize_and_send_udp(struct outside_network* outnet, struct pending* pend,
-	sldns_buffer* packet, int timeout)
+randomize_and_send_udp(struct pending* pend, sldns_buffer* packet, int timeout)
 {
 	struct timeval tv;
+	struct outside_network* outnet = pend->sq->outnet;
 
 	/* select id */
 	if(!select_id(outnet, pend, packet)) {
@@ -1027,30 +1037,38 @@ randomize_and_send_udp(struct outside_network* outnet, struct pending* pend,
 	tv.tv_usec = (timeout%1000)*1000;
 #endif
 	comm_timer_set(pend->timer, &tv);
+
+#ifdef USE_DNSTAP
+	if(outnet->dtenv &&
+	   (outnet->dtenv->log_resolver_query_messages ||
+	    outnet->dtenv->log_forwarder_query_messages))
+		dt_msg_send_outside_query(outnet->dtenv, &pend->addr, comm_udp,
+		pend->sq->zone, pend->sq->zonelen, packet);
+#endif
 	return 1;
 }
 
 struct pending* 
-pending_udp_query(struct outside_network* outnet, sldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* cb, void* cb_arg)
+pending_udp_query(struct serviced_query* sq, struct sldns_buffer* packet,
+	int timeout, comm_point_callback_t* cb, void* cb_arg)
 {
 	struct pending* pend = (struct pending*)calloc(1, sizeof(*pend));
 	if(!pend) return NULL;
-	pend->outnet = outnet;
-	pend->addrlen = addrlen;
-	memmove(&pend->addr, addr, addrlen);
+	pend->outnet = sq->outnet;
+	pend->sq = sq;
+	pend->addrlen = sq->addrlen;
+	memmove(&pend->addr, &sq->addr, sq->addrlen);
 	pend->cb = cb;
 	pend->cb_arg = cb_arg;
 	pend->node.key = pend;
-	pend->timer = comm_timer_create(outnet->base, pending_udp_timer_cb, 
+	pend->timer = comm_timer_create(sq->outnet->base, pending_udp_timer_cb,
 		pend);
 	if(!pend->timer) {
 		free(pend);
 		return NULL;
 	}
 
-	if(outnet->unused_fds == NULL) {
+	if(sq->outnet->unused_fds == NULL) {
 		/* no unused fd, cannot create a new port (randomly) */
 		verbose(VERB_ALGO, "no fds available, udp query waiting");
 		pend->timeout = timeout;
@@ -1063,15 +1081,15 @@ pending_udp_query(struct outside_network* outnet, sldns_buffer* packet,
 			return NULL;
 		}
 		/* put at end of waiting list */
-		if(outnet->udp_wait_last)
-			outnet->udp_wait_last->next_waiting = pend;
+		if(sq->outnet->udp_wait_last)
+			sq->outnet->udp_wait_last->next_waiting = pend;
 		else 
-			outnet->udp_wait_first = pend;
-		outnet->udp_wait_last = pend;
+			sq->outnet->udp_wait_first = pend;
+		sq->outnet->udp_wait_last = pend;
 		return pend;
 	}
-	if(!randomize_and_send_udp(outnet, pend, packet, timeout)) {
-		pending_delete(outnet, pend);
+	if(!randomize_and_send_udp(pend, packet, timeout)) {
+		pending_delete(sq->outnet, pend);
 		return NULL;
 	}
 	return pend;
@@ -1086,17 +1104,7 @@ outnet_tcptimer(void* arg)
 	void* cb_arg;
 	if(w->pkt) {
 		/* it is on the waiting list */
-		struct waiting_tcp* p=outnet->tcp_wait_first, *prev=NULL;
-		while(p) {
-			if(p == w) {
-				if(prev) prev->next_waiting = w->next_waiting;
-				else	outnet->tcp_wait_first=w->next_waiting;
-				outnet->tcp_wait_last = prev;
-				break;
-			}
-			prev = p;
-			p=p->next_waiting;
-		}
+		waiting_list_remove(outnet, w);
 	} else {
 		/* it was in use */
 		struct pending_tcp* pend=(struct pending_tcp*)w->next_waiting;
@@ -1113,12 +1121,11 @@ outnet_tcptimer(void* arg)
 	use_free_buffer(outnet);
 }
 
-struct waiting_tcp* 
-pending_tcp_query(struct outside_network* outnet, sldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg, int ssl_upstream)
+struct waiting_tcp*
+pending_tcp_query(struct serviced_query* sq, sldns_buffer* packet,
+	int timeout, comm_point_callback_t* callback, void* callback_arg)
 {
-	struct pending_tcp* pend = outnet->tcp_free;
+	struct pending_tcp* pend = sq->outnet->tcp_free;
 	struct waiting_tcp* w;
 	struct timeval tv;
 	uint16_t id;
@@ -1128,20 +1135,20 @@ pending_tcp_query(struct outside_network* outnet, sldns_buffer* packet,
 	if(!w) {
 		return NULL;
 	}
-	if(!(w->timer = comm_timer_create(outnet->base, outnet_tcptimer, w))) {
+	if(!(w->timer = comm_timer_create(sq->outnet->base, outnet_tcptimer, w))) {
 		free(w);
 		return NULL;
 	}
 	w->pkt = NULL;
 	w->pkt_len = 0;
-	id = ((unsigned)ub_random(outnet->rnd)>>8) & 0xffff;
+	id = ((unsigned)ub_random(sq->outnet->rnd)>>8) & 0xffff;
 	LDNS_ID_SET(sldns_buffer_begin(packet), id);
-	memcpy(&w->addr, addr, addrlen);
-	w->addrlen = addrlen;
-	w->outnet = outnet;
+	memcpy(&w->addr, &sq->addr, sq->addrlen);
+	w->addrlen = sq->addrlen;
+	w->outnet = sq->outnet;
 	w->cb = callback;
 	w->cb_arg = callback_arg;
-	w->ssl_upstream = ssl_upstream;
+	w->ssl_upstream = sq->ssl_upstream;
 #ifndef S_SPLINT_S
 	tv.tv_sec = timeout;
 	tv.tv_usec = 0;
@@ -1154,16 +1161,23 @@ pending_tcp_query(struct outside_network* outnet, sldns_buffer* packet,
 			waiting_tcp_delete(w);
 			return NULL;
 		}
+#ifdef USE_DNSTAP
+		if(sq->outnet->dtenv &&
+		   (sq->outnet->dtenv->log_resolver_query_messages ||
+		    sq->outnet->dtenv->log_forwarder_query_messages))
+		dt_msg_send_outside_query(sq->outnet->dtenv, &sq->addr,
+		comm_tcp, sq->zone, sq->zonelen, packet);
+#endif
 	} else {
 		/* queue up */
 		w->pkt = (uint8_t*)w + sizeof(struct waiting_tcp);
 		w->pkt_len = sldns_buffer_limit(packet);
 		memmove(w->pkt, sldns_buffer_begin(packet), w->pkt_len);
 		w->next_waiting = NULL;
-		if(outnet->tcp_wait_last)
-			outnet->tcp_wait_last->next_waiting = w;
-		else	outnet->tcp_wait_first = w;
-		outnet->tcp_wait_last = w;
+		if(sq->outnet->tcp_wait_last)
+			sq->outnet->tcp_wait_last->next_waiting = w;
+		else	sq->outnet->tcp_wait_first = w;
+		sq->outnet->tcp_wait_last = w;
 	}
 	return w;
 }
@@ -1205,7 +1219,7 @@ lookup_serviced(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
 /** Create new serviced entry */
 static struct serviced_query*
 serviced_create(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
-	int want_dnssec, int tcp_upstream, int ssl_upstream,
+	int want_dnssec, int nocaps, int tcp_upstream, int ssl_upstream,
 	struct sockaddr_storage* addr, socklen_t addrlen, uint8_t* zone,
 	size_t zonelen, int qtype)
 {
@@ -1232,6 +1246,7 @@ serviced_create(struct outside_network* outnet, sldns_buffer* buff, int dnssec,
 	sq->qtype = qtype;
 	sq->dnssec = dnssec;
 	sq->want_dnssec = want_dnssec;
+	sq->nocaps = nocaps;
 	sq->tcp_upstream = tcp_upstream;
 	sq->ssl_upstream = ssl_upstream;
 	memcpy(&sq->addr, addr, addrlen);
@@ -1319,16 +1334,16 @@ serviced_perturb_qname(struct ub_randstate* rnd, uint8_t* qbuf, size_t len)
 	while(lablen) {
 		while(lablen--) {
 			/* only perturb A-Z, a-z */
-			if(isalpha((int)*d)) {
+			if(isalpha((unsigned char)*d)) {
 				/* get a random bit */	
 				if(bits == 0) {
 					random = ub_random(rnd);
 					bits = 30;
 				}
 				if(random & 0x1) {
-					*d = (uint8_t)toupper((int)*d);
+					*d = (uint8_t)toupper((unsigned char)*d);
 				} else {
-					*d = (uint8_t)tolower((int)*d);
+					*d = (uint8_t)tolower((unsigned char)*d);
 				}
 				random >>= 1;
 				bits--;
@@ -1349,7 +1364,7 @@ static void
 serviced_encode(struct serviced_query* sq, sldns_buffer* buff, int with_edns)
 {
 	/* if we are using 0x20 bits for ID randomness, perturb them */
-	if(sq->outnet->use_caps_for_id) {
+	if(sq->outnet->use_caps_for_id && !sq->nocaps) {
 		serviced_perturb_qname(sq->outnet->rnd, sq->qbuf, sq->qbuflen);
 	}
 	/* generate query */
@@ -1424,8 +1439,8 @@ serviced_udp_send(struct serviced_query* sq, sldns_buffer* buff)
 	sq->last_sent_time = *sq->outnet->now_tv;
 	sq->edns_lame_known = (int)edns_lame_known;
 	verbose(VERB_ALGO, "serviced query UDP timeout=%d msec", rtt);
-	sq->pending = pending_udp_query(sq->outnet, buff, &sq->addr, 
-		sq->addrlen, rtt, serviced_udp_callback, sq);
+	sq->pending = pending_udp_query(sq, buff, rtt,
+		serviced_udp_callback, sq);
 	if(!sq->pending)
 		return 0;
 	return 1;
@@ -1574,13 +1589,21 @@ serviced_tcp_callback(struct comm_point* c, void* arg, int error,
 	if(error==NETEVENT_NOERROR)
 		infra_update_tcp_works(sq->outnet->infra, &sq->addr,
 			sq->addrlen, sq->zone, sq->zonelen);
+#ifdef USE_DNSTAP
+	if(sq->outnet->dtenv &&
+	   (sq->outnet->dtenv->log_resolver_response_messages ||
+	    sq->outnet->dtenv->log_forwarder_response_messages))
+		dt_msg_send_outside_response(sq->outnet->dtenv, &sq->addr,
+		c->type, sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
+		&sq->last_sent_time, sq->outnet->now_tv, c->buffer);
+#endif
 	if(error==NETEVENT_NOERROR && sq->status == serviced_query_TCP_EDNS &&
 		(LDNS_RCODE_WIRE(sldns_buffer_begin(c->buffer)) == 
 		LDNS_RCODE_FORMERR || LDNS_RCODE_WIRE(sldns_buffer_begin(
 		c->buffer)) == LDNS_RCODE_NOTIMPL) ) {
 		/* attempt to fallback to nonEDNS */
 		sq->status = serviced_query_TCP_EDNS_fallback;
-		serviced_tcp_initiate(sq->outnet, sq, c->buffer);
+		serviced_tcp_initiate(sq, c->buffer);
 		return 0;
 	} else if(error==NETEVENT_NOERROR && 
 		sq->status == serviced_query_TCP_EDNS_fallback &&
@@ -1632,16 +1655,14 @@ serviced_tcp_callback(struct comm_point* c, void* arg, int error,
 }
 
 static void
-serviced_tcp_initiate(struct outside_network* outnet, 
-	struct serviced_query* sq, sldns_buffer* buff)
+serviced_tcp_initiate(struct serviced_query* sq, sldns_buffer* buff)
 {
 	verbose(VERB_ALGO, "initiate TCP query %s", 
 		sq->status==serviced_query_TCP_EDNS?"EDNS":"");
 	serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
 	sq->last_sent_time = *sq->outnet->now_tv;
-	sq->pending = pending_tcp_query(outnet, buff, &sq->addr,
-		sq->addrlen, TCP_AUTH_QUERY_TIMEOUT, serviced_tcp_callback, 
-		sq, sq->ssl_upstream);
+	sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
+		serviced_tcp_callback, sq);
 	if(!sq->pending) {
 		/* delete from tree so that a retry by above layer does not
 		 * clash with this entry */
@@ -1665,9 +1686,8 @@ serviced_tcp_send(struct serviced_query* sq, sldns_buffer* buff)
 	else 	sq->status = serviced_query_TCP;
 	serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
 	sq->last_sent_time = *sq->outnet->now_tv;
-	sq->pending = pending_tcp_query(sq->outnet, buff, &sq->addr,
-		sq->addrlen, TCP_AUTH_QUERY_TIMEOUT, serviced_tcp_callback, 
-		sq, sq->ssl_upstream);
+	sq->pending = pending_tcp_query(sq, buff, TCP_AUTH_QUERY_TIMEOUT,
+		serviced_tcp_callback, sq);
 	return sq->pending != NULL;
 }
 
@@ -1728,6 +1748,14 @@ serviced_udp_callback(struct comm_point* c, void* arg, int error,
 		serviced_callbacks(sq, error, c, rep);
 		return 0;
 	}
+#ifdef USE_DNSTAP
+	if(outnet->dtenv &&
+	   (outnet->dtenv->log_resolver_response_messages ||
+	    outnet->dtenv->log_forwarder_response_messages))
+		dt_msg_send_outside_response(outnet->dtenv, &sq->addr, c->type,
+		sq->zone, sq->zonelen, sq->qbuf, sq->qbuflen,
+		&sq->last_sent_time, sq->outnet->now_tv, c->buffer);
+#endif
 	if(!fallback_tcp) {
 	    if( (sq->status == serviced_query_UDP_EDNS 
 	        ||sq->status == serviced_query_UDP_EDNS_FRAG)
@@ -1816,7 +1844,7 @@ serviced_udp_callback(struct comm_point* c, void* arg, int error,
 			/* if we have unfinished EDNS_fallback, start again */
 			sq->status = serviced_query_TCP_EDNS;
 		else	sq->status = serviced_query_TCP;
-		serviced_tcp_initiate(outnet, sq, c->buffer);
+		serviced_tcp_initiate(sq, c->buffer);
 		return 0;
 	}
 	/* yay! an answer */
@@ -1827,10 +1855,11 @@ serviced_udp_callback(struct comm_point* c, void* arg, int error,
 struct serviced_query* 
 outnet_serviced_query(struct outside_network* outnet,
 	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
-	uint16_t flags, int dnssec, int want_dnssec, int tcp_upstream,
-	int ssl_upstream, struct sockaddr_storage* addr, socklen_t addrlen,
-	uint8_t* zone, size_t zonelen, comm_point_callback_t* callback,
-	void* callback_arg, sldns_buffer* buff)
+	uint16_t flags, int dnssec, int want_dnssec, int nocaps,
+	int tcp_upstream, int ssl_upstream, struct sockaddr_storage* addr,
+	socklen_t addrlen, uint8_t* zone, size_t zonelen,
+	comm_point_callback_t* callback, void* callback_arg,
+	sldns_buffer* buff)
 {
 	struct serviced_query* sq;
 	struct service_callback* cb;
@@ -1843,7 +1872,7 @@ outnet_serviced_query(struct outside_network* outnet,
 		return NULL;
 	if(!sq) {
 		/* make new serviced query entry */
-		sq = serviced_create(outnet, buff, dnssec, want_dnssec,
+		sq = serviced_create(outnet, buff, dnssec, want_dnssec, nocaps,
 			tcp_upstream, ssl_upstream, addr, addrlen, zone,
 			zonelen, (int)qtype);
 		if(!sq) {
diff --git a/services/outside_network.h b/services/outside_network.h
index dda9d6f5a235..9959676d33f4 100644
--- a/services/outside_network.h
+++ b/services/outside_network.h
@@ -45,6 +45,7 @@
 
 #include "util/rbtree.h"
 #include "util/netevent.h"
+#include "dnstap/dnstap_config.h"
 struct pending;
 struct pending_timeout;
 struct ub_randstate;
@@ -55,6 +56,8 @@ struct infra_cache;
 struct port_comm;
 struct port_if;
 struct sldns_buffer;
+struct serviced_query;
+struct dt_env;
 
 /**
  * Send queries to outside servers and wait for answers from servers.
@@ -125,6 +128,10 @@ struct outside_network {
 	struct ub_randstate* rnd;
 	/** ssl context to create ssl wrapped TCP with DNS connections */
 	void* sslctx;
+#ifdef USE_DNSTAP
+	/** dnstap environment */
+	struct dt_env* dtenv;
+#endif
 
 	/**
 	 * Array of tcp pending used for outgoing TCP connections.
@@ -135,6 +142,8 @@ struct outside_network {
 	struct pending_tcp **tcp_conns;
 	/** number of tcp communication points. */
 	size_t num_tcp;
+	/** number of tcp communication points in use. */
+	size_t num_tcp_outgoing;
 	/** list of tcp comm points that are free for use */
 	struct pending_tcp* tcp_free;
 	/** list of tcp queries waiting for a buffer */
@@ -210,6 +219,8 @@ struct pending {
 	void* cb_arg;
 	/** the outside network it is part of */
 	struct outside_network* outnet;
+	/** the corresponding serviced_query */
+	struct serviced_query* sq;
 
 	/*---- filled if udp pending is waiting -----*/
 	/** next in waiting list. */
@@ -307,6 +318,8 @@ struct serviced_query {
 	int dnssec;
 	/** We want signatures, or else the answer is likely useless */
 	int want_dnssec;
+	/** ignore capsforid */
+	int nocaps;
 	/** tcp upstream used, use tcp, or ssl_upstream for SSL */
 	int tcp_upstream, ssl_upstream;
 	/** where to send it */
@@ -383,6 +396,7 @@ struct serviced_query {
  * @param sslctx: context to create outgoing connections with (if enabled).
  * @param delayclose: if not 0, udp sockets are delayed before timeout closure.
  * 	msec to wait on timeouted udp sockets.
+ * @param dtenv: environment to send dnstap events with (if enabled).
  * @return: the new structure (with no pending answers) or NULL on error.
  */
 struct outside_network* outside_network_create(struct comm_base* base,
@@ -391,7 +405,7 @@ struct outside_network* outside_network_create(struct comm_base* base,
 	struct ub_randstate* rnd, int use_caps_for_id, int* availports, 
 	int numavailports, size_t unwanted_threshold,
 	void (*unwanted_action)(void*), void* unwanted_param, int do_udp,
-	void* sslctx, int delayclose);
+	void* sslctx, int delayclose, struct dt_env *dtenv);
 
 /**
  * Delete outside_network structure.
@@ -408,39 +422,32 @@ void outside_network_quit_prepare(struct outside_network* outnet);
 /**
  * Send UDP query, create pending answer.
  * Changes the ID for the query to be random and unique for that destination.
- * @param outnet: provides the event handling
+ * @param sq: serviced query.
  * @param packet: wireformat query to send to destination.
- * @param addr: address to send to.
- * @param addrlen: length of addr.
  * @param timeout: in milliseconds from now.
  * @param callback: function to call on error, timeout or reply.
  * @param callback_arg: user argument for callback function.
  * @return: NULL on error for malloc or socket. Else the pending query object.
  */
-struct pending* pending_udp_query(struct outside_network* outnet, 
-	struct sldns_buffer* packet, struct sockaddr_storage* addr, 
-	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+struct pending* pending_udp_query(struct serviced_query* sq,
+	struct sldns_buffer* packet, int timeout, comm_point_callback_t* callback,
 	void* callback_arg);
 
 /**
  * Send TCP query. May wait for TCP buffer. Selects ID to be random, and 
  * checks id.
- * @param outnet: provides the event handling.
+ * @param sq: serviced query.
  * @param packet: wireformat query to send to destination. copied from.
- * @param addr: address to send to.
- * @param addrlen: length of addr.
  * @param timeout: in seconds from now.
  *    Timer starts running now. Timer may expire if all buffers are used,
  *    without any query been sent to the server yet.
  * @param callback: function to call on error, timeout or reply.
  * @param callback_arg: user argument for callback function.
- * @param ssl_upstream: if the tcp connection must use SSL.
  * @return: false on error for malloc or socket. Else the pending TCP object.
  */
-struct waiting_tcp* pending_tcp_query(struct outside_network* outnet, 
-	struct sldns_buffer* packet, struct sockaddr_storage* addr, 
-	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
-	void* callback_arg, int ssl_upstream);
+struct waiting_tcp* pending_tcp_query(struct serviced_query* sq,
+	struct sldns_buffer* packet, int timeout, comm_point_callback_t* callback,
+	void* callback_arg);
 
 /**
  * Delete pending answer.
@@ -464,6 +471,7 @@ void pending_delete(struct outside_network* outnet, struct pending* p);
  *	If the value includes BIT_DO, DO bit is set when in EDNS queries.
  * @param want_dnssec: signatures are needed, without EDNS the answer is
  * 	likely to be useless.
+ * @param nocaps: ignore use_caps_for_id and use unperturbed qname.
  * @param tcp_upstream: use TCP for upstream queries.
  * @param ssl_upstream: use SSL for upstream queries.
  * @param callback: callback function.
@@ -480,10 +488,11 @@ void pending_delete(struct outside_network* outnet, struct pending* p);
  */
 struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
 	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
-	uint16_t flags, int dnssec, int want_dnssec, int tcp_upstream,
-	int ssl_upstream, struct sockaddr_storage* addr, socklen_t addrlen,
-	uint8_t* zone, size_t zonelen, comm_point_callback_t* callback,
-	void* callback_arg, struct sldns_buffer* buff);
+	uint16_t flags, int dnssec, int want_dnssec, int nocaps,
+	int tcp_upstream, int ssl_upstream, struct sockaddr_storage* addr,
+	socklen_t addrlen, uint8_t* zone, size_t zonelen,
+	comm_point_callback_t* callback, void* callback_arg,
+	struct sldns_buffer* buff);
 
 /**
  * Remove service query callback.