Implement flowid calculation for outbound connections to balance

 connections over multiple paths.

Multipath routing relies on mbuf flowid data for both transit
 and outbound traffic. Current code fills mbuf flowid from inp_flowid
 for connection-oriented sockets. However, inp_flowid is currently
 not calculated for outbound connections.

This change creates simple hashing functions and starts calculating hashes
 for TCP,UDP/UDP-Lite and raw IP if multipath routes are present in the
 system.

Reviewed by:	glebius (previous version),ae
Differential Revision:	https://reviews.freebsd.org/D26523

20 files changed, 312 insertions, 77 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 9b1a52f9ba59..f4a296546f1e 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -4180,7 +4180,7 @@ net/rss_config.c		optional inet rss | inet6 rss
 net/rtsock.c			standard
 net/slcompress.c		optional netgraph_vjc | sppp | \
 					 netgraph_sppp
-net/toeplitz.c			optional inet rss | inet6 rss
+net/toeplitz.c			optional inet rss | inet6 rss | route_mpath
 net/vnet.c			optional vimage
 net80211/ieee80211.c		optional wlan
 net80211/ieee80211_acl.c	optional wlan wlan_acl
diff --git a/sys/net/radix.c b/sys/net/radix.c
index f65153393d74..410278e9a2e9 100644
--- a/sys/net/radix.c
+++ b/sys/net/radix.c
@@ -624,21 +624,6 @@ rn_addroute(void *v_arg, void *n_arg, struct radix_head *head,
 	saved_tt = tt = rn_insert(v, head, &keyduplicated, treenodes);
 	if (keyduplicated) {
 		for (t = tt; tt; t = tt, tt = tt->rn_dupedkey) {
-#ifdef RADIX_MPATH
-			/* permit multipath, if enabled for the family */
-			if (rn_mpath_capable(head) && netmask == tt->rn_mask) {
-				/*
-				 * go down to the end of multipaths, so that
-				 * new entry goes into the end of rn_dupedkey
-				 * chain.
-				 */
-				do {
-					t = tt;
-					tt = tt->rn_dupedkey;
-				} while (tt && t->rn_mask == tt->rn_mask);
-				break;
-			}
-#endif
 			if (tt->rn_mask == netmask)
 				return (0);
 			if (netmask == 0 ||
@@ -744,10 +729,8 @@ on2:
 		if (m->rm_flags & RNF_NORMAL) {
 			mmask = m->rm_leaf->rn_mask;
 			if (tt->rn_flags & RNF_NORMAL) {
-#if !defined(RADIX_MPATH)
 			    log(LOG_ERR,
 			        "Non-unique normal route, mask not entered\n");
-#endif
 				return (tt);
 			}
 		} else
diff --git a/sys/net/route.h b/sys/net/route.h
index 021b4621692b..1ce808b6c1e1 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -125,8 +125,42 @@ VNET_DECLARE(uint32_t, _rt_numfibs);	/* number of existing route tables */
 #define	rt_numfibs		V_rt_numfibs
 VNET_DECLARE(u_int, rt_add_addr_allfibs); /* Announce interfaces to all fibs */
 #define	V_rt_add_addr_allfibs	VNET(rt_add_addr_allfibs)
+
+/* Calculate flowid for locally-originated packets */
+#define	V_fib_hash_outbound	VNET(fib_hash_outbound)
+VNET_DECLARE(u_int, fib_hash_outbound);
+
+/* Outbound flowid generation rules */
+#ifdef RSS
+
+#define fib4_calc_packet_hash		xps_proto_software_hash_v4
+#define fib6_calc_packet_hash		xps_proto_software_hash_v6
+#define	CALC_FLOWID_OUTBOUND_SENDTO	true
+
+#ifdef ROUTE_MPATH
+#define	CALC_FLOWID_OUTBOUND		V_fib_hash_outbound
+#else
+#define	CALC_FLOWID_OUTBOUND		false
 #endif
 
+#else /* !RSS */
+
+#define fib4_calc_packet_hash		fib4_calc_software_hash
+#define fib6_calc_packet_hash		fib6_calc_software_hash
+
+#ifdef ROUTE_MPATH
+#define	CALC_FLOWID_OUTBOUND_SENDTO	V_fib_hash_outbound
+#define	CALC_FLOWID_OUTBOUND		V_fib_hash_outbound
+#else
+#define	CALC_FLOWID_OUTBOUND_SENDTO	false
+#define	CALC_FLOWID_OUTBOUND		false
+#endif
+
+#endif /* RSS */
+
+
+#endif /* _KERNEL */
+
 /*
  * We distinguish between routes to hosts and routes to networks,
  * preferring the former if available.  For each route we infer
diff --git a/sys/net/route/mpath_ctl.c b/sys/net/route/mpath_ctl.c
index 1ac7c191ed05..5632750f466d 100644
--- a/sys/net/route/mpath_ctl.c
+++ b/sys/net/route/mpath_ctl.c
@@ -64,6 +64,20 @@
  */
 
 SYSCTL_DECL(_net_route);
+VNET_DEFINE(u_int, fib_hash_outbound) = 0;
+SYSCTL_UINT(_net_route, OID_AUTO, hash_outbound, CTLFLAG_RD | CTLFLAG_VNET,
+    &VNET_NAME(fib_hash_outbound), 0,
+    "Compute flowid for locally-originated packets");
+
+/* Default entropy to add to the hash calculation for the outbound connections*/
+uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN] = {
+	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
+	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
+	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
+	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
+	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
+};
+
 
 /*
  * Tries to add @rnd_add nhop to the existing set of nhops (@nh_orig) for the
@@ -115,6 +129,17 @@ add_route_mpath(struct rib_head *rnh, struct rt_addrinfo *info,
 		RTSTAT_INC(rts_add_retry);
 	}
 
+	if (V_fib_hash_outbound == 0 && error == 0 &&
+	    NH_IS_NHGRP(rc->rc_nh_new)) {
+		/*
+		 * First multipath route got installed. Enable local
+		 * outbound connections hashing.
+		 */
+		if (bootverbose)
+			printf("FIB: enabled flowid calculation for locally-originated packets\n");
+		V_fib_hash_outbound = 1;
+	}
+
 	return (error);
 }
 
diff --git a/sys/net/route/route_var.h b/sys/net/route/route_var.h
index 12d081d410a2..00e6c3a59b04 100644
--- a/sys/net/route/route_var.h
+++ b/sys/net/route/route_var.h
@@ -307,4 +307,8 @@ int nhgrp_get_addition_group(struct rib_head *rnh,
 
 void nhgrp_free(struct nhgrp_object *nhg);
 
+/* Entropy data used for outbound hashing */
+#define MPATH_ENTROPY_KEY_LEN	40
+extern uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN];
+
 #endif
diff --git a/sys/netinet/in_fib.c b/sys/netinet/in_fib.c
index 4c84de2c7281..031277add777 100644
--- a/sys/netinet/in_fib.c
+++ b/sys/netinet/in_fib.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
 #include <net/route/route_ctl.h>
 #include <net/route/route_var.h>
 #include <net/route/nhop.h>
+#include <net/toeplitz.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
@@ -62,6 +63,40 @@ __FBSDID("$FreeBSD$");
 /* Assert 'struct route_in' is compatible with 'struct route' */
 CHK_STRUCT_ROUTE_COMPAT(struct route_in, ro_dst4);
 
+#ifdef ROUTE_MPATH
+struct _hash_5tuple_ipv4 {
+	struct in_addr src;
+	struct in_addr dst;
+	unsigned short src_port;
+	unsigned short dst_port;
+	char proto;
+	char spare[3];
+};
+_Static_assert(sizeof(struct _hash_5tuple_ipv4) == 16,
+    "_hash_5tuple_ipv4 size is wrong");
+
+
+uint32_t
+fib4_calc_software_hash(struct in_addr src, struct in_addr dst,
+    unsigned short src_port, unsigned short dst_port, char proto,
+    uint32_t *phashtype)
+{
+	struct _hash_5tuple_ipv4 data;
+
+	data.src = src;
+	data.dst = dst;
+	data.src_port = src_port;
+	data.dst_port = dst_port;
+	data.proto = proto;
+	data.spare[0] = data.spare[1] = data.spare[2] = 0;
+
+	*phashtype = M_HASHTYPE_OPAQUE;
+
+	return (toeplitz_hash(MPATH_ENTROPY_KEY_LEN, mpath_entropy_key,
+	  sizeof(data), (uint8_t *)&data));
+}
+#endif
+
 /*
  * Looks up path in fib @fibnum specified by @dst.
  * Returns path nexthop on success. Nexthop is safe to use
diff --git a/sys/netinet/in_fib.h b/sys/netinet/in_fib.h
index bc2a2ad6a30e..0d93086bc116 100644
--- a/sys/netinet/in_fib.h
+++ b/sys/netinet/in_fib.h
@@ -51,4 +51,7 @@ int fib4_check_urpf(uint32_t fibnum, struct in_addr dst, uint32_t scopeid,
     uint32_t flags, const struct ifnet *src_if);
 struct nhop_object *fib4_lookup_debugnet(uint32_t fibnum, struct in_addr dst,
     uint32_t scopeid, uint32_t flags);
+uint32_t fib4_calc_software_hash(struct in_addr src, struct in_addr dst,
+    unsigned short src_port, unsigned short dst_port, char proto,
+    uint32_t *phashtype);
 #endif
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 30bf72f83e33..37438dd68c71 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet6.h"
 #include "opt_ratelimit.h"
 #include "opt_pcbgroup.h"
+#include "opt_route.h"
 #include "opt_rss.h"
 
 #include <sys/param.h>
@@ -1327,7 +1328,17 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
 	lport = *lportp;
 	faddr = sin->sin_addr;
 	fport = sin->sin_port;
+#ifdef ROUTE_MPATH
+	if (CALC_FLOWID_OUTBOUND) {
+		uint32_t hash_val, hash_type;
 
+		hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
+		    inp->inp_socket->so_proto->pr_protocol, &hash_type);
+
+		inp->inp_flowid = hash_val;
+		inp->inp_flowtype = hash_type;
+	}
+#endif
 	if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
 		/*
 		 * If the destination address is INADDR_ANY,
diff --git a/sys/netinet/in_rss.c b/sys/netinet/in_rss.c
index f3184175a7cf..05659b97fe7c 100644
--- a/sys/netinet/in_rss.c
+++ b/sys/netinet/in_rss.c
@@ -152,6 +152,48 @@ rss_proto_software_hash_v4(struct in_addr s, struct in_addr d,
 }
 
 /*
+ * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
+ * IPv4 source/destination address, UDP or TCP source/destination ports
+ * and the protocol type.
+ *
+ * The protocol code may wish to do a software hash of the given
+ * tuple.  This depends upon the currently configured RSS hash types.
+ *
+ * It assumes the packet source/destination address
+ * are in "outgoing" packet order (ie, destination is "far" address.)
+ */
+uint32_t
+xps_proto_software_hash_v4(struct in_addr s, struct in_addr d,
+    u_short sp, u_short dp, int proto, uint32_t *hashtype)
+{
+	uint32_t hash;
+
+	/*
+	 * Next, choose the hash type depending upon the protocol
+	 * identifier.
+	 */
+	if ((proto == IPPROTO_TCP) &&
+	    (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
+		hash = rss_hash_ip4_4tuple(d, dp, s, sp);
+		*hashtype = M_HASHTYPE_RSS_TCP_IPV4;
+		return (hash);
+	} else if ((proto == IPPROTO_UDP) &&
+	    (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
+		hash = rss_hash_ip4_4tuple(d, dp, s, sp);
+		*hashtype = M_HASHTYPE_RSS_UDP_IPV4;
+		return (hash);
+	} else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
+		/* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
+		hash = rss_hash_ip4_2tuple(d, s);
+		*hashtype = M_HASHTYPE_RSS_IPV4;
+		return (hash);
+	}
+
+	*hashtype = M_HASHTYPE_NONE;
+	return (0);
+}
+
+/*
  * Do a software calculation of the RSS for the given mbuf.
  *
  * This is typically used by the input path to recalculate the RSS after
diff --git a/sys/netinet/in_rss.h b/sys/netinet/in_rss.h
index fd300ac5fdff..cdc69bc64709 100644
--- a/sys/netinet/in_rss.h
+++ b/sys/netinet/in_rss.h
@@ -53,5 +53,7 @@ int		rss_proto_software_hash_v4(struct in_addr src,
 		    uint32_t *hashtype);
 struct mbuf *	rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source,
 		    u_int *cpuid);
+uint32_t	xps_proto_software_hash_v4(struct in_addr s, struct in_addr d,
+		    u_short sp, u_short dp, int proto, uint32_t *hashtype);
 
 #endif /* !_NETINET_IN_RSS_H_ */
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index 6240a77bdb39..a63fc19587f9 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
+#include "opt_route.h"
 
 #include <sys/param.h>
 #include <sys/jail.h>
@@ -67,6 +68,7 @@ __FBSDID("$FreeBSD$");
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
+#include <netinet/in_fib.h>
 #include <netinet/in_pcb.h>
 #include <netinet/in_var.h>
 #include <netinet/if_ether.h>
@@ -484,6 +486,17 @@ rip_output(struct mbuf *m, struct socket *so, ...)
 		ip->ip_len = htons(m->m_pkthdr.len);
 		ip->ip_src = inp->inp_laddr;
 		ip->ip_dst.s_addr = dst;
+#ifdef ROUTE_MPATH
+		if (CALC_FLOWID_OUTBOUND) {
+			uint32_t hash_type, hash_val;
+
+			hash_val = fib4_calc_software_hash(ip->ip_src,
+			    ip->ip_dst, 0, 0, ip->ip_p, &hash_type);
+			m->m_pkthdr.flowid = hash_val;
+			M_HASHTYPE_SET(m, hash_type);
+			flags |= IP_NODEFAULTFLOWID;
+		}
+#endif
 		if (jailed(inp->inp_cred)) {
 			/*
 			 * prison_local_ip4() would be good enough but would
@@ -519,7 +532,17 @@ rip_output(struct mbuf *m, struct socket *so, ...)
 				return (EINVAL);
 			ip = mtod(m, struct ip *);
 		}
-
+#ifdef ROUTE_MPATH
+		if (CALC_FLOWID_OUTBOUND) {
+			uint32_t hash_type, hash_val;
+
+			hash_val = fib4_calc_software_hash(ip->ip_dst,
+			    ip->ip_src, 0, 0, ip->ip_p, &hash_type);
+			m->m_pkthdr.flowid = hash_val;
+			M_HASHTYPE_SET(m, hash_type);
+			flags |= IP_NODEFAULTFLOWID;
+		}
+#endif
 		INP_RLOCK(inp);
 		/*
 		 * Don't allow both user specified and setsockopt options,
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index e1bb5f07c0d0..52304ddd6584 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
+#include "opt_route.h"
 #include "opt_rss.h"
 
 #include <sys/param.h>
@@ -76,6 +77,7 @@ __FBSDID("$FreeBSD$");
 
 #include <netinet/in.h>
 #include <netinet/in_kdtrace.h>
+#include <netinet/in_fib.h>
 #include <netinet/in_pcb.h>
 #include <netinet/in_systm.h>
 #include <netinet/in_var.h>
@@ -1483,30 +1485,14 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
 		m->m_pkthdr.flowid = flowid;
 		M_HASHTYPE_SET(m, flowtype);
 	}
-#ifdef	RSS
-	else {
+#if defined(ROUTE_MPATH) || defined(RSS)
+	else if (CALC_FLOWID_OUTBOUND_SENDTO) {
 		uint32_t hash_val, hash_type;
-		/*
-		 * Calculate an appropriate RSS hash for UDP and
-		 * UDP Lite.
-		 *
-		 * The called function will take care of figuring out
-		 * whether a 2-tuple or 4-tuple hash is required based
-		 * on the currently configured scheme.
-		 *
-		 * Later later on connected socket values should be
-		 * cached in the inpcb and reused, rather than constantly
-		 * re-calculating it.
-		 *
-		 * UDP Lite is a different protocol number and will
-		 * likely end up being hashed as a 2-tuple until
-		 * RSS / NICs grow UDP Lite protocol awareness.
-		 */
-		if (rss_proto_software_hash_v4(faddr, laddr, fport, lport,
-		    pr, &hash_val, &hash_type) == 0) {
-			m->m_pkthdr.flowid = hash_val;
-			M_HASHTYPE_SET(m, hash_type);
-		}
+
+		hash_val = fib4_calc_packet_hash(laddr, faddr,
+		    lport, fport, pr, &hash_type);
+		m->m_pkthdr.flowid = hash_val;
+		M_HASHTYPE_SET(m, hash_type);
 	}
 
 	/*
diff --git a/sys/netinet6/in6_fib.c b/sys/netinet6/in6_fib.c
index 9fd869b2730b..a3fd20f8ba25 100644
--- a/sys/netinet6/in6_fib.c
+++ b/sys/netinet6/in6_fib.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <net/route/route_ctl.h>
 #include <net/route/route_var.h>
 #include <net/route/nhop.h>
+#include <net/toeplitz.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
@@ -68,6 +69,39 @@ __FBSDID("$FreeBSD$");
 
 CHK_STRUCT_ROUTE_COMPAT(struct route_in6, ro_dst);
 
+#ifdef ROUTE_MPATH
+struct _hash_5tuple_ipv6 {
+	struct in6_addr src;
+	struct in6_addr dst;
+	unsigned short src_port;
+	unsigned short dst_port;
+	char proto;
+	char spare[3];
+};
+_Static_assert(sizeof(struct _hash_5tuple_ipv6) == 40,
+    "_hash_5tuple_ipv6 size is wrong");
+
+uint32_t
+fib6_calc_software_hash(const struct in6_addr *src, const struct in6_addr *dst,
+    unsigned short src_port, unsigned short dst_port, char proto,
+    uint32_t *phashtype)
+{
+	struct _hash_5tuple_ipv6 data;
+
+	data.src = *src;
+	data.dst = *dst;
+	data.src_port = src_port;
+	data.dst_port = dst_port;
+	data.proto = proto;
+	data.spare[0] = data.spare[1] = data.spare[2] = 0;
+
+	*phashtype = M_HASHTYPE_OPAQUE_HASH;
+
+	return (toeplitz_hash(MPATH_ENTROPY_KEY_LEN, mpath_entropy_key,
+	  sizeof(data), (uint8_t *)&data));
+}
+#endif
+
 /*
  * Looks up path in fib @fibnum specified by @dst.
  * Assumes scope is deembedded and provided in @scopeid.
diff --git a/sys/netinet6/in6_fib.h b/sys/netinet6/in6_fib.h
index a716674bce23..78edb5e5b776 100644
--- a/sys/netinet6/in6_fib.h
+++ b/sys/netinet6/in6_fib.h
@@ -39,4 +39,7 @@ int fib6_check_urpf(uint32_t fibnum, const struct in6_addr *dst6,
     uint32_t scopeid, uint32_t flags, const struct ifnet *src_if);
 struct nhop_object *fib6_lookup_debugnet(uint32_t fibnum,
     const struct in6_addr *dst6, uint32_t scopeid, uint32_t flags);
+uint32_t fib6_calc_software_hash(const struct in6_addr *src,
+    const struct in6_addr *dst, unsigned short src_port, unsigned short dst_port,
+    char proto, uint32_t *phashtype);
 #endif
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 3444ca1b2fe7..ab498bf91734 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -73,6 +73,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
 #include "opt_pcbgroup.h"
+#include "opt_route.h"
 #include "opt_rss.h"
 
 #include <sys/param.h>
@@ -423,6 +424,17 @@ in6_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
 	INP_WLOCK_ASSERT(inp);
 	INP_HASH_WLOCK_ASSERT(pcbinfo);
 
+#ifdef ROUTE_MPATH
+	if (CALC_FLOWID_OUTBOUND) {
+		uint32_t hash_type, hash_val;
+
+		hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
+		    &sin6->sin6_addr, 0, sin6->sin6_port,
+		    inp->inp_socket->so_proto->pr_protocol, &hash_type);
+		inp->inp_flowid = hash_val;
+		inp->inp_flowtype = hash_type;
+	}
+#endif
 	/*
 	 * Call inner routine, to assign local interface address.
 	 * in6_pcbladdr() may automatically fill in sin6_scope_id.
diff --git a/sys/netinet6/in6_rss.c b/sys/netinet6/in6_rss.c
index 72dd0a8831af..2b86d961515b 100644
--- a/sys/netinet6/in6_rss.c
+++ b/sys/netinet6/in6_rss.c
@@ -153,6 +153,50 @@ rss_proto_software_hash_v6(const struct in6_addr *s, const struct in6_addr *d,
 }
 
 /*
+ * Calculate an appropriate ipv6 2-tuple or 4-tuple given the given
+ * IPv6 source/destination address, UDP or TCP source/destination ports
+ * and the protocol type.
+ *
+ * The protocol code may wish to do a software hash of the given
+ * tuple. This depends upon the currently configured RSS hash types.
+ *
+ * It assumes the packet source/destination address
+ * are in "outgoin" packet order (ie, destination is "far" address.)
+ */
+uint32_t
+xps_proto_software_hash_v6(const struct in6_addr *s, const struct in6_addr *d,
+    u_short sp, u_short dp, int proto, uint32_t *hashtype)
+{
+
+	uint32_t hash;
+
+	/*
+	 * Next, choose the hash type depending upon the protocol
+	 * identifier.
+	 */
+	if ((proto == IPPROTO_TCP) &&
+	    (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
+		hash = rss_hash_ip6_4tuple(d, dp, s, sp);
+		*hashtype = M_HASHTYPE_RSS_TCP_IPV6;
+		return (hash);
+	} else if ((proto == IPPROTO_UDP) &&
+	    (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
+		hash = rss_hash_ip6_4tuple(d, dp, s, sp);
+		*hashtype = M_HASHTYPE_RSS_UDP_IPV6;
+		return (hash);
+	} else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) {
+		/* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
+		hash = rss_hash_ip6_2tuple(d, s);
+		*hashtype = M_HASHTYPE_RSS_IPV6;
+		return (hash);
+	}
+
+	*hashtype = M_HASHTYPE_NONE;
+	return (0);
+}
+
+
+/*
  * Do a software calculation of the RSS for the given mbuf.
  *
  * This is typically used by the input path to recalculate the RSS after
diff --git a/sys/netinet6/in6_rss.h b/sys/netinet6/in6_rss.h
index f5b48c71408a..5846e515d185 100644
--- a/sys/netinet6/in6_rss.h
+++ b/sys/netinet6/in6_rss.h
@@ -54,5 +54,8 @@ int		rss_proto_software_hash_v6(const struct in6_addr *src,
 		    uint32_t *hashtype);
 struct mbuf *	rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source,
 		    u_int *cpuid);
+uint32_t	xps_proto_software_hash_v6(const struct in6_addr *s,
+		    const struct in6_addr *d, u_short sp, u_short dp,
+		    int proto, uint32_t *hashtype);
 
 #endif /* !_NETINET6_IN6_RSS_H_ */
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 59aabaa258f5..4e8f22a01b0a 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -419,9 +419,6 @@ done:
  *
  * ifpp - XXX: just for statistics
  */
-/*
- * XXX TODO: no flowid is assigned for outbound flows?
- */
 int
 ip6_output(struct mbuf *m0, struct ip6_pktopts *opt,
     struct route_in6 *ro, int flags, struct ip6_moptions *im6o,
@@ -775,7 +772,8 @@ again:
 			}
 		}
 
-		nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE, 0);
+		nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE,
+		    m->m_pkthdr.flowid);
 		if (nh == NULL) {
 			IP6STAT_INC(ip6s_noroute);
 			/* No ifp in6_ifstat_inc(ifp, ifs6_out_discard); */
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index 636828eb4ef4..aea99add4391 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_ipsec.h"
 #include "opt_inet6.h"
+#include "opt_route.h"
 
 #include <sys/param.h>
 #include <sys/errno.h>
@@ -103,6 +104,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet6/ip6_var.h>
 #include <netinet6/nd6.h>
 #include <netinet6/raw_ip6.h>
+#include <netinet6/in6_fib.h>
 #include <netinet6/scope6_var.h>
 #include <netinet6/send.h>
 
@@ -462,6 +464,17 @@ rip6_output(struct mbuf *m, struct socket *so, ...)
 	}
 	ip6 = mtod(m, struct ip6_hdr *);
 
+#ifdef ROUTE_MPATH
+	if (CALC_FLOWID_OUTBOUND) {
+		uint32_t hash_type, hash_val;
+
+		hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
+		    &dstsock->sin6_addr, 0, 0, so->so_proto->pr_protocol,
+		    &hash_type);
+		inp->inp_flowid = hash_val;
+		inp->inp_flowtype = hash_type;
+	}
+#endif
 	/*
 	 * Source address selection.
 	 */
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index 32aa21cb24a1..1535be90e1b0 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
+#include "opt_route.h"
 #include "opt_rss.h"
 
 #include <sys/param.h>
@@ -115,6 +116,7 @@ __FBSDID("$FreeBSD$");
 
 #include <netinet6/ip6protosw.h>
 #include <netinet6/ip6_var.h>
+#include <netinet6/in6_fib.h>
 #include <netinet6/in6_pcb.h>
 #include <netinet6/in6_rss.h>
 #include <netinet6/udp6_var.h>
@@ -954,42 +956,20 @@ udp6_output(struct socket *so, int flags_arg, struct mbuf *m,
 	}
 
 	flags = 0;
-#ifdef	RSS
-	{
-		uint32_t hash_val, hash_type;
+#if defined(ROUTE_MPATH) || defined(RSS)
+	if (CALC_FLOWID_OUTBOUND_SENDTO) {
+		uint32_t hash_type, hash_val;
 		uint8_t pr;
 
 		pr = inp->inp_socket->so_proto->pr_protocol;
-		/*
-		 * Calculate an appropriate RSS hash for UDP and
-		 * UDP Lite.
-		 *
-		 * The called function will take care of figuring out
-		 * whether a 2-tuple or 4-tuple hash is required based
-		 * on the currently configured scheme.
-		 *
-		 * Later later on connected socket values should be
-		 * cached in the inpcb and reused, rather than constantly
-		 * re-calculating it.
-		 *
-		 * UDP Lite is a different protocol number and will
-		 * likely end up being hashed as a 2-tuple until
-		 * RSS / NICs grow UDP Lite protocol awareness.
-		 */
-		if (rss_proto_software_hash_v6(faddr, laddr, fport,
-		    inp->inp_lport, pr, &hash_val, &hash_type) == 0) {
-			m->m_pkthdr.flowid = hash_val;
-			M_HASHTYPE_SET(m, hash_type);
-		}
 
-		/*
-		 * Don't override with the inp cached flowid.
-		 *
-		 * Until the whole UDP path is vetted, it may actually
-		 * be incorrect.
-		 */
-		flags |= IP_NODEFAULTFLOWID;
+		hash_val = fib6_calc_packet_hash(laddr, faddr,
+		    inp->inp_lport, fport, pr, &hash_type);
+		m->m_pkthdr.flowid = hash_val;
+		M_HASHTYPE_SET(m, hash_type);
 	}
+	/* do not use inp flowid */
+	flags |= IP_NODEFAULTFLOWID;
 #endif
 
 	UDPSTAT_INC(udps_opackets);