aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--sys/conf/files2
-rw-r--r--sys/net/radix.c17
-rw-r--r--sys/net/route.h34
-rw-r--r--sys/net/route/mpath_ctl.c25
-rw-r--r--sys/net/route/route_var.h4
-rw-r--r--sys/netinet/in_fib.c35
-rw-r--r--sys/netinet/in_fib.h3
-rw-r--r--sys/netinet/in_pcb.c11
-rw-r--r--sys/netinet/in_rss.c42
-rw-r--r--sys/netinet/in_rss.h2
-rw-r--r--sys/netinet/raw_ip.c25
-rw-r--r--sys/netinet/udp_usrreq.c32
-rw-r--r--sys/netinet6/in6_fib.c34
-rw-r--r--sys/netinet6/in6_fib.h3
-rw-r--r--sys/netinet6/in6_pcb.c12
-rw-r--r--sys/netinet6/in6_rss.c44
-rw-r--r--sys/netinet6/in6_rss.h3
-rw-r--r--sys/netinet6/ip6_output.c6
-rw-r--r--sys/netinet6/raw_ip6.c13
-rw-r--r--sys/netinet6/udp6_usrreq.c42
20 files changed, 312 insertions, 77 deletions
diff --git a/sys/conf/files b/sys/conf/files
index 9b1a52f9ba59..f4a296546f1e 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -4180,7 +4180,7 @@ net/rss_config.c optional inet rss | inet6 rss
net/rtsock.c standard
net/slcompress.c optional netgraph_vjc | sppp | \
netgraph_sppp
-net/toeplitz.c optional inet rss | inet6 rss
+net/toeplitz.c optional inet rss | inet6 rss | route_mpath
net/vnet.c optional vimage
net80211/ieee80211.c optional wlan
net80211/ieee80211_acl.c optional wlan wlan_acl
diff --git a/sys/net/radix.c b/sys/net/radix.c
index f65153393d74..410278e9a2e9 100644
--- a/sys/net/radix.c
+++ b/sys/net/radix.c
@@ -624,21 +624,6 @@ rn_addroute(void *v_arg, void *n_arg, struct radix_head *head,
saved_tt = tt = rn_insert(v, head, &keyduplicated, treenodes);
if (keyduplicated) {
for (t = tt; tt; t = tt, tt = tt->rn_dupedkey) {
-#ifdef RADIX_MPATH
- /* permit multipath, if enabled for the family */
- if (rn_mpath_capable(head) && netmask == tt->rn_mask) {
- /*
- * go down to the end of multipaths, so that
- * new entry goes into the end of rn_dupedkey
- * chain.
- */
- do {
- t = tt;
- tt = tt->rn_dupedkey;
- } while (tt && t->rn_mask == tt->rn_mask);
- break;
- }
-#endif
if (tt->rn_mask == netmask)
return (0);
if (netmask == 0 ||
@@ -744,10 +729,8 @@ on2:
if (m->rm_flags & RNF_NORMAL) {
mmask = m->rm_leaf->rn_mask;
if (tt->rn_flags & RNF_NORMAL) {
-#if !defined(RADIX_MPATH)
log(LOG_ERR,
"Non-unique normal route, mask not entered\n");
-#endif
return (tt);
}
} else
diff --git a/sys/net/route.h b/sys/net/route.h
index 021b4621692b..1ce808b6c1e1 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -125,8 +125,42 @@ VNET_DECLARE(uint32_t, _rt_numfibs); /* number of existing route tables */
#define rt_numfibs V_rt_numfibs
VNET_DECLARE(u_int, rt_add_addr_allfibs); /* Announce interfaces to all fibs */
#define V_rt_add_addr_allfibs VNET(rt_add_addr_allfibs)
+
+/* Calculate flowid for locally-originated packets */
+#define V_fib_hash_outbound VNET(fib_hash_outbound)
+VNET_DECLARE(u_int, fib_hash_outbound);
+
+/* Outbound flowid generation rules */
+#ifdef RSS
+
+#define fib4_calc_packet_hash xps_proto_software_hash_v4
+#define fib6_calc_packet_hash xps_proto_software_hash_v6
+#define CALC_FLOWID_OUTBOUND_SENDTO true
+
+#ifdef ROUTE_MPATH
+#define CALC_FLOWID_OUTBOUND V_fib_hash_outbound
+#else
+#define CALC_FLOWID_OUTBOUND false
#endif
+#else /* !RSS */
+
+#define fib4_calc_packet_hash fib4_calc_software_hash
+#define fib6_calc_packet_hash fib6_calc_software_hash
+
+#ifdef ROUTE_MPATH
+#define CALC_FLOWID_OUTBOUND_SENDTO V_fib_hash_outbound
+#define CALC_FLOWID_OUTBOUND V_fib_hash_outbound
+#else
+#define CALC_FLOWID_OUTBOUND_SENDTO false
+#define CALC_FLOWID_OUTBOUND false
+#endif
+
+#endif /* RSS */
+
+
+#endif /* _KERNEL */
+
/*
* We distinguish between routes to hosts and routes to networks,
* preferring the former if available. For each route we infer
diff --git a/sys/net/route/mpath_ctl.c b/sys/net/route/mpath_ctl.c
index 1ac7c191ed05..5632750f466d 100644
--- a/sys/net/route/mpath_ctl.c
+++ b/sys/net/route/mpath_ctl.c
@@ -64,6 +64,20 @@
*/
SYSCTL_DECL(_net_route);
+VNET_DEFINE(u_int, fib_hash_outbound) = 0;
+SYSCTL_UINT(_net_route, OID_AUTO, hash_outbound, CTLFLAG_RD | CTLFLAG_VNET,
+ &VNET_NAME(fib_hash_outbound), 0,
+ "Compute flowid for locally-originated packets");
+
+/* Default entropy to add to the hash calculation for the outbound connections*/
+uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN] = {
+ 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
+ 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
+ 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
+ 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
+ 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
+};
+
/*
* Tries to add @rnd_add nhop to the existing set of nhops (@nh_orig) for the
@@ -115,6 +129,17 @@ add_route_mpath(struct rib_head *rnh, struct rt_addrinfo *info,
RTSTAT_INC(rts_add_retry);
}
+ if (V_fib_hash_outbound == 0 && error == 0 &&
+ NH_IS_NHGRP(rc->rc_nh_new)) {
+ /*
+ * First multipath route got installed. Enable local
+ * outbound connections hashing.
+ */
+ if (bootverbose)
+ printf("FIB: enabled flowid calculation for locally-originated packets\n");
+ V_fib_hash_outbound = 1;
+ }
+
return (error);
}
diff --git a/sys/net/route/route_var.h b/sys/net/route/route_var.h
index 12d081d410a2..00e6c3a59b04 100644
--- a/sys/net/route/route_var.h
+++ b/sys/net/route/route_var.h
@@ -307,4 +307,8 @@ int nhgrp_get_addition_group(struct rib_head *rnh,
void nhgrp_free(struct nhgrp_object *nhg);
+/* Entropy data used for outbound hashing */
+#define MPATH_ENTROPY_KEY_LEN 40
+extern uint8_t mpath_entropy_key[MPATH_ENTROPY_KEY_LEN];
+
#endif
diff --git a/sys/netinet/in_fib.c b/sys/netinet/in_fib.c
index 4c84de2c7281..031277add777 100644
--- a/sys/netinet/in_fib.c
+++ b/sys/netinet/in_fib.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
#include <net/route/route_ctl.h>
#include <net/route/route_var.h>
#include <net/route/nhop.h>
+#include <net/toeplitz.h>
#include <net/vnet.h>
#include <netinet/in.h>
@@ -62,6 +63,40 @@ __FBSDID("$FreeBSD$");
/* Assert 'struct route_in' is compatible with 'struct route' */
CHK_STRUCT_ROUTE_COMPAT(struct route_in, ro_dst4);
+#ifdef ROUTE_MPATH
+struct _hash_5tuple_ipv4 {
+ struct in_addr src;
+ struct in_addr dst;
+ unsigned short src_port;
+ unsigned short dst_port;
+ char proto;
+ char spare[3];
+};
+_Static_assert(sizeof(struct _hash_5tuple_ipv4) == 16,
+ "_hash_5tuple_ipv4 size is wrong");
+
+
+uint32_t
+fib4_calc_software_hash(struct in_addr src, struct in_addr dst,
+ unsigned short src_port, unsigned short dst_port, char proto,
+ uint32_t *phashtype)
+{
+ struct _hash_5tuple_ipv4 data;
+
+ data.src = src;
+ data.dst = dst;
+ data.src_port = src_port;
+ data.dst_port = dst_port;
+ data.proto = proto;
+ data.spare[0] = data.spare[1] = data.spare[2] = 0;
+
+ *phashtype = M_HASHTYPE_OPAQUE;
+
+ return (toeplitz_hash(MPATH_ENTROPY_KEY_LEN, mpath_entropy_key,
+ sizeof(data), (uint8_t *)&data));
+}
+#endif
+
/*
* Looks up path in fib @fibnum specified by @dst.
* Returns path nexthop on success. Nexthop is safe to use
diff --git a/sys/netinet/in_fib.h b/sys/netinet/in_fib.h
index bc2a2ad6a30e..0d93086bc116 100644
--- a/sys/netinet/in_fib.h
+++ b/sys/netinet/in_fib.h
@@ -51,4 +51,7 @@ int fib4_check_urpf(uint32_t fibnum, struct in_addr dst, uint32_t scopeid,
uint32_t flags, const struct ifnet *src_if);
struct nhop_object *fib4_lookup_debugnet(uint32_t fibnum, struct in_addr dst,
uint32_t scopeid, uint32_t flags);
+uint32_t fib4_calc_software_hash(struct in_addr src, struct in_addr dst,
+ unsigned short src_port, unsigned short dst_port, char proto,
+ uint32_t *phashtype);
#endif
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 30bf72f83e33..37438dd68c71 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_ratelimit.h"
#include "opt_pcbgroup.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -1327,7 +1328,17 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
lport = *lportp;
faddr = sin->sin_addr;
fport = sin->sin_port;
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_val, hash_type;
+ hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
+ inp->inp_socket->so_proto->pr_protocol, &hash_type);
+
+ inp->inp_flowid = hash_val;
+ inp->inp_flowtype = hash_type;
+ }
+#endif
if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
/*
* If the destination address is INADDR_ANY,
diff --git a/sys/netinet/in_rss.c b/sys/netinet/in_rss.c
index f3184175a7cf..05659b97fe7c 100644
--- a/sys/netinet/in_rss.c
+++ b/sys/netinet/in_rss.c
@@ -152,6 +152,48 @@ rss_proto_software_hash_v4(struct in_addr s, struct in_addr d,
}
/*
+ * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
+ * IPv4 source/destination address, UDP or TCP source/destination ports
+ * and the protocol type.
+ *
+ * The protocol code may wish to do a software hash of the given
+ * tuple. This depends upon the currently configured RSS hash types.
+ *
+ * It assumes the packet source/destination address
+ * are in "outgoing" packet order (ie, destination is "far" address.)
+ */
+uint32_t
+xps_proto_software_hash_v4(struct in_addr s, struct in_addr d,
+ u_short sp, u_short dp, int proto, uint32_t *hashtype)
+{
+ uint32_t hash;
+
+ /*
+ * Next, choose the hash type depending upon the protocol
+ * identifier.
+ */
+ if ((proto == IPPROTO_TCP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
+ hash = rss_hash_ip4_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_TCP_IPV4;
+ return (hash);
+ } else if ((proto == IPPROTO_UDP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
+ hash = rss_hash_ip4_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_UDP_IPV4;
+ return (hash);
+ } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
+ /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
+ hash = rss_hash_ip4_2tuple(d, s);
+ *hashtype = M_HASHTYPE_RSS_IPV4;
+ return (hash);
+ }
+
+ *hashtype = M_HASHTYPE_NONE;
+ return (0);
+}
+
+/*
* Do a software calculation of the RSS for the given mbuf.
*
* This is typically used by the input path to recalculate the RSS after
diff --git a/sys/netinet/in_rss.h b/sys/netinet/in_rss.h
index fd300ac5fdff..cdc69bc64709 100644
--- a/sys/netinet/in_rss.h
+++ b/sys/netinet/in_rss.h
@@ -53,5 +53,7 @@ int rss_proto_software_hash_v4(struct in_addr src,
uint32_t *hashtype);
struct mbuf * rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source,
u_int *cpuid);
+uint32_t xps_proto_software_hash_v4(struct in_addr s, struct in_addr d,
+ u_short sp, u_short dp, int proto, uint32_t *hashtype);
#endif /* !_NETINET_IN_RSS_H_ */
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index 6240a77bdb39..a63fc19587f9 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
+#include "opt_route.h"
#include <sys/param.h>
#include <sys/jail.h>
@@ -67,6 +68,7 @@ __FBSDID("$FreeBSD$");
#include <netinet/in.h>
#include <netinet/in_systm.h>
+#include <netinet/in_fib.h>
#include <netinet/in_pcb.h>
#include <netinet/in_var.h>
#include <netinet/if_ether.h>
@@ -484,6 +486,17 @@ rip_output(struct mbuf *m, struct socket *so, ...)
ip->ip_len = htons(m->m_pkthdr.len);
ip->ip_src = inp->inp_laddr;
ip->ip_dst.s_addr = dst;
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib4_calc_software_hash(ip->ip_src,
+ ip->ip_dst, 0, 0, ip->ip_p, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
+ flags |= IP_NODEFAULTFLOWID;
+ }
+#endif
if (jailed(inp->inp_cred)) {
/*
* prison_local_ip4() would be good enough but would
@@ -519,7 +532,17 @@ rip_output(struct mbuf *m, struct socket *so, ...)
return (EINVAL);
ip = mtod(m, struct ip *);
}
-
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib4_calc_software_hash(ip->ip_dst,
+ ip->ip_src, 0, 0, ip->ip_p, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
+ flags |= IP_NODEFAULTFLOWID;
+ }
+#endif
INP_RLOCK(inp);
/*
* Don't allow both user specified and setsockopt options,
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index e1bb5f07c0d0..52304ddd6584 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -76,6 +77,7 @@ __FBSDID("$FreeBSD$");
#include <netinet/in.h>
#include <netinet/in_kdtrace.h>
+#include <netinet/in_fib.h>
#include <netinet/in_pcb.h>
#include <netinet/in_systm.h>
#include <netinet/in_var.h>
@@ -1483,30 +1485,14 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
m->m_pkthdr.flowid = flowid;
M_HASHTYPE_SET(m, flowtype);
}
-#ifdef RSS
- else {
+#if defined(ROUTE_MPATH) || defined(RSS)
+ else if (CALC_FLOWID_OUTBOUND_SENDTO) {
uint32_t hash_val, hash_type;
- /*
- * Calculate an appropriate RSS hash for UDP and
- * UDP Lite.
- *
- * The called function will take care of figuring out
- * whether a 2-tuple or 4-tuple hash is required based
- * on the currently configured scheme.
- *
- * Later later on connected socket values should be
- * cached in the inpcb and reused, rather than constantly
- * re-calculating it.
- *
- * UDP Lite is a different protocol number and will
- * likely end up being hashed as a 2-tuple until
- * RSS / NICs grow UDP Lite protocol awareness.
- */
- if (rss_proto_software_hash_v4(faddr, laddr, fport, lport,
- pr, &hash_val, &hash_type) == 0) {
- m->m_pkthdr.flowid = hash_val;
- M_HASHTYPE_SET(m, hash_type);
- }
+
+ hash_val = fib4_calc_packet_hash(laddr, faddr,
+ lport, fport, pr, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
}
/*
diff --git a/sys/netinet6/in6_fib.c b/sys/netinet6/in6_fib.c
index 9fd869b2730b..a3fd20f8ba25 100644
--- a/sys/netinet6/in6_fib.c
+++ b/sys/netinet6/in6_fib.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
#include <net/route/route_ctl.h>
#include <net/route/route_var.h>
#include <net/route/nhop.h>
+#include <net/toeplitz.h>
#include <net/vnet.h>
#include <netinet/in.h>
@@ -68,6 +69,39 @@ __FBSDID("$FreeBSD$");
CHK_STRUCT_ROUTE_COMPAT(struct route_in6, ro_dst);
+#ifdef ROUTE_MPATH
+struct _hash_5tuple_ipv6 {
+ struct in6_addr src;
+ struct in6_addr dst;
+ unsigned short src_port;
+ unsigned short dst_port;
+ char proto;
+ char spare[3];
+};
+_Static_assert(sizeof(struct _hash_5tuple_ipv6) == 40,
+ "_hash_5tuple_ipv6 size is wrong");
+
+uint32_t
+fib6_calc_software_hash(const struct in6_addr *src, const struct in6_addr *dst,
+ unsigned short src_port, unsigned short dst_port, char proto,
+ uint32_t *phashtype)
+{
+ struct _hash_5tuple_ipv6 data;
+
+ data.src = *src;
+ data.dst = *dst;
+ data.src_port = src_port;
+ data.dst_port = dst_port;
+ data.proto = proto;
+ data.spare[0] = data.spare[1] = data.spare[2] = 0;
+
+ *phashtype = M_HASHTYPE_OPAQUE_HASH;
+
+ return (toeplitz_hash(MPATH_ENTROPY_KEY_LEN, mpath_entropy_key,
+ sizeof(data), (uint8_t *)&data));
+}
+#endif
+
/*
* Looks up path in fib @fibnum specified by @dst.
* Assumes scope is deembedded and provided in @scopeid.
diff --git a/sys/netinet6/in6_fib.h b/sys/netinet6/in6_fib.h
index a716674bce23..78edb5e5b776 100644
--- a/sys/netinet6/in6_fib.h
+++ b/sys/netinet6/in6_fib.h
@@ -39,4 +39,7 @@ int fib6_check_urpf(uint32_t fibnum, const struct in6_addr *dst6,
uint32_t scopeid, uint32_t flags, const struct ifnet *src_if);
struct nhop_object *fib6_lookup_debugnet(uint32_t fibnum,
const struct in6_addr *dst6, uint32_t scopeid, uint32_t flags);
+uint32_t fib6_calc_software_hash(const struct in6_addr *src,
+ const struct in6_addr *dst, unsigned short src_port, unsigned short dst_port,
+ char proto, uint32_t *phashtype);
#endif
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 3444ca1b2fe7..ab498bf91734 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -73,6 +73,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet6.h"
#include "opt_ipsec.h"
#include "opt_pcbgroup.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -423,6 +424,17 @@ in6_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
INP_WLOCK_ASSERT(inp);
INP_HASH_WLOCK_ASSERT(pcbinfo);
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
+ &sin6->sin6_addr, 0, sin6->sin6_port,
+ inp->inp_socket->so_proto->pr_protocol, &hash_type);
+ inp->inp_flowid = hash_val;
+ inp->inp_flowtype = hash_type;
+ }
+#endif
/*
* Call inner routine, to assign local interface address.
* in6_pcbladdr() may automatically fill in sin6_scope_id.
diff --git a/sys/netinet6/in6_rss.c b/sys/netinet6/in6_rss.c
index 72dd0a8831af..2b86d961515b 100644
--- a/sys/netinet6/in6_rss.c
+++ b/sys/netinet6/in6_rss.c
@@ -153,6 +153,50 @@ rss_proto_software_hash_v6(const struct in6_addr *s, const struct in6_addr *d,
}
/*
+ * Calculate an appropriate ipv6 2-tuple or 4-tuple given the given
+ * IPv6 source/destination address, UDP or TCP source/destination ports
+ * and the protocol type.
+ *
+ * The protocol code may wish to do a software hash of the given
+ * tuple. This depends upon the currently configured RSS hash types.
+ *
+ * It assumes the packet source/destination address
+ * are in "outgoin" packet order (ie, destination is "far" address.)
+ */
+uint32_t
+xps_proto_software_hash_v6(const struct in6_addr *s, const struct in6_addr *d,
+ u_short sp, u_short dp, int proto, uint32_t *hashtype)
+{
+
+ uint32_t hash;
+
+ /*
+ * Next, choose the hash type depending upon the protocol
+ * identifier.
+ */
+ if ((proto == IPPROTO_TCP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
+ hash = rss_hash_ip6_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_TCP_IPV6;
+ return (hash);
+ } else if ((proto == IPPROTO_UDP) &&
+ (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
+ hash = rss_hash_ip6_4tuple(d, dp, s, sp);
+ *hashtype = M_HASHTYPE_RSS_UDP_IPV6;
+ return (hash);
+ } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) {
+ /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
+ hash = rss_hash_ip6_2tuple(d, s);
+ *hashtype = M_HASHTYPE_RSS_IPV6;
+ return (hash);
+ }
+
+ *hashtype = M_HASHTYPE_NONE;
+ return (0);
+}
+
+
+/*
* Do a software calculation of the RSS for the given mbuf.
*
* This is typically used by the input path to recalculate the RSS after
diff --git a/sys/netinet6/in6_rss.h b/sys/netinet6/in6_rss.h
index f5b48c71408a..5846e515d185 100644
--- a/sys/netinet6/in6_rss.h
+++ b/sys/netinet6/in6_rss.h
@@ -54,5 +54,8 @@ int rss_proto_software_hash_v6(const struct in6_addr *src,
uint32_t *hashtype);
struct mbuf * rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source,
u_int *cpuid);
+uint32_t xps_proto_software_hash_v6(const struct in6_addr *s,
+ const struct in6_addr *d, u_short sp, u_short dp,
+ int proto, uint32_t *hashtype);
#endif /* !_NETINET6_IN6_RSS_H_ */
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 59aabaa258f5..4e8f22a01b0a 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -419,9 +419,6 @@ done:
*
* ifpp - XXX: just for statistics
*/
-/*
- * XXX TODO: no flowid is assigned for outbound flows?
- */
int
ip6_output(struct mbuf *m0, struct ip6_pktopts *opt,
struct route_in6 *ro, int flags, struct ip6_moptions *im6o,
@@ -775,7 +772,8 @@ again:
}
}
- nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE, 0);
+ nh = fib6_lookup(fibnum, &kdst, scopeid, NHR_NONE,
+ m->m_pkthdr.flowid);
if (nh == NULL) {
IP6STAT_INC(ip6s_noroute);
/* No ifp in6_ifstat_inc(ifp, ifs6_out_discard); */
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index 636828eb4ef4..aea99add4391 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
#include "opt_ipsec.h"
#include "opt_inet6.h"
+#include "opt_route.h"
#include <sys/param.h>
#include <sys/errno.h>
@@ -103,6 +104,7 @@ __FBSDID("$FreeBSD$");
#include <netinet6/ip6_var.h>
#include <netinet6/nd6.h>
#include <netinet6/raw_ip6.h>
+#include <netinet6/in6_fib.h>
#include <netinet6/scope6_var.h>
#include <netinet6/send.h>
@@ -462,6 +464,17 @@ rip6_output(struct mbuf *m, struct socket *so, ...)
}
ip6 = mtod(m, struct ip6_hdr *);
+#ifdef ROUTE_MPATH
+ if (CALC_FLOWID_OUTBOUND) {
+ uint32_t hash_type, hash_val;
+
+ hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
+ &dstsock->sin6_addr, 0, 0, so->so_proto->pr_protocol,
+ &hash_type);
+ inp->inp_flowid = hash_val;
+ inp->inp_flowtype = hash_type;
+ }
+#endif
/*
* Source address selection.
*/
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index 32aa21cb24a1..1535be90e1b0 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_ipsec.h"
+#include "opt_route.h"
#include "opt_rss.h"
#include <sys/param.h>
@@ -115,6 +116,7 @@ __FBSDID("$FreeBSD$");
#include <netinet6/ip6protosw.h>
#include <netinet6/ip6_var.h>
+#include <netinet6/in6_fib.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/in6_rss.h>
#include <netinet6/udp6_var.h>
@@ -954,42 +956,20 @@ udp6_output(struct socket *so, int flags_arg, struct mbuf *m,
}
flags = 0;
-#ifdef RSS
- {
- uint32_t hash_val, hash_type;
+#if defined(ROUTE_MPATH) || defined(RSS)
+ if (CALC_FLOWID_OUTBOUND_SENDTO) {
+ uint32_t hash_type, hash_val;
uint8_t pr;
pr = inp->inp_socket->so_proto->pr_protocol;
- /*
- * Calculate an appropriate RSS hash for UDP and
- * UDP Lite.
- *
- * The called function will take care of figuring out
- * whether a 2-tuple or 4-tuple hash is required based
- * on the currently configured scheme.
- *
- * Later later on connected socket values should be
- * cached in the inpcb and reused, rather than constantly
- * re-calculating it.
- *
- * UDP Lite is a different protocol number and will
- * likely end up being hashed as a 2-tuple until
- * RSS / NICs grow UDP Lite protocol awareness.
- */
- if (rss_proto_software_hash_v6(faddr, laddr, fport,
- inp->inp_lport, pr, &hash_val, &hash_type) == 0) {
- m->m_pkthdr.flowid = hash_val;
- M_HASHTYPE_SET(m, hash_type);
- }
- /*
- * Don't override with the inp cached flowid.
- *
- * Until the whole UDP path is vetted, it may actually
- * be incorrect.
- */
- flags |= IP_NODEFAULTFLOWID;
+ hash_val = fib6_calc_packet_hash(laddr, faddr,
+ inp->inp_lport, fport, pr, &hash_type);
+ m->m_pkthdr.flowid = hash_val;
+ M_HASHTYPE_SET(m, hash_type);
}
+ /* do not use inp flowid */
+ flags |= IP_NODEFAULTFLOWID;
#endif
UDPSTAT_INC(udps_opackets);
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-07 03:26:15 +0000