Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3270,9 +3270,11 @@ net/raw_cb.c standard net/raw_usrreq.c standard net/route.c standard +net/rss_config.c optional inet rss | inet6 rss net/rtsock.c standard net/slcompress.c optional netgraph_vjc | sppp | \ netgraph_sppp +net/toeplitz.c optional inet rss | inet6 rss net/vnet.c optional vimage net/zlib.c optional crypto | geom_uzip | ipsec | \ mxge | netgraph_deflate | \ @@ -3422,7 +3424,7 @@ netinet/in_pcbgroup.c optional inet pcbgroup | inet6 pcbgroup netinet/in_proto.c optional inet | inet6 netinet/in_rmx.c optional inet -netinet/in_rss.c optional inet rss | inet6 rss +netinet/in_rss.c optional inet rss netinet/ip_divert.c optional inet ipdivert ipfirewall netinet/ip_ecn.c optional inet | inet6 netinet/ip_encap.c optional inet | inet6 @@ -3465,7 +3467,6 @@ netinet/tcp_timer.c optional inet | inet6 netinet/tcp_timewait.c optional inet | inet6 netinet/tcp_usrreq.c optional inet | inet6 -netinet/toeplitz.c optional inet rss | inet6 rss netinet/udp_usrreq.c optional inet | inet6 netinet/libalias/alias.c optional libalias inet | netgraph_nat inet netinet/libalias/alias_db.c optional libalias inet | netgraph_nat inet @@ -3485,6 +3486,7 @@ netinet6/in6_pcbgroup.c optional inet6 pcbgroup netinet6/in6_proto.c optional inet6 netinet6/in6_rmx.c optional inet6 +netinet6/in6_rss.c optional inet6 rss netinet6/in6_src.c optional inet6 netinet6/ip6_forward.c optional inet6 netinet6/ip6_gre.c optional gre inet6 Index: sys/dev/e1000/if_igb.c =================================================================== --- sys/dev/e1000/if_igb.c +++ sys/dev/e1000/if_igb.c @@ -73,6 +73,9 @@ #include #include #include +#ifdef RSS +#include +#endif #include #include @@ -85,9 +88,6 @@ #include #include #include -#ifdef RSS -#include -#endif #include #include Index: sys/dev/ixgbe/ixgbe.c =================================================================== --- sys/dev/ixgbe/ixgbe.c +++ sys/dev/ixgbe/ixgbe.c @@ -39,7 +39,7 @@ #include "ixgbe.h" #ifdef RSS -#include +#include #endif /********************************************************************* Index: sys/dev/ixl/if_ixl.c =================================================================== --- sys/dev/ixl/if_ixl.c +++ sys/dev/ixl/if_ixl.c @@ -38,6 +38,10 @@ #include "ixl.h" #include "ixl_pf.h" +#ifdef RSS +#include +#endif /* RSS */ + /********************************************************************* * Driver version *********************************************************************/ @@ -3249,7 +3253,7 @@ * num_queues.) */ que_id = rss_get_indirection_to_bucket(i); - que_id = que_id % adapter->num_queues; + que_id = que_id % vsi->num_queues; #else que_id = j; #endif Index: sys/dev/ixl/if_ixlv.c =================================================================== --- sys/dev/ixl/if_ixlv.c +++ sys/dev/ixl/if_ixlv.c @@ -38,6 +38,10 @@ #include "ixl.h" #include "ixlv.h" +#ifdef RSS +#include +#endif /* RSS */ + /********************************************************************* * Driver version *********************************************************************/ @@ -2600,8 +2604,10 @@ set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); +#if 0 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) set_hena |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; +#endif #else set_hena = ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | @@ -2633,7 +2639,7 @@ * num_queues.) */ que_id = rss_get_indirection_to_bucket(i); - que_id = que_id % adapter->num_queues; + que_id = que_id % vsi->num_queues; #else que_id = j; #endif Index: sys/dev/ixl/ixl_txrx.c =================================================================== --- sys/dev/ixl/ixl_txrx.c +++ sys/dev/ixl/ixl_txrx.c @@ -43,6 +43,10 @@ #include "opt_rss.h" #include "ixl.h" +#ifdef RSS +#include +#endif /* RSS */ + /* Local Prototypes */ static void ixl_rx_checksum(struct mbuf *, u32, u32, u8); static void ixl_refresh_mbufs(struct ixl_queue *, int); @@ -1374,9 +1378,9 @@ ixl_ptype_to_hash(u8 ptype) { struct i40e_rx_ptype_decoded decoded; - u8 ex = 0 + u8 ex = 0; - decode = decode_rx_desc_ptype(ptype); + decoded = decode_rx_desc_ptype(ptype); ex = decoded.outer_frag; if (!decoded.known) Index: sys/net/if_ethersubr.c =================================================================== --- sys/net/if_ethersubr.c +++ sys/net/if_ethersubr.c @@ -63,6 +63,7 @@ #include #include #include +#include #include #include @@ -71,7 +72,6 @@ #include #include #include -#include #include #include #endif Index: sys/net/rss_config.h =================================================================== --- sys/net/rss_config.h +++ sys/net/rss_config.h @@ -26,11 +26,11 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: head/sys/netinet/in_rss.h 276484 2014-12-31 22:52:43Z adrian $ + * $FreeBSD: 276484 2014-12-31 22:52:43Z adrian $ */ -#ifndef _NETINET_IN_RSS_H_ -#define _NETINET_IN_RSS_H_ +#ifndef _NET_RSS_CONFIG_H_ +#define _NET_RSS_CONFIG_H_ #include /* in_addr_t */ @@ -107,36 +107,17 @@ u_int rss_gethashconfig(void); /* - * Network stack interface to generate a hash for a protocol tuple. + * Hash calculation functions. */ -uint32_t rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, - struct in_addr dst, u_short dstport); -uint32_t rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst); -uint32_t rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport, - const struct in6_addr *dst, u_short dstport); -uint32_t rss_hash_ip6_2tuple(const struct in6_addr *src, - const struct in6_addr *dst); +uint32_t rss_hash(u_int datalen, const uint8_t *data); /* * Network stack interface to query desired CPU affinity of a packet. */ -struct mbuf *rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid); -u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type); -int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, - uint32_t *bucket_id); -int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id); - -/* - * Functions to calculate a software RSS hash for a given mbuf or - * packet detail. - */ -int rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, - uint32_t *hashval, uint32_t *hashtype); -int rss_proto_software_hash_v4(struct in_addr src, - struct in_addr dst, u_short src_port, u_short dst_port, - int proto, uint32_t *hashval, - uint32_t *hashtype); -struct mbuf * rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, - u_int *cpuid); +struct mbuf * rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid); +u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type); +int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, + uint32_t *bucket_id); +int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id); -#endif /* !_NETINET_IN_RSS_H_ */ +#endif /* !_NET_RSS_CONFIG_H_ */ Index: sys/net/rss_config.c =================================================================== --- sys/net/rss_config.c +++ sys/net/rss_config.c @@ -29,7 +29,7 @@ #include -__FBSDID("$FreeBSD: head/sys/netinet/in_rss.c 276484 2014-12-31 22:52:43Z adrian $"); +__FBSDID("$FreeBSD: 276484 2014-12-31 22:52:43Z adrian $"); #include "opt_inet6.h" #include "opt_pcbgroup.h" @@ -50,17 +50,20 @@ #include #include #include +#include +#include +#if 0 #include #include #include #include -#include /* for software rss hash support */ #include #include #include +#endif /*- * Operating system parts of receiver-side scaling (RSS), which allows @@ -89,6 +92,7 @@ * placement and pcbgroup expectations. */ +SYSCTL_DECL(_net_inet); SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); /* @@ -175,8 +179,6 @@ }; static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; -static inline u_int rss_gethashconfig_local(void); - static void rss_init(__unused void *arg) { @@ -282,7 +284,7 @@ return (v); } -static uint32_t +uint32_t rss_hash(u_int datalen, const uint8_t *data) { @@ -302,88 +304,6 @@ } /* - * Hash an IPv4 2-tuple. - */ -uint32_t -rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst) -{ - uint8_t data[sizeof(src) + sizeof(dst)]; - u_int datalen; - - datalen = 0; - bcopy(&src, &data[datalen], sizeof(src)); - datalen += sizeof(src); - bcopy(&dst, &data[datalen], sizeof(dst)); - datalen += sizeof(dst); - return (rss_hash(datalen, data)); -} - -/* - * Hash an IPv4 4-tuple. - */ -uint32_t -rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst, - u_short dstport) -{ - uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) + - sizeof(dstport)]; - u_int datalen; - - datalen = 0; - bcopy(&src, &data[datalen], sizeof(src)); - datalen += sizeof(src); - bcopy(&dst, &data[datalen], sizeof(dst)); - datalen += sizeof(dst); - bcopy(&srcport, &data[datalen], sizeof(srcport)); - datalen += sizeof(srcport); - bcopy(&dstport, &data[datalen], sizeof(dstport)); - datalen += sizeof(dstport); - return (rss_hash(datalen, data)); -} - -#ifdef INET6 -/* - * Hash an IPv6 2-tuple. - */ -uint32_t -rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst) -{ - uint8_t data[sizeof(*src) + sizeof(*dst)]; - u_int datalen; - - datalen = 0; - bcopy(src, &data[datalen], sizeof(*src)); - datalen += sizeof(*src); - bcopy(dst, &data[datalen], sizeof(*dst)); - datalen += sizeof(*dst); - return (rss_hash(datalen, data)); -} - -/* - * Hash an IPv6 4-tuple. - */ -uint32_t -rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport, - const struct in6_addr *dst, u_short dstport) -{ - uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) + - sizeof(dstport)]; - u_int datalen; - - datalen = 0; - bcopy(src, &data[datalen], sizeof(*src)); - datalen += sizeof(*src); - bcopy(dst, &data[datalen], sizeof(*dst)); - datalen += sizeof(*dst); - bcopy(&srcport, &data[datalen], sizeof(srcport)); - datalen += sizeof(srcport); - bcopy(&dstport, &data[datalen], sizeof(dstport)); - datalen += sizeof(dstport); - return (rss_hash(datalen, data)); -} -#endif /* INET6 */ - -/* * Query the number of RSS bits in use. */ u_int @@ -498,260 +418,6 @@ } /* - * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given - * IPv4 source/destination address, UDP or TCP source/destination ports - * and the protocol type. - * - * The protocol code may wish to do a software hash of the given - * tuple. This depends upon the currently configured RSS hash types. - * - * This assumes that the packet in question isn't a fragment. - * - * It also assumes the packet source/destination address - * are in "incoming" packet order (ie, source is "far" address.) - */ -int -rss_proto_software_hash_v4(struct in_addr s, struct in_addr d, - u_short sp, u_short dp, int proto, - uint32_t *hashval, uint32_t *hashtype) -{ - uint32_t hash; - - /* - * Next, choose the hash type depending upon the protocol - * identifier. - */ - if ((proto == IPPROTO_TCP) && - (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4)) { - hash = rss_hash_ip4_4tuple(s, sp, d, dp); - *hashval = hash; - *hashtype = M_HASHTYPE_RSS_TCP_IPV4; - return (0); - } else if ((proto == IPPROTO_UDP) && - (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4)) { - hash = rss_hash_ip4_4tuple(s, sp, d, dp); - *hashval = hash; - *hashtype = M_HASHTYPE_RSS_UDP_IPV4; - return (0); - } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) { - /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */ - hash = rss_hash_ip4_2tuple(s, d); - *hashval = hash; - *hashtype = M_HASHTYPE_RSS_IPV4; - return (0); - } - - /* No configured available hashtypes! */ - printf("%s: no available hashtypes!\n", __func__); - return (-1); -} - -/* - * Do a software calculation of the RSS for the given mbuf. - * - * This is typically used by the input path to recalculate the RSS after - * some form of packet processing (eg de-capsulation, IP fragment reassembly.) - * - * dir is the packet direction - RSS_HASH_PKT_INGRESS for incoming and - * RSS_HASH_PKT_EGRESS for outgoing. - * - * Returns 0 if a hash was done, -1 if no hash was done, +1 if - * the mbuf already had a valid RSS flowid. - * - * This function doesn't modify the mbuf. It's up to the caller to - * assign flowid/flowtype as appropriate. - */ -int -rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval, - uint32_t *hashtype) -{ - const struct ip *ip; - const struct tcphdr *th; - const struct udphdr *uh; - uint32_t flowid; - uint32_t flowtype; - uint8_t proto; - int iphlen; - int is_frag = 0; - - /* - * XXX For now this only handles hashing on incoming mbufs. - */ - if (dir != RSS_HASH_PKT_INGRESS) { - printf("%s: called on EGRESS packet!\n", __func__); - return (-1); - } - - /* - * First, validate that the mbuf we have is long enough - * to have an IPv4 header in it. - */ - if (m->m_pkthdr.len < (sizeof(struct ip))) { - printf("%s: short mbuf pkthdr\n", __func__); - return (-1); - } - if (m->m_len < (sizeof(struct ip))) { - printf("%s: short mbuf len\n", __func__); - return (-1); - } - - /* Ok, let's dereference that */ - ip = mtod(m, struct ip *); - proto = ip->ip_p; - iphlen = ip->ip_hl << 2; - - /* - * If this is a fragment then it shouldn't be four-tuple - * hashed just yet. Once it's reassembled into a full - * frame it should be re-hashed. - */ - if (ip->ip_off & htons(IP_MF | IP_OFFMASK)) - is_frag = 1; - - /* - * If the mbuf flowid/flowtype matches the packet type, - * and we don't support the 4-tuple version of the given protocol, - * then signal to the owner that it can trust the flowid/flowtype - * details. - * - * This is a little picky - eg, if TCPv4 / UDPv4 hashing - * is supported but we got a TCP/UDP frame only 2-tuple hashed, - * then we shouldn't just "trust" the 2-tuple hash. We need - * a 4-tuple hash. - */ - flowid = m->m_pkthdr.flowid; - flowtype = M_HASHTYPE_GET(m); - - if (flowtype != M_HASHTYPE_NONE) { - switch (proto) { - case IPPROTO_UDP: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) && - (flowtype == M_HASHTYPE_RSS_UDP_IPV4) && - (is_frag == 0)) { - return (1); - } - /* - * Only allow 2-tuple for UDP frames if we don't also - * support 4-tuple for UDP. - */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) && - flowtype == M_HASHTYPE_RSS_IPV4) { - return (1); - } - break; - case IPPROTO_TCP: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) && - (flowtype == M_HASHTYPE_RSS_TCP_IPV4) && - (is_frag == 0)) { - return (1); - } - /* - * Only allow 2-tuple for TCP frames if we don't also - * support 2-tuple for TCP. - */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) && - flowtype == M_HASHTYPE_RSS_IPV4) { - return (1); - } - break; - default: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - flowtype == M_HASHTYPE_RSS_IPV4) { - return (1); - } - break; - } - } - - /* - * Decode enough information to make a hash decision. - * - * XXX TODO: does the hardware hash on 4-tuple if IP - * options are present? - */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) && - (proto == IPPROTO_TCP) && - (is_frag == 0)) { - if (m->m_len < iphlen + sizeof(struct tcphdr)) { - printf("%s: short TCP frame?\n", __func__); - return (-1); - } - th = (struct tcphdr *)((caddr_t)ip + iphlen); - return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst, - th->th_sport, - th->th_dport, - proto, - hashval, - hashtype); - } else if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) && - (proto == IPPROTO_UDP) && - (is_frag == 0)) { - uh = (struct udphdr *)((caddr_t)ip + iphlen); - if (m->m_len < iphlen + sizeof(struct udphdr)) { - printf("%s: short UDP frame?\n", __func__); - return (-1); - } - return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst, - uh->uh_sport, - uh->uh_dport, - proto, - hashval, - hashtype); - } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) { - /* Default to 2-tuple hash */ - return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst, - 0, /* source port */ - 0, /* destination port */ - 0, /* IPPROTO_IP */ - hashval, - hashtype); - } else { - printf("%s: no available hashtypes!\n", __func__); - return (-1); - } -} - -/* - * Similar to rss_m2cpuid, but designed to be used by the IP NETISR - * on incoming frames. - * - * If an existing RSS hash exists and it matches what the configured - * hashing is, then use it. - * - * If there's an existing RSS hash but the desired hash is different, - * or if there's no useful RSS hash, then calculate it via - * the software path. - * - * XXX TODO: definitely want statistics here! - */ -struct mbuf * -rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) -{ - uint32_t hash_val, hash_type; - int ret; - - M_ASSERTPKTHDR(m); - - ret = rss_mbuf_software_hash_v4(m, RSS_HASH_PKT_INGRESS, - &hash_val, &hash_type); - if (ret > 0) { - /* mbuf has a valid hash already; don't need to modify it */ - *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); - } else if (ret == 0) { - /* hash was done; update */ - m->m_pkthdr.flowid = hash_val; - M_HASHTYPE_SET(m, hash_type); - *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); - } else { /* ret < 0 */ - /* no hash was done */ - *cpuid = NETISR_CPUID_NONE; - } - return (m); -} - -/* * Query the RSS hash algorithm. */ u_int @@ -799,8 +465,14 @@ return (rss_ncpus); } -static inline u_int -rss_gethashconfig_local(void) +/* + * Return the supported RSS hash configuration. + * + * NICs should query this to determine what to configure in their redirection + * matching table. + */ +inline u_int +rss_gethashconfig(void) { /* Return 4-tuple for TCP; 2-tuple for others */ @@ -829,19 +501,6 @@ } /* - * Return the supported RSS hash configuration. - * - * NICs should query this to determine what to configure in their redirection - * matching table. - */ -u_int -rss_gethashconfig(void) -{ - - return (rss_gethashconfig_local()); -} - -/* * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want * it appearing in debugging output unnecessarily. */ Index: sys/net/toeplitz.c =================================================================== --- sys/net/toeplitz.c +++ sys/net/toeplitz.c @@ -29,8 +29,8 @@ #include -#include -#include +#include +#include #include Index: sys/netinet/in_pcb.c =================================================================== --- sys/netinet/in_pcb.c +++ sys/netinet/in_pcb.c @@ -71,12 +71,13 @@ #include #include #include +#include #include #if defined(INET) || defined(INET6) #include #include -#include +//#include #include #include #include @@ -87,6 +88,7 @@ #endif #ifdef INET6 #include +//#include #include #include #include Index: sys/netinet/in_pcbgroup.c =================================================================== --- sys/netinet/in_pcbgroup.c +++ sys/netinet/in_pcbgroup.c @@ -42,7 +42,10 @@ #include #include +#include + #include + #include #include #ifdef INET6 Index: sys/netinet/in_rss.h =================================================================== --- sys/netinet/in_rss.h +++ sys/netinet/in_rss.h @@ -35,96 +35,11 @@ #include /* in_addr_t */ /* - * Supported RSS hash functions. - */ -#define RSS_HASH_NAIVE 0x00000001 /* Poor but fast hash. */ -#define RSS_HASH_TOEPLITZ 0x00000002 /* Required by RSS. */ -#define RSS_HASH_CRC32 0x00000004 /* Future; some NICs do it. */ - -#define RSS_HASH_MASK (RSS_HASH_NAIVE | RSS_HASH_TOEPLITZ) - -/* - * Instances of struct inpcbinfo declare an RSS hash type indicating what - * header fields are covered. - */ -#define RSS_HASHFIELDS_NONE 0 -#define RSS_HASHFIELDS_4TUPLE 1 -#define RSS_HASHFIELDS_2TUPLE 2 - -/* - * Define RSS representations of the M_HASHTYPE_* values, representing - * which particular bits are supported. The NICs can then use this to - * calculate which hash types to enable and which not to enable. - * - * The fact that these line up with M_HASHTYPE_* is not to be relied - * upon. - */ -#define RSS_HASHTYPE_RSS_IPV4 (1 << 1) /* IPv4 2-tuple */ -#define RSS_HASHTYPE_RSS_TCP_IPV4 (1 << 2) /* TCPv4 4-tuple */ -#define RSS_HASHTYPE_RSS_IPV6 (1 << 3) /* IPv6 2-tuple */ -#define RSS_HASHTYPE_RSS_TCP_IPV6 (1 << 4) /* TCPv6 4-tuple */ -#define RSS_HASHTYPE_RSS_IPV6_EX (1 << 5) /* IPv6 2-tuple + ext hdrs */ -#define RSS_HASHTYPE_RSS_TCP_IPV6_EX (1 << 6) /* TCPv6 4-tiple + ext hdrs */ -#define RSS_HASHTYPE_RSS_UDP_IPV4 (1 << 7) /* IPv4 UDP 4-tuple */ -#define RSS_HASHTYPE_RSS_UDP_IPV4_EX (1 << 8) /* IPv4 UDP 4-tuple + ext hdrs */ -#define RSS_HASHTYPE_RSS_UDP_IPV6 (1 << 9) /* IPv6 UDP 4-tuple */ -#define RSS_HASHTYPE_RSS_UDP_IPV6_EX (1 << 10) /* IPv6 UDP 4-tuple + ext hdrs */ - -/* - * Compile-time limits on the size of the indirection table. - */ -#define RSS_MAXBITS 7 -#define RSS_TABLE_MAXLEN (1 << RSS_MAXBITS) - -/* - * Maximum key size used throughout. It's OK for hardware to use only the - * first 16 bytes, which is all that's required for IPv4. - */ -#define RSS_KEYSIZE 40 - -/* - * For RSS hash methods that do a software hash on an mbuf, the packet - * direction (ingress / egress) is required. - * - * The default direction (INGRESS) is the "receive into the NIC" - ie, - * what the hardware is hashing on. - */ -#define RSS_HASH_PKT_INGRESS 0 -#define RSS_HASH_PKT_EGRESS 1 - -/* - * Device driver interfaces to query RSS properties that must be programmed - * into hardware. - */ -u_int rss_getbits(void); -u_int rss_getbucket(u_int hash); -u_int rss_get_indirection_to_bucket(u_int index); -u_int rss_getcpu(u_int bucket); -void rss_getkey(uint8_t *key); -u_int rss_gethashalgo(void); -u_int rss_getnumbuckets(void); -u_int rss_getnumcpus(void); -u_int rss_gethashconfig(void); - -/* * Network stack interface to generate a hash for a protocol tuple. */ uint32_t rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst, u_short dstport); uint32_t rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst); -uint32_t rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport, - const struct in6_addr *dst, u_short dstport); -uint32_t rss_hash_ip6_2tuple(const struct in6_addr *src, - const struct in6_addr *dst); - -/* - * Network stack interface to query desired CPU affinity of a packet. - */ -struct mbuf *rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid); -u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type); -int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, - uint32_t *bucket_id); -int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id); /* * Functions to calculate a software RSS hash for a given mbuf or Index: sys/netinet/in_rss.c =================================================================== --- sys/netinet/in_rss.c +++ sys/netinet/in_rss.c @@ -50,257 +50,18 @@ #include #include #include +#include #include #include #include #include -#include /* for software rss hash support */ #include #include #include -/*- - * Operating system parts of receiver-side scaling (RSS), which allows - * network cards to direct flows to particular receive queues based on hashes - * of header tuples. This implementation aligns RSS buckets with connection - * groups at the TCP/IP layer, so each bucket is associated with exactly one - * group. As a result, the group lookup structures (and lock) should have an - * effective affinity with exactly one CPU. - * - * Network device drivers needing to configure RSS will query this framework - * for parameters, such as the current RSS key, hashing policies, number of - * bits, and indirection table mapping hashes to buckets and CPUs. They may - * provide their own supplementary information, such as queue<->CPU bindings. - * It is the responsibility of the network device driver to inject packets - * into the stack on as close to the right CPU as possible, if playing by RSS - * rules. - * - * TODO: - * - * - Synchronization for rss_key and other future-configurable parameters. - * - Event handler drivers can register to pick up RSS configuration changes. - * - Should we allow rss_basecpu to be configured? - * - Randomize key on boot. - * - IPv6 support. - * - Statistics on how often there's a misalignment between hardware - * placement and pcbgroup expectations. - */ - -SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); - -/* - * Toeplitz is the only required hash function in the RSS spec, so use it by - * default. - */ -static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; -SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, - "RSS hash algorithm"); - -/* - * Size of the indirection table; at most 128 entries per the RSS spec. We - * size it to at least 2 times the number of CPUs by default to allow useful - * rebalancing. If not set explicitly with a loader tunable, we tune based - * on the number of CPUs present. - * - * XXXRW: buckets might be better to use for the tunable than bits. - */ -static u_int rss_bits; -SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, - "RSS bits"); - -static u_int rss_mask; -SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, - "RSS mask"); - -static const u_int rss_maxbits = RSS_MAXBITS; -SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, - __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); - -/* - * RSS's own count of the number of CPUs it could be using for processing. - * Bounded to 64 by RSS constants. - */ -static u_int rss_ncpus; -SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, - "Number of CPUs available to RSS"); - -#define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) -static const u_int rss_maxcpus = RSS_MAXCPUS; -SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, - __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); - -/* - * Variable exists just for reporting rss_bits in a user-friendly way. - */ -static u_int rss_buckets; -SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, - "RSS buckets"); - -/* - * Base CPU number; devices will add this to all CPU numbers returned by the - * RSS indirection table. Currently unmodifable in FreeBSD. - */ -static const u_int rss_basecpu; -SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, - __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); - -/* - * RSS secret key, intended to prevent attacks on load-balancing. Its - * effectiveness may be limited by algorithm choice and available entropy - * during the boot. - * - * XXXRW: And that we don't randomize it yet! - * - * This is the default Microsoft RSS specification key which is also - * the Chelsio T5 firmware default key. - */ -static uint8_t rss_key[RSS_KEYSIZE] = { - 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, - 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, - 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, - 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, - 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, -}; - -/* - * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. - * Drivers may supplement this table with a seperate CPU<->queue table when - * programming devices. - */ -struct rss_table_entry { - uint8_t rte_cpu; /* CPU affinity of bucket. */ -}; -static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; - -static inline u_int rss_gethashconfig_local(void); - -static void -rss_init(__unused void *arg) -{ - u_int i; - u_int cpuid; - - /* - * Validate tunables, coerce to sensible values. - */ - switch (rss_hashalgo) { - case RSS_HASH_TOEPLITZ: - case RSS_HASH_NAIVE: - break; - - default: - printf("%s: invalid RSS hashalgo %u, coercing to %u", - __func__, rss_hashalgo, RSS_HASH_TOEPLITZ); - rss_hashalgo = RSS_HASH_TOEPLITZ; - } - - /* - * Count available CPUs. - * - * XXXRW: Note incorrect assumptions regarding contiguity of this set - * elsewhere. - */ - rss_ncpus = 0; - for (i = 0; i <= mp_maxid; i++) { - if (CPU_ABSENT(i)) - continue; - rss_ncpus++; - } - if (rss_ncpus > RSS_MAXCPUS) - rss_ncpus = RSS_MAXCPUS; - - /* - * Tune RSS table entries to be no less than 2x the number of CPUs - * -- unless we're running uniprocessor, in which case there's not - * much point in having buckets to rearrange for load-balancing! - */ - if (rss_ncpus > 1) { - if (rss_bits == 0) - rss_bits = fls(rss_ncpus - 1) + 1; - - /* - * Microsoft limits RSS table entries to 128, so apply that - * limit to both auto-detected CPU counts and user-configured - * ones. - */ - if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { - printf("%s: RSS bits %u not valid, coercing to %u", - __func__, rss_bits, RSS_MAXBITS); - rss_bits = RSS_MAXBITS; - } - - /* - * Figure out how many buckets to use; warn if less than the - * number of configured CPUs, although this is not a fatal - * problem. - */ - rss_buckets = (1 << rss_bits); - if (rss_buckets < rss_ncpus) - printf("%s: WARNING: rss_buckets (%u) less than " - "rss_ncpus (%u)\n", __func__, rss_buckets, - rss_ncpus); - rss_mask = rss_buckets - 1; - } else { - rss_bits = 0; - rss_buckets = 1; - rss_mask = 0; - } - - /* - * Set up initial CPU assignments: round-robin by default. - */ - cpuid = CPU_FIRST(); - for (i = 0; i < rss_buckets; i++) { - rss_table[i].rte_cpu = cpuid; - cpuid = CPU_NEXT(cpuid); - } - - /* - * Randomize rrs_key. - * - * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() - * loop to check for "bad" RSS keys. - */ -} -SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); - -static uint32_t -rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, - const uint8_t *data) -{ - uint32_t v; - u_int i; - - v = 0; - for (i = 0; i < keylen; i++) - v += key[i]; - for (i = 0; i < datalen; i++) - v += data[i]; - return (v); -} - -static uint32_t -rss_hash(u_int datalen, const uint8_t *data) -{ - - switch (rss_hashalgo) { - case RSS_HASH_TOEPLITZ: - return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, - data)); - - case RSS_HASH_NAIVE: - return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, - data)); - - default: - panic("%s: unsupported/unknown hashalgo %d", __func__, - rss_hashalgo); - } -} - /* * Hash an IPv4 2-tuple. */ @@ -341,162 +102,6 @@ return (rss_hash(datalen, data)); } -#ifdef INET6 -/* - * Hash an IPv6 2-tuple. - */ -uint32_t -rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst) -{ - uint8_t data[sizeof(*src) + sizeof(*dst)]; - u_int datalen; - - datalen = 0; - bcopy(src, &data[datalen], sizeof(*src)); - datalen += sizeof(*src); - bcopy(dst, &data[datalen], sizeof(*dst)); - datalen += sizeof(*dst); - return (rss_hash(datalen, data)); -} - -/* - * Hash an IPv6 4-tuple. - */ -uint32_t -rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport, - const struct in6_addr *dst, u_short dstport) -{ - uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) + - sizeof(dstport)]; - u_int datalen; - - datalen = 0; - bcopy(src, &data[datalen], sizeof(*src)); - datalen += sizeof(*src); - bcopy(dst, &data[datalen], sizeof(*dst)); - datalen += sizeof(*dst); - bcopy(&srcport, &data[datalen], sizeof(srcport)); - datalen += sizeof(srcport); - bcopy(&dstport, &data[datalen], sizeof(dstport)); - datalen += sizeof(dstport); - return (rss_hash(datalen, data)); -} -#endif /* INET6 */ - -/* - * Query the number of RSS bits in use. - */ -u_int -rss_getbits(void) -{ - - return (rss_bits); -} - -/* - * Query the RSS bucket associated with an RSS hash. - */ -u_int -rss_getbucket(u_int hash) -{ - - return (hash & rss_mask); -} - -/* - * Query the RSS layer bucket associated with the given - * entry in the RSS hash space. - * - * The RSS indirection table is 0 .. rss_buckets-1, - * covering the low 'rss_bits' of the total 128 slot - * RSS indirection table. So just mask off rss_bits and - * return that. - * - * NIC drivers can then iterate over the 128 slot RSS - * indirection table and fetch which RSS bucket to - * map it to. This will typically be a CPU queue - */ -u_int -rss_get_indirection_to_bucket(u_int index) -{ - - return (index & rss_mask); -} - -/* - * Query the RSS CPU associated with an RSS bucket. - */ -u_int -rss_getcpu(u_int bucket) -{ - - return (rss_table[bucket].rte_cpu); -} - -/* - * netisr CPU affinity lookup given just the hash and hashtype. - */ -u_int -rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) -{ - - switch (hash_type) { - case M_HASHTYPE_RSS_IPV4: - case M_HASHTYPE_RSS_TCP_IPV4: - case M_HASHTYPE_RSS_UDP_IPV4: - case M_HASHTYPE_RSS_IPV6: - case M_HASHTYPE_RSS_TCP_IPV6: - case M_HASHTYPE_RSS_UDP_IPV6: - return (rss_getcpu(rss_getbucket(hash_val))); - default: - return (NETISR_CPUID_NONE); - } -} - -/* - * Query the RSS bucket associated with the given hash value and - * type. - */ -int -rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) -{ - - switch (hash_type) { - case M_HASHTYPE_RSS_IPV4: - case M_HASHTYPE_RSS_TCP_IPV4: - case M_HASHTYPE_RSS_UDP_IPV4: - case M_HASHTYPE_RSS_IPV6: - case M_HASHTYPE_RSS_TCP_IPV6: - case M_HASHTYPE_RSS_UDP_IPV6: - *bucket_id = rss_getbucket(hash_val); - return (0); - default: - return (-1); - } -} - -/* - * netisr CPU affinity lookup routine for use by protocols. - */ -struct mbuf * -rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) -{ - - M_ASSERTPKTHDR(m); - *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); - return (m); -} - -int -rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) -{ - - M_ASSERTPKTHDR(m); - - return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), - bucket_id)); -} - /* * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given * IPv4 source/destination address, UDP or TCP source/destination ports @@ -522,18 +127,18 @@ * identifier. */ if ((proto == IPPROTO_TCP) && - (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4)) { + (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) { hash = rss_hash_ip4_4tuple(s, sp, d, dp); *hashval = hash; *hashtype = M_HASHTYPE_RSS_TCP_IPV4; return (0); } else if ((proto == IPPROTO_UDP) && - (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4)) { + (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) { hash = rss_hash_ip4_4tuple(s, sp, d, dp); *hashval = hash; *hashtype = M_HASHTYPE_RSS_UDP_IPV4; return (0); - } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) { + } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) { /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */ hash = rss_hash_ip4_2tuple(s, d); *hashval = hash; @@ -625,7 +230,7 @@ if (flowtype != M_HASHTYPE_NONE) { switch (proto) { case IPPROTO_UDP: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) && + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) && (flowtype == M_HASHTYPE_RSS_UDP_IPV4) && (is_frag == 0)) { return (1); @@ -634,14 +239,14 @@ * Only allow 2-tuple for UDP frames if we don't also * support 4-tuple for UDP. */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) && + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) && + ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) && flowtype == M_HASHTYPE_RSS_IPV4) { return (1); } break; case IPPROTO_TCP: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) && + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) && (flowtype == M_HASHTYPE_RSS_TCP_IPV4) && (is_frag == 0)) { return (1); @@ -650,14 +255,14 @@ * Only allow 2-tuple for TCP frames if we don't also * support 2-tuple for TCP. */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) && + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) && + ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) && flowtype == M_HASHTYPE_RSS_IPV4) { return (1); } break; default: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) && flowtype == M_HASHTYPE_RSS_IPV4) { return (1); } @@ -671,7 +276,7 @@ * XXX TODO: does the hardware hash on 4-tuple if IP * options are present? */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) && + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) && (proto == IPPROTO_TCP) && (is_frag == 0)) { if (m->m_len < iphlen + sizeof(struct tcphdr)) { @@ -685,7 +290,7 @@ proto, hashval, hashtype); - } else if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) && + } else if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) && (proto == IPPROTO_UDP) && (is_frag == 0)) { uh = (struct udphdr *)((caddr_t)ip + iphlen); @@ -699,7 +304,7 @@ proto, hashval, hashtype); - } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) { + } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) { /* Default to 2-tuple hash */ return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst, 0, /* source port */ @@ -750,150 +355,3 @@ } return (m); } - -/* - * Query the RSS hash algorithm. - */ -u_int -rss_gethashalgo(void) -{ - - return (rss_hashalgo); -} - -/* - * Query the current RSS key; likely to be used by device drivers when - * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. - * - * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? - */ -void -rss_getkey(uint8_t *key) -{ - - bcopy(rss_key, key, sizeof(rss_key)); -} - -/* - * Query the number of buckets; this may be used by both network device - * drivers, which will need to populate hardware shadows of the software - * indirection table, and the network stack itself (such as when deciding how - * many connection groups to allocate). - */ -u_int -rss_getnumbuckets(void) -{ - - return (rss_buckets); -} - -/* - * Query the number of CPUs in use by RSS; may be useful to device drivers - * trying to figure out how to map a larger number of CPUs into a smaller - * number of receive queues. - */ -u_int -rss_getnumcpus(void) -{ - - return (rss_ncpus); -} - -static inline u_int -rss_gethashconfig_local(void) -{ - - /* Return 4-tuple for TCP; 2-tuple for others */ - /* - * UDP may fragment more often than TCP and thus we'll end up with - * NICs returning 2-tuple fragments. - * udp_init() and udplite_init() both currently initialise things - * as 2-tuple. - * So for now disable UDP 4-tuple hashing until all of the other - * pieces are in place. - */ - return ( - RSS_HASHTYPE_RSS_IPV4 - | RSS_HASHTYPE_RSS_TCP_IPV4 - | RSS_HASHTYPE_RSS_IPV6 - | RSS_HASHTYPE_RSS_TCP_IPV6 - | RSS_HASHTYPE_RSS_IPV6_EX - | RSS_HASHTYPE_RSS_TCP_IPV6_EX -#if 0 - | RSS_HASHTYPE_RSS_UDP_IPV4 - | RSS_HASHTYPE_RSS_UDP_IPV4_EX - | RSS_HASHTYPE_RSS_UDP_IPV6 - | RSS_HASHTYPE_RSS_UDP_IPV6_EX -#endif - ); -} - -/* - * Return the supported RSS hash configuration. - * - * NICs should query this to determine what to configure in their redirection - * matching table. - */ -u_int -rss_gethashconfig(void) -{ - - return (rss_gethashconfig_local()); -} - -/* - * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want - * it appearing in debugging output unnecessarily. - */ -static int -sysctl_rss_key(SYSCTL_HANDLER_ARGS) -{ - uint8_t temp_rss_key[RSS_KEYSIZE]; - int error; - - error = priv_check(req->td, PRIV_NETINET_HASHKEY); - if (error) - return (error); - - bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); - error = sysctl_handle_opaque(oidp, temp_rss_key, - sizeof(temp_rss_key), req); - if (error) - return (error); - if (req->newptr != NULL) { - /* XXXRW: Not yet. */ - return (EINVAL); - } - return (0); -} -SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, - CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, - "", "RSS keying material"); - -static int -sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) -{ - struct sbuf *sb; - int error; - int i; - - error = 0; - error = sysctl_wire_old_buffer(req, 0); - if (error != 0) - return (error); - sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); - if (sb == NULL) - return (ENOMEM); - for (i = 0; i < rss_buckets; i++) { - sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", - i, - rss_getcpu(i)); - } - error = sbuf_finish(sb); - sbuf_delete(sb); - - return (error); -} -SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, - CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, - sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); Index: sys/netinet/ip_input.c =================================================================== --- sys/netinet/ip_input.c +++ sys/netinet/ip_input.c @@ -61,6 +61,7 @@ #include #include #include +#include #include #include Index: sys/netinet/ip_output.c =================================================================== --- sys/netinet/ip_output.c +++ sys/netinet/ip_output.c @@ -65,6 +65,7 @@ #ifdef RADIX_MPATH #include #endif +#include #include #include Index: sys/netinet/tcp_timer.c =================================================================== --- sys/netinet/tcp_timer.c +++ sys/netinet/tcp_timer.c @@ -51,6 +51,7 @@ #include #include +#include #include #include Index: sys/netinet/toeplitz.h =================================================================== --- sys/netinet/toeplitz.h +++ sys/netinet/toeplitz.h @@ -1,40 +0,0 @@ -/*- - * Copyright (c) 2010 David Malone - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _NETINET_TOEPLITZ_H_ -#define _NETINET_TOEPLITZ_H_ - -/* - * Toeplitz (RSS) hash algorithm; possibly we should cache intermediate - * results between runs, in which case we'll need explicit init/destroy and - * state management. - */ -uint32_t toeplitz_hash(u_int keylen, const uint8_t *key, - u_int datalen, const uint8_t *data); - -#endif /* !_NETINET_TOEPLITZ_H_ */ Index: sys/netinet/toeplitz.c =================================================================== --- sys/netinet/toeplitz.c +++ sys/netinet/toeplitz.c @@ -1,58 +0,0 @@ -/*- - * Copyright (c) 2010 David Malone - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include - -#include -#include - -#include - -uint32_t -toeplitz_hash(u_int keylen, const uint8_t *key, u_int datalen, - const uint8_t *data) -{ - uint32_t hash = 0, v; - u_int i, b; - - /* XXXRW: Perhaps an assertion about key length vs. data length? */ - - v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; - for (i = 0; i < datalen; i++) { - for (b = 0; b < 8; b++) { - if (data[i] & (1<<(7-b))) - hash ^= v; - v <<= 1; - if ((i + 4) < RSS_KEYSIZE && - (key[i+4] & (1<<(7-b)))) - v |= 1; - } - } - return (hash); -} Index: sys/netinet/udp_usrreq.c =================================================================== --- sys/netinet/udp_usrreq.c +++ sys/netinet/udp_usrreq.c @@ -70,6 +70,7 @@ #include #include #include +#include #include #include Index: sys/netinet6/in6_pcbgroup.c =================================================================== --- sys/netinet6/in6_pcbgroup.c +++ sys/netinet6/in6_pcbgroup.c @@ -37,11 +37,13 @@ #include #include +#include + #include #include -#include #ifdef INET6 #include +#include #endif /* INET6 */ /* Index: sys/netinet6/in6_rss.h =================================================================== --- sys/netinet6/in6_rss.h +++ sys/netinet6/in6_rss.h @@ -26,117 +26,20 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: head/sys/netinet/in_rss.h 276484 2014-12-31 22:52:43Z adrian $ + * $FreeBSD: 276484 2014-12-31 22:52:43Z adrian $ */ -#ifndef _NETINET_IN_RSS_H_ -#define _NETINET_IN_RSS_H_ +#ifndef _NETINET6_IN6_RSS_H_ +#define _NETINET6_IN6_RSS_H_ #include /* in_addr_t */ /* - * Supported RSS hash functions. - */ -#define RSS_HASH_NAIVE 0x00000001 /* Poor but fast hash. */ -#define RSS_HASH_TOEPLITZ 0x00000002 /* Required by RSS. */ -#define RSS_HASH_CRC32 0x00000004 /* Future; some NICs do it. */ - -#define RSS_HASH_MASK (RSS_HASH_NAIVE | RSS_HASH_TOEPLITZ) - -/* - * Instances of struct inpcbinfo declare an RSS hash type indicating what - * header fields are covered. - */ -#define RSS_HASHFIELDS_NONE 0 -#define RSS_HASHFIELDS_4TUPLE 1 -#define RSS_HASHFIELDS_2TUPLE 2 - -/* - * Define RSS representations of the M_HASHTYPE_* values, representing - * which particular bits are supported. The NICs can then use this to - * calculate which hash types to enable and which not to enable. - * - * The fact that these line up with M_HASHTYPE_* is not to be relied - * upon. - */ -#define RSS_HASHTYPE_RSS_IPV4 (1 << 1) /* IPv4 2-tuple */ -#define RSS_HASHTYPE_RSS_TCP_IPV4 (1 << 2) /* TCPv4 4-tuple */ -#define RSS_HASHTYPE_RSS_IPV6 (1 << 3) /* IPv6 2-tuple */ -#define RSS_HASHTYPE_RSS_TCP_IPV6 (1 << 4) /* TCPv6 4-tuple */ -#define RSS_HASHTYPE_RSS_IPV6_EX (1 << 5) /* IPv6 2-tuple + ext hdrs */ -#define RSS_HASHTYPE_RSS_TCP_IPV6_EX (1 << 6) /* TCPv6 4-tiple + ext hdrs */ -#define RSS_HASHTYPE_RSS_UDP_IPV4 (1 << 7) /* IPv4 UDP 4-tuple */ -#define RSS_HASHTYPE_RSS_UDP_IPV4_EX (1 << 8) /* IPv4 UDP 4-tuple + ext hdrs */ -#define RSS_HASHTYPE_RSS_UDP_IPV6 (1 << 9) /* IPv6 UDP 4-tuple */ -#define RSS_HASHTYPE_RSS_UDP_IPV6_EX (1 << 10) /* IPv6 UDP 4-tuple + ext hdrs */ - -/* - * Compile-time limits on the size of the indirection table. - */ -#define RSS_MAXBITS 7 -#define RSS_TABLE_MAXLEN (1 << RSS_MAXBITS) - -/* - * Maximum key size used throughout. It's OK for hardware to use only the - * first 16 bytes, which is all that's required for IPv4. - */ -#define RSS_KEYSIZE 40 - -/* - * For RSS hash methods that do a software hash on an mbuf, the packet - * direction (ingress / egress) is required. - * - * The default direction (INGRESS) is the "receive into the NIC" - ie, - * what the hardware is hashing on. - */ -#define RSS_HASH_PKT_INGRESS 0 -#define RSS_HASH_PKT_EGRESS 1 - -/* - * Device driver interfaces to query RSS properties that must be programmed - * into hardware. - */ -u_int rss_getbits(void); -u_int rss_getbucket(u_int hash); -u_int rss_get_indirection_to_bucket(u_int index); -u_int rss_getcpu(u_int bucket); -void rss_getkey(uint8_t *key); -u_int rss_gethashalgo(void); -u_int rss_getnumbuckets(void); -u_int rss_getnumcpus(void); -u_int rss_gethashconfig(void); - -/* * Network stack interface to generate a hash for a protocol tuple. */ -uint32_t rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, - struct in_addr dst, u_short dstport); -uint32_t rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst); uint32_t rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport, const struct in6_addr *dst, u_short dstport); uint32_t rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst); -/* - * Network stack interface to query desired CPU affinity of a packet. - */ -struct mbuf *rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid); -u_int rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type); -int rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, - uint32_t *bucket_id); -int rss_m2bucket(struct mbuf *m, uint32_t *bucket_id); - -/* - * Functions to calculate a software RSS hash for a given mbuf or - * packet detail. - */ -int rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, - uint32_t *hashval, uint32_t *hashtype); -int rss_proto_software_hash_v4(struct in_addr src, - struct in_addr dst, u_short src_port, u_short dst_port, - int proto, uint32_t *hashval, - uint32_t *hashtype); -struct mbuf * rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, - u_int *cpuid); - -#endif /* !_NETINET_IN_RSS_H_ */ +#endif /* !_NETINET6_IN6_RSS_H_ */ Index: sys/netinet6/in6_rss.c =================================================================== --- sys/netinet6/in6_rss.c +++ sys/netinet6/in6_rss.c @@ -29,7 +29,7 @@ #include -__FBSDID("$FreeBSD: head/sys/netinet/in_rss.c 276484 2014-12-31 22:52:43Z adrian $"); +__FBSDID("$FreeBSD: 276484 2014-12-31 22:52:43Z adrian $"); #include "opt_inet6.h" #include "opt_pcbgroup.h" @@ -50,298 +50,18 @@ #include #include #include +#include #include #include -#include +#include #include -#include /* for software rss hash support */ #include #include #include -/*- - * Operating system parts of receiver-side scaling (RSS), which allows - * network cards to direct flows to particular receive queues based on hashes - * of header tuples. This implementation aligns RSS buckets with connection - * groups at the TCP/IP layer, so each bucket is associated with exactly one - * group. As a result, the group lookup structures (and lock) should have an - * effective affinity with exactly one CPU. - * - * Network device drivers needing to configure RSS will query this framework - * for parameters, such as the current RSS key, hashing policies, number of - * bits, and indirection table mapping hashes to buckets and CPUs. They may - * provide their own supplementary information, such as queue<->CPU bindings. - * It is the responsibility of the network device driver to inject packets - * into the stack on as close to the right CPU as possible, if playing by RSS - * rules. - * - * TODO: - * - * - Synchronization for rss_key and other future-configurable parameters. - * - Event handler drivers can register to pick up RSS configuration changes. - * - Should we allow rss_basecpu to be configured? - * - Randomize key on boot. - * - IPv6 support. - * - Statistics on how often there's a misalignment between hardware - * placement and pcbgroup expectations. - */ - -SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); - -/* - * Toeplitz is the only required hash function in the RSS spec, so use it by - * default. - */ -static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; -SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, - "RSS hash algorithm"); - -/* - * Size of the indirection table; at most 128 entries per the RSS spec. We - * size it to at least 2 times the number of CPUs by default to allow useful - * rebalancing. If not set explicitly with a loader tunable, we tune based - * on the number of CPUs present. - * - * XXXRW: buckets might be better to use for the tunable than bits. - */ -static u_int rss_bits; -SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, - "RSS bits"); - -static u_int rss_mask; -SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, - "RSS mask"); - -static const u_int rss_maxbits = RSS_MAXBITS; -SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, - __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); - -/* - * RSS's own count of the number of CPUs it could be using for processing. - * Bounded to 64 by RSS constants. - */ -static u_int rss_ncpus; -SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, - "Number of CPUs available to RSS"); - -#define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) -static const u_int rss_maxcpus = RSS_MAXCPUS; -SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, - __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); - -/* - * Variable exists just for reporting rss_bits in a user-friendly way. - */ -static u_int rss_buckets; -SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, - "RSS buckets"); - -/* - * Base CPU number; devices will add this to all CPU numbers returned by the - * RSS indirection table. Currently unmodifable in FreeBSD. - */ -static const u_int rss_basecpu; -SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, - __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); - -/* - * RSS secret key, intended to prevent attacks on load-balancing. Its - * effectiveness may be limited by algorithm choice and available entropy - * during the boot. - * - * XXXRW: And that we don't randomize it yet! - * - * This is the default Microsoft RSS specification key which is also - * the Chelsio T5 firmware default key. - */ -static uint8_t rss_key[RSS_KEYSIZE] = { - 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, - 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, - 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, - 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, - 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, -}; - -/* - * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. - * Drivers may supplement this table with a seperate CPU<->queue table when - * programming devices. - */ -struct rss_table_entry { - uint8_t rte_cpu; /* CPU affinity of bucket. */ -}; -static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; - -static inline u_int rss_gethashconfig_local(void); - -static void -rss_init(__unused void *arg) -{ - u_int i; - u_int cpuid; - - /* - * Validate tunables, coerce to sensible values. - */ - switch (rss_hashalgo) { - case RSS_HASH_TOEPLITZ: - case RSS_HASH_NAIVE: - break; - - default: - printf("%s: invalid RSS hashalgo %u, coercing to %u", - __func__, rss_hashalgo, RSS_HASH_TOEPLITZ); - rss_hashalgo = RSS_HASH_TOEPLITZ; - } - - /* - * Count available CPUs. - * - * XXXRW: Note incorrect assumptions regarding contiguity of this set - * elsewhere. - */ - rss_ncpus = 0; - for (i = 0; i <= mp_maxid; i++) { - if (CPU_ABSENT(i)) - continue; - rss_ncpus++; - } - if (rss_ncpus > RSS_MAXCPUS) - rss_ncpus = RSS_MAXCPUS; - - /* - * Tune RSS table entries to be no less than 2x the number of CPUs - * -- unless we're running uniprocessor, in which case there's not - * much point in having buckets to rearrange for load-balancing! - */ - if (rss_ncpus > 1) { - if (rss_bits == 0) - rss_bits = fls(rss_ncpus - 1) + 1; - - /* - * Microsoft limits RSS table entries to 128, so apply that - * limit to both auto-detected CPU counts and user-configured - * ones. - */ - if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { - printf("%s: RSS bits %u not valid, coercing to %u", - __func__, rss_bits, RSS_MAXBITS); - rss_bits = RSS_MAXBITS; - } - - /* - * Figure out how many buckets to use; warn if less than the - * number of configured CPUs, although this is not a fatal - * problem. - */ - rss_buckets = (1 << rss_bits); - if (rss_buckets < rss_ncpus) - printf("%s: WARNING: rss_buckets (%u) less than " - "rss_ncpus (%u)\n", __func__, rss_buckets, - rss_ncpus); - rss_mask = rss_buckets - 1; - } else { - rss_bits = 0; - rss_buckets = 1; - rss_mask = 0; - } - - /* - * Set up initial CPU assignments: round-robin by default. - */ - cpuid = CPU_FIRST(); - for (i = 0; i < rss_buckets; i++) { - rss_table[i].rte_cpu = cpuid; - cpuid = CPU_NEXT(cpuid); - } - - /* - * Randomize rrs_key. - * - * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() - * loop to check for "bad" RSS keys. - */ -} -SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); - -static uint32_t -rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, - const uint8_t *data) -{ - uint32_t v; - u_int i; - - v = 0; - for (i = 0; i < keylen; i++) - v += key[i]; - for (i = 0; i < datalen; i++) - v += data[i]; - return (v); -} - -static uint32_t -rss_hash(u_int datalen, const uint8_t *data) -{ - - switch (rss_hashalgo) { - case RSS_HASH_TOEPLITZ: - return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, - data)); - - case RSS_HASH_NAIVE: - return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, - data)); - - default: - panic("%s: unsupported/unknown hashalgo %d", __func__, - rss_hashalgo); - } -} - -/* - * Hash an IPv4 2-tuple. - */ -uint32_t -rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst) -{ - uint8_t data[sizeof(src) + sizeof(dst)]; - u_int datalen; - - datalen = 0; - bcopy(&src, &data[datalen], sizeof(src)); - datalen += sizeof(src); - bcopy(&dst, &data[datalen], sizeof(dst)); - datalen += sizeof(dst); - return (rss_hash(datalen, data)); -} - -/* - * Hash an IPv4 4-tuple. - */ -uint32_t -rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst, - u_short dstport) -{ - uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) + - sizeof(dstport)]; - u_int datalen; - - datalen = 0; - bcopy(&src, &data[datalen], sizeof(src)); - datalen += sizeof(src); - bcopy(&dst, &data[datalen], sizeof(dst)); - datalen += sizeof(dst); - bcopy(&srcport, &data[datalen], sizeof(srcport)); - datalen += sizeof(srcport); - bcopy(&dstport, &data[datalen], sizeof(dstport)); - datalen += sizeof(dstport); - return (rss_hash(datalen, data)); -} - -#ifdef INET6 /* * Hash an IPv6 2-tuple. */ @@ -381,519 +101,3 @@ datalen += sizeof(dstport); return (rss_hash(datalen, data)); } -#endif /* INET6 */ - -/* - * Query the number of RSS bits in use. - */ -u_int -rss_getbits(void) -{ - - return (rss_bits); -} - -/* - * Query the RSS bucket associated with an RSS hash. - */ -u_int -rss_getbucket(u_int hash) -{ - - return (hash & rss_mask); -} - -/* - * Query the RSS layer bucket associated with the given - * entry in the RSS hash space. - * - * The RSS indirection table is 0 .. rss_buckets-1, - * covering the low 'rss_bits' of the total 128 slot - * RSS indirection table. So just mask off rss_bits and - * return that. - * - * NIC drivers can then iterate over the 128 slot RSS - * indirection table and fetch which RSS bucket to - * map it to. This will typically be a CPU queue - */ -u_int -rss_get_indirection_to_bucket(u_int index) -{ - - return (index & rss_mask); -} - -/* - * Query the RSS CPU associated with an RSS bucket. - */ -u_int -rss_getcpu(u_int bucket) -{ - - return (rss_table[bucket].rte_cpu); -} - -/* - * netisr CPU affinity lookup given just the hash and hashtype. - */ -u_int -rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) -{ - - switch (hash_type) { - case M_HASHTYPE_RSS_IPV4: - case M_HASHTYPE_RSS_TCP_IPV4: - case M_HASHTYPE_RSS_UDP_IPV4: - case M_HASHTYPE_RSS_IPV6: - case M_HASHTYPE_RSS_TCP_IPV6: - case M_HASHTYPE_RSS_UDP_IPV6: - return (rss_getcpu(rss_getbucket(hash_val))); - default: - return (NETISR_CPUID_NONE); - } -} - -/* - * Query the RSS bucket associated with the given hash value and - * type. - */ -int -rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) -{ - - switch (hash_type) { - case M_HASHTYPE_RSS_IPV4: - case M_HASHTYPE_RSS_TCP_IPV4: - case M_HASHTYPE_RSS_UDP_IPV4: - case M_HASHTYPE_RSS_IPV6: - case M_HASHTYPE_RSS_TCP_IPV6: - case M_HASHTYPE_RSS_UDP_IPV6: - *bucket_id = rss_getbucket(hash_val); - return (0); - default: - return (-1); - } -} - -/* - * netisr CPU affinity lookup routine for use by protocols. - */ -struct mbuf * -rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) -{ - - M_ASSERTPKTHDR(m); - *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); - return (m); -} - -int -rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) -{ - - M_ASSERTPKTHDR(m); - - return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), - bucket_id)); -} - -/* - * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given - * IPv4 source/destination address, UDP or TCP source/destination ports - * and the protocol type. - * - * The protocol code may wish to do a software hash of the given - * tuple. This depends upon the currently configured RSS hash types. - * - * This assumes that the packet in question isn't a fragment. - * - * It also assumes the packet source/destination address - * are in "incoming" packet order (ie, source is "far" address.) - */ -int -rss_proto_software_hash_v4(struct in_addr s, struct in_addr d, - u_short sp, u_short dp, int proto, - uint32_t *hashval, uint32_t *hashtype) -{ - uint32_t hash; - - /* - * Next, choose the hash type depending upon the protocol - * identifier. - */ - if ((proto == IPPROTO_TCP) && - (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4)) { - hash = rss_hash_ip4_4tuple(s, sp, d, dp); - *hashval = hash; - *hashtype = M_HASHTYPE_RSS_TCP_IPV4; - return (0); - } else if ((proto == IPPROTO_UDP) && - (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4)) { - hash = rss_hash_ip4_4tuple(s, sp, d, dp); - *hashval = hash; - *hashtype = M_HASHTYPE_RSS_UDP_IPV4; - return (0); - } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) { - /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */ - hash = rss_hash_ip4_2tuple(s, d); - *hashval = hash; - *hashtype = M_HASHTYPE_RSS_IPV4; - return (0); - } - - /* No configured available hashtypes! */ - printf("%s: no available hashtypes!\n", __func__); - return (-1); -} - -/* - * Do a software calculation of the RSS for the given mbuf. - * - * This is typically used by the input path to recalculate the RSS after - * some form of packet processing (eg de-capsulation, IP fragment reassembly.) - * - * dir is the packet direction - RSS_HASH_PKT_INGRESS for incoming and - * RSS_HASH_PKT_EGRESS for outgoing. - * - * Returns 0 if a hash was done, -1 if no hash was done, +1 if - * the mbuf already had a valid RSS flowid. - * - * This function doesn't modify the mbuf. It's up to the caller to - * assign flowid/flowtype as appropriate. - */ -int -rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval, - uint32_t *hashtype) -{ - const struct ip *ip; - const struct tcphdr *th; - const struct udphdr *uh; - uint32_t flowid; - uint32_t flowtype; - uint8_t proto; - int iphlen; - int is_frag = 0; - - /* - * XXX For now this only handles hashing on incoming mbufs. - */ - if (dir != RSS_HASH_PKT_INGRESS) { - printf("%s: called on EGRESS packet!\n", __func__); - return (-1); - } - - /* - * First, validate that the mbuf we have is long enough - * to have an IPv4 header in it. - */ - if (m->m_pkthdr.len < (sizeof(struct ip))) { - printf("%s: short mbuf pkthdr\n", __func__); - return (-1); - } - if (m->m_len < (sizeof(struct ip))) { - printf("%s: short mbuf len\n", __func__); - return (-1); - } - - /* Ok, let's dereference that */ - ip = mtod(m, struct ip *); - proto = ip->ip_p; - iphlen = ip->ip_hl << 2; - - /* - * If this is a fragment then it shouldn't be four-tuple - * hashed just yet. Once it's reassembled into a full - * frame it should be re-hashed. - */ - if (ip->ip_off & htons(IP_MF | IP_OFFMASK)) - is_frag = 1; - - /* - * If the mbuf flowid/flowtype matches the packet type, - * and we don't support the 4-tuple version of the given protocol, - * then signal to the owner that it can trust the flowid/flowtype - * details. - * - * This is a little picky - eg, if TCPv4 / UDPv4 hashing - * is supported but we got a TCP/UDP frame only 2-tuple hashed, - * then we shouldn't just "trust" the 2-tuple hash. We need - * a 4-tuple hash. - */ - flowid = m->m_pkthdr.flowid; - flowtype = M_HASHTYPE_GET(m); - - if (flowtype != M_HASHTYPE_NONE) { - switch (proto) { - case IPPROTO_UDP: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) && - (flowtype == M_HASHTYPE_RSS_UDP_IPV4) && - (is_frag == 0)) { - return (1); - } - /* - * Only allow 2-tuple for UDP frames if we don't also - * support 4-tuple for UDP. - */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) && - flowtype == M_HASHTYPE_RSS_IPV4) { - return (1); - } - break; - case IPPROTO_TCP: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) && - (flowtype == M_HASHTYPE_RSS_TCP_IPV4) && - (is_frag == 0)) { - return (1); - } - /* - * Only allow 2-tuple for TCP frames if we don't also - * support 2-tuple for TCP. - */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) && - flowtype == M_HASHTYPE_RSS_IPV4) { - return (1); - } - break; - default: - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) && - flowtype == M_HASHTYPE_RSS_IPV4) { - return (1); - } - break; - } - } - - /* - * Decode enough information to make a hash decision. - * - * XXX TODO: does the hardware hash on 4-tuple if IP - * options are present? - */ - if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) && - (proto == IPPROTO_TCP) && - (is_frag == 0)) { - if (m->m_len < iphlen + sizeof(struct tcphdr)) { - printf("%s: short TCP frame?\n", __func__); - return (-1); - } - th = (struct tcphdr *)((caddr_t)ip + iphlen); - return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst, - th->th_sport, - th->th_dport, - proto, - hashval, - hashtype); - } else if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) && - (proto == IPPROTO_UDP) && - (is_frag == 0)) { - uh = (struct udphdr *)((caddr_t)ip + iphlen); - if (m->m_len < iphlen + sizeof(struct udphdr)) { - printf("%s: short UDP frame?\n", __func__); - return (-1); - } - return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst, - uh->uh_sport, - uh->uh_dport, - proto, - hashval, - hashtype); - } else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) { - /* Default to 2-tuple hash */ - return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst, - 0, /* source port */ - 0, /* destination port */ - 0, /* IPPROTO_IP */ - hashval, - hashtype); - } else { - printf("%s: no available hashtypes!\n", __func__); - return (-1); - } -} - -/* - * Similar to rss_m2cpuid, but designed to be used by the IP NETISR - * on incoming frames. - * - * If an existing RSS hash exists and it matches what the configured - * hashing is, then use it. - * - * If there's an existing RSS hash but the desired hash is different, - * or if there's no useful RSS hash, then calculate it via - * the software path. - * - * XXX TODO: definitely want statistics here! - */ -struct mbuf * -rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) -{ - uint32_t hash_val, hash_type; - int ret; - - M_ASSERTPKTHDR(m); - - ret = rss_mbuf_software_hash_v4(m, RSS_HASH_PKT_INGRESS, - &hash_val, &hash_type); - if (ret > 0) { - /* mbuf has a valid hash already; don't need to modify it */ - *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); - } else if (ret == 0) { - /* hash was done; update */ - m->m_pkthdr.flowid = hash_val; - M_HASHTYPE_SET(m, hash_type); - *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); - } else { /* ret < 0 */ - /* no hash was done */ - *cpuid = NETISR_CPUID_NONE; - } - return (m); -} - -/* - * Query the RSS hash algorithm. - */ -u_int -rss_gethashalgo(void) -{ - - return (rss_hashalgo); -} - -/* - * Query the current RSS key; likely to be used by device drivers when - * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. - * - * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? - */ -void -rss_getkey(uint8_t *key) -{ - - bcopy(rss_key, key, sizeof(rss_key)); -} - -/* - * Query the number of buckets; this may be used by both network device - * drivers, which will need to populate hardware shadows of the software - * indirection table, and the network stack itself (such as when deciding how - * many connection groups to allocate). - */ -u_int -rss_getnumbuckets(void) -{ - - return (rss_buckets); -} - -/* - * Query the number of CPUs in use by RSS; may be useful to device drivers - * trying to figure out how to map a larger number of CPUs into a smaller - * number of receive queues. - */ -u_int -rss_getnumcpus(void) -{ - - return (rss_ncpus); -} - -static inline u_int -rss_gethashconfig_local(void) -{ - - /* Return 4-tuple for TCP; 2-tuple for others */ - /* - * UDP may fragment more often than TCP and thus we'll end up with - * NICs returning 2-tuple fragments. - * udp_init() and udplite_init() both currently initialise things - * as 2-tuple. - * So for now disable UDP 4-tuple hashing until all of the other - * pieces are in place. - */ - return ( - RSS_HASHTYPE_RSS_IPV4 - | RSS_HASHTYPE_RSS_TCP_IPV4 - | RSS_HASHTYPE_RSS_IPV6 - | RSS_HASHTYPE_RSS_TCP_IPV6 - | RSS_HASHTYPE_RSS_IPV6_EX - | RSS_HASHTYPE_RSS_TCP_IPV6_EX -#if 0 - | RSS_HASHTYPE_RSS_UDP_IPV4 - | RSS_HASHTYPE_RSS_UDP_IPV4_EX - | RSS_HASHTYPE_RSS_UDP_IPV6 - | RSS_HASHTYPE_RSS_UDP_IPV6_EX -#endif - ); -} - -/* - * Return the supported RSS hash configuration. - * - * NICs should query this to determine what to configure in their redirection - * matching table. - */ -u_int -rss_gethashconfig(void) -{ - - return (rss_gethashconfig_local()); -} - -/* - * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want - * it appearing in debugging output unnecessarily. - */ -static int -sysctl_rss_key(SYSCTL_HANDLER_ARGS) -{ - uint8_t temp_rss_key[RSS_KEYSIZE]; - int error; - - error = priv_check(req->td, PRIV_NETINET_HASHKEY); - if (error) - return (error); - - bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); - error = sysctl_handle_opaque(oidp, temp_rss_key, - sizeof(temp_rss_key), req); - if (error) - return (error); - if (req->newptr != NULL) { - /* XXXRW: Not yet. */ - return (EINVAL); - } - return (0); -} -SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, - CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, - "", "RSS keying material"); - -static int -sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) -{ - struct sbuf *sb; - int error; - int i; - - error = 0; - error = sysctl_wire_old_buffer(req, 0); - if (error != 0) - return (error); - sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); - if (sb == NULL) - return (ENOMEM); - for (i = 0; i < rss_buckets; i++) { - sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", - i, - rss_getcpu(i)); - } - error = sbuf_finish(sb); - sbuf_delete(sb); - - return (error); -} -SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, - CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, - sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); Index: sys/netinet6/ip6_output.c =================================================================== --- sys/netinet6/ip6_output.c +++ sys/netinet6/ip6_output.c @@ -91,6 +91,7 @@ #include #include #include +#include #include #include @@ -103,7 +104,7 @@ #include #include #include -#include +#include #ifdef IPSEC #include Index: sys/netinet6/udp6_usrreq.c =================================================================== --- sys/netinet6/udp6_usrreq.c +++ sys/netinet6/udp6_usrreq.c @@ -97,6 +97,7 @@ #include #include #include +#include #include #include @@ -112,11 +113,11 @@ #include #include #include -#include #include #include #include +#include #include #include