Index: sys/netinet/in_rss.h =================================================================== --- sys/netinet/in_rss.h +++ sys/netinet/in_rss.h @@ -51,7 +51,7 @@ struct in_addr dst, u_short src_port, u_short dst_port, int proto, uint32_t *hashval, uint32_t *hashtype); -struct mbuf * rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, +struct mbuf * rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source, u_int *cpuid); #endif /* !_NETINET_IN_RSS_H_ */ Index: sys/netinet/in_rss.c =================================================================== --- sys/netinet/in_rss.c +++ sys/netinet/in_rss.c @@ -332,7 +332,7 @@ * XXX TODO: definitely want statistics here! */ struct mbuf * -rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) +rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source, u_int *cpuid) { uint32_t hash_val, hash_type; int ret; Index: sys/netinet/ip_input.c =================================================================== --- sys/netinet/ip_input.c +++ sys/netinet/ip_input.c @@ -140,7 +140,7 @@ .nh_handler = ip_input, .nh_proto = NETISR_IP, #ifdef RSS - .nh_m2cpuid = rss_soft_m2cpuid, + .nh_m2cpuid = rss_soft_m2cpuid_v4, .nh_policy = NETISR_POLICY_CPU, .nh_dispatch = NETISR_DISPATCH_HYBRID, #else Index: sys/netinet6/in6_rss.h =================================================================== --- sys/netinet6/in6_rss.h +++ sys/netinet6/in6_rss.h @@ -46,8 +46,13 @@ * Functions to calculate a software RSS hash for a given mbuf or * packet detail. */ +int rss_mbuf_software_hash_v6(const struct mbuf *m, int dir, + uint32_t *hashval, uint32_t *hashtype); int rss_proto_software_hash_v6(const struct in6_addr *src, const struct in6_addr *dst, u_short src_port, u_short dst_port, int proto, uint32_t *hashval, uint32_t *hashtype); +struct mbuf * rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source, + u_int *cpuid); + #endif /* !_NETINET6_IN6_RSS_H_ */ Index: sys/netinet6/in6_rss.c =================================================================== --- sys/netinet6/in6_rss.c +++ sys/netinet6/in6_rss.c @@ -58,7 +58,8 @@ #include /* for software rss hash support */ -#include +#include +#include #include #include @@ -150,3 +151,207 @@ RSS_DEBUG("no available hashtypes!\n"); return (-1); } + +/* + * Do a software calculation of the RSS for the given mbuf. + * + * This is typically used by the input path to recalculate the RSS after + * some form of packet processing (eg de-capsulation, IP fragment reassembly.) + * + * dir is the packet direction - RSS_HASH_PKT_INGRESS for incoming and + * RSS_HASH_PKT_EGRESS for outgoing. + * + * Returns 0 if a hash was done, -1 if no hash was done, +1 if + * the mbuf already had a valid RSS flowid. + * + * This function doesn't modify the mbuf. It's up to the caller to + * assign flowid/flowtype as appropriate. + */ +int +rss_mbuf_software_hash_v6(const struct mbuf *m, int dir, uint32_t *hashval, + uint32_t *hashtype) +{ + const struct ip6_hdr *ip6; + const struct tcphdr *th; + const struct udphdr *uh; + uint32_t flowtype; + uint8_t proto; + int off, newoff; + int nxt; + + /* + * XXX For now this only handles hashing on incoming mbufs. + */ + if (dir != RSS_HASH_PKT_INGRESS) { + RSS_DEBUG("called on EGRESS packet!\n"); + return (-1); + } + + off = sizeof(struct ip6_hdr); + + /* + * First, validate that the mbuf we have is long enough + * to have an IPv6 header in it. + */ + if (m->m_pkthdr.len < off) { + RSS_DEBUG("short mbuf pkthdr\n"); + return (-1); + } + if (m->m_len < off) { + RSS_DEBUG("short mbuf len\n"); + return (-1); + } + + /* Ok, let's dereference that */ + ip6 = mtod(m, struct ip6_hdr *); + proto = ip6->ip6_nxt; + + /* + * Find the beginning of the TCP/UDP header. + * + * If this is a fragment then it shouldn't be four-tuple + * hashed just yet. Once it's reassembled into a full + * frame it should be re-hashed. + */ + while (proto != IPPROTO_FRAGMENT) { + newoff = ip6_nexthdr(m, off, proto, &nxt); + if (newoff < 0) + break; + off = newoff; + proto = nxt; + } + + /* + * If the mbuf flowid/flowtype matches the packet type, + * and we don't support the 4-tuple version of the given protocol, + * then signal to the owner that it can trust the flowid/flowtype + * details. + * + * This is a little picky - eg, if TCPv6 / UDPv6 hashing + * is supported but we got a TCP/UDP frame only 2-tuple hashed, + * then we shouldn't just "trust" the 2-tuple hash. We need + * a 4-tuple hash. + */ + flowtype = M_HASHTYPE_GET(m); + + if (flowtype != M_HASHTYPE_NONE) { + switch (proto) { + case IPPROTO_UDP: + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6) && + (flowtype == M_HASHTYPE_RSS_UDP_IPV6)) { + return (1); + } + /* + * Only allow 2-tuple for UDP frames if we don't also + * support 4-tuple for UDP. + */ + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) && + ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6) == 0) && + flowtype == M_HASHTYPE_RSS_IPV6) { + return (1); + } + break; + case IPPROTO_TCP: + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6) && + (flowtype == M_HASHTYPE_RSS_TCP_IPV6)) { + return (1); + } + /* + * Only allow 2-tuple for TCP frames if we don't also + * support 4-tuple for TCP. + */ + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) && + ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6) == 0) && + flowtype == M_HASHTYPE_RSS_IPV6) { + return (1); + } + break; + default: + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) && + flowtype == M_HASHTYPE_RSS_IPV6) { + return (1); + } + break; + } + } + + /* + * Decode enough information to make a hash decision. + */ + if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6) && + (proto == IPPROTO_TCP)) { + if (m->m_len < off + sizeof(struct tcphdr)) { + RSS_DEBUG("short TCP frame?\n"); + return (-1); + } + th = (const struct tcphdr *)((c_caddr_t)ip6 + off); + return rss_proto_software_hash_v6(&ip6->ip6_src, &ip6->ip6_dst, + th->th_sport, + th->th_dport, + proto, + hashval, + hashtype); + } else if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6) && + (proto == IPPROTO_UDP)) { + if (m->m_len < off + sizeof(struct udphdr)) { + RSS_DEBUG("short UDP frame?\n"); + return (-1); + } + uh = (const struct udphdr *)((c_caddr_t)ip6 + off); + return rss_proto_software_hash_v6(&ip6->ip6_src, &ip6->ip6_dst, + uh->uh_sport, + uh->uh_dport, + proto, + hashval, + hashtype); + } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV6) { + /* Default to 2-tuple hash */ + return rss_proto_software_hash_v6(&ip6->ip6_src, &ip6->ip6_dst, + 0, /* source port */ + 0, /* destination port */ + 0, /* IPPROTO_IP */ + hashval, + hashtype); + } else { + RSS_DEBUG("no available hashtypes!\n"); + return (-1); + } +} + +/* + * Similar to rss_m2cpuid, but designed to be used by the IPv6 NETISR + * on incoming frames. + * + * If an existing RSS hash exists and it matches what the configured + * hashing is, then use it. + * + * If there's an existing RSS hash but the desired hash is different, + * or if there's no useful RSS hash, then calculate it via + * the software path. + * + * XXX TODO: definitely want statistics here! + */ +struct mbuf * +rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source, u_int *cpuid) +{ + uint32_t hash_val, hash_type; + int ret; + + M_ASSERTPKTHDR(m); + + ret = rss_mbuf_software_hash_v6(m, RSS_HASH_PKT_INGRESS, + &hash_val, &hash_type); + if (ret > 0) { + /* mbuf has a valid hash already; don't need to modify it */ + *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); + } else if (ret == 0) { + /* hash was done; update */ + m->m_pkthdr.flowid = hash_val; + M_HASHTYPE_SET(m, hash_type); + *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); + } else { /* ret < 0 */ + /* no hash was done */ + *cpuid = NETISR_CPUID_NONE; + } + return (m); +} Index: sys/netinet6/ip6_input.c =================================================================== --- sys/netinet6/ip6_input.c +++ sys/netinet6/ip6_input.c @@ -68,6 +68,7 @@ #include "opt_ipfw.h" #include "opt_ipsec.h" #include "opt_route.h" +#include "opt_rss.h" #include #include @@ -112,6 +113,7 @@ #include #include #include +#include #ifdef IPSEC #include @@ -132,7 +134,13 @@ .nh_name = "ip6", .nh_handler = ip6_input, .nh_proto = NETISR_IPV6, +#ifdef RSS + .nh_m2cpuid = rss_soft_m2cpuid_v6, + .nh_policy = NETISR_POLICY_CPU, + .nh_dispatch = NETISR_DISPATCH_HYBRID, +#else .nh_policy = NETISR_POLICY_FLOW, +#endif }; VNET_DECLARE(struct callout, in6_tmpaddrtimer_ch); @@ -1440,7 +1448,7 @@ * we develop `neater' mechanism to process extension headers. */ char * -ip6_get_prevhdr(struct mbuf *m, int off) +ip6_get_prevhdr(const struct mbuf *m, int off) { struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); @@ -1479,7 +1487,7 @@ * get next header offset. m will be retained. */ int -ip6_nexthdr(struct mbuf *m, int off, int proto, int *nxtp) +ip6_nexthdr(const struct mbuf *m, int off, int proto, int *nxtp) { struct ip6_hdr ip6; struct ip6_ext ip6e; @@ -1547,14 +1555,14 @@ return -1; } - return -1; + /* NOTREACHED */ } /* * get offset for the last header in the chain. m will be kept untainted. */ int -ip6_lasthdr(struct mbuf *m, int off, int proto, int *nxtp) +ip6_lasthdr(const struct mbuf *m, int off, int proto, int *nxtp) { int newoff; int nxt; Index: sys/netinet6/ip6_var.h =================================================================== --- sys/netinet6/ip6_var.h +++ sys/netinet6/ip6_var.h @@ -356,9 +356,9 @@ void ip6_freepcbopts(struct ip6_pktopts *); int ip6_unknown_opt(u_int8_t *, struct mbuf *, int); -char * ip6_get_prevhdr(struct mbuf *, int); -int ip6_nexthdr(struct mbuf *, int, int, int *); -int ip6_lasthdr(struct mbuf *, int, int, int *); +char * ip6_get_prevhdr(const struct mbuf *, int); +int ip6_nexthdr(const struct mbuf *, int, int, int *); +int ip6_lasthdr(const struct mbuf *, int, int, int *); extern int (*ip6_mforward)(struct ip6_hdr *, struct ifnet *, struct mbuf *);