Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3102,6 +3102,7 @@ kern/uipc_domain.c standard kern/uipc_mbuf.c standard kern/uipc_mbuf2.c standard +kern/uipc_mbufhash.c standard kern/uipc_mqueue.c optional p1003_1b_mqueue kern/uipc_sem.c optional p1003_1b_semaphores kern/uipc_shm.c standard Index: sys/kern/uipc_mbufhash.c =================================================================== --- sys/kern/uipc_mbufhash.c +++ sys/kern/uipc_mbufhash.c @@ -25,57 +25,32 @@ #include #include -#include #include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include -#include + +#include #if defined(INET) || defined(INET6) #include #endif + #ifdef INET -#include -#include #include #endif #ifdef INET6 #include -#include -#include #endif #include -#include "utils.h" - -/* XXX this code should be factored out */ -/* XXX copied from if_lagg.c */ - static const void * -mlx4_en_gethdr(struct mbuf *m, u_int off, u_int len, void *buf) +m_ether_tcpip_hash_gethdr(const struct mbuf *m, const u_int off, + const u_int len, void *buf) { if (m->m_pkthdr.len < (off + len)) { return (NULL); @@ -87,7 +62,17 @@ } uint32_t -mlx4_en_hashmbuf(uint32_t flags, struct mbuf *m, uint32_t key) +m_ether_tcpip_hash_init(void) +{ + uint32_t seed; + + seed = arc4random(); + return (fnv_32_buf(&seed, sizeof(seed), FNV1_32_INIT)); +} + +uint32_t +m_ether_tcpip_hash(const uint32_t flags, const struct mbuf *m, + const uint32_t key) { uint16_t etype; uint32_t p = key; @@ -96,12 +81,9 @@ const struct ether_vlan_header *vlan; #ifdef INET const struct ip *ip; - const uint32_t *ports; - int iphlen; #endif #ifdef INET6 const struct ip6_hdr *ip6; - uint32_t flow; #endif union { #ifdef INET @@ -112,48 +94,47 @@ #endif struct ether_vlan_header vlan; uint32_t port; - } buf; + } buf; off = sizeof(*eh); if (m->m_len < off) - goto out; + goto done; eh = mtod(m, struct ether_header *); etype = ntohs(eh->ether_type); - if (flags & MLX4_F_HASHL2) { - p = hash32_buf(&eh->ether_shost, ETHER_ADDR_LEN, p); - p = hash32_buf(&eh->ether_dhost, ETHER_ADDR_LEN, p); + if (flags & MBUF_HASHFLAG_L2) { + p = fnv_32_buf(&eh->ether_shost, ETHER_ADDR_LEN, p); + p = fnv_32_buf(&eh->ether_dhost, ETHER_ADDR_LEN, p); } - /* Special handling for encapsulating VLAN frames */ - if ((m->m_flags & M_VLANTAG) && (flags & MLX4_F_HASHL2)) { - p = hash32_buf(&m->m_pkthdr.ether_vtag, + if ((m->m_flags & M_VLANTAG) && (flags & MBUF_HASHFLAG_L2)) { + p = fnv_32_buf(&m->m_pkthdr.ether_vtag, sizeof(m->m_pkthdr.ether_vtag), p); } else if (etype == ETHERTYPE_VLAN) { - vlan = mlx4_en_gethdr(m, off, sizeof(*vlan), &buf); + vlan = m_ether_tcpip_hash_gethdr(m, off, sizeof(*vlan), &buf); if (vlan == NULL) - goto out; + goto done; - if (flags & MLX4_F_HASHL2) - p = hash32_buf(&vlan->evl_tag, sizeof(vlan->evl_tag), p); + if (flags & MBUF_HASHFLAG_L2) + p = fnv_32_buf(&vlan->evl_tag, sizeof(vlan->evl_tag), p); etype = ntohs(vlan->evl_proto); off += sizeof(*vlan) - sizeof(*eh); } - switch (etype) { #ifdef INET case ETHERTYPE_IP: - ip = mlx4_en_gethdr(m, off, sizeof(*ip), &buf); + ip = m_ether_tcpip_hash_gethdr(m, off, sizeof(*ip), &buf); if (ip == NULL) - goto out; - - if (flags & MLX4_F_HASHL3) { - p = hash32_buf(&ip->ip_src, sizeof(struct in_addr), p); - p = hash32_buf(&ip->ip_dst, sizeof(struct in_addr), p); - } - if (!(flags & MLX4_F_HASHL4)) break; - switch (ip->ip_p) { + if (flags & MBUF_HASHFLAG_L3) { + p = fnv_32_buf(&ip->ip_src, sizeof(struct in_addr), p); + p = fnv_32_buf(&ip->ip_dst, sizeof(struct in_addr), p); + } + if (flags & MBUF_HASHFLAG_L4) { + const uint32_t *ports; + int iphlen; + + switch (ip->ip_p) { case IPPROTO_TCP: case IPPROTO_UDP: case IPPROTO_SCTP: @@ -161,29 +142,37 @@ if (iphlen < sizeof(*ip)) break; off += iphlen; - ports = mlx4_en_gethdr(m, off, sizeof(*ports), &buf); + ports = m_ether_tcpip_hash_gethdr(m, + off, sizeof(*ports), &buf); if (ports == NULL) break; - p = hash32_buf(ports, sizeof(*ports), p); + p = fnv_32_buf(ports, sizeof(*ports), p); break; + } } break; #endif #ifdef INET6 case ETHERTYPE_IPV6: - if (!(flags & MLX4_F_HASHL3)) - break; - ip6 = mlx4_en_gethdr(m, off, sizeof(*ip6), &buf); + ip6 = m_ether_tcpip_hash_gethdr(m, off, sizeof(*ip6), &buf); if (ip6 == NULL) - goto out; + break; + if (flags & MBUF_HASHFLAG_L3) { + p = fnv_32_buf(&ip6->ip6_src, sizeof(struct in6_addr), p); + p = fnv_32_buf(&ip6->ip6_dst, sizeof(struct in6_addr), p); + } + if (flags & MBUF_HASHFLAG_L4) { + uint32_t flow; - p = hash32_buf(&ip6->ip6_src, sizeof(struct in6_addr), p); - p = hash32_buf(&ip6->ip6_dst, sizeof(struct in6_addr), p); - flow = ip6->ip6_flow & IPV6_FLOWLABEL_MASK; - p = hash32_buf(&flow, sizeof(flow), p); /* IPv6 flow label */ + /* IPv6 flow label */ + flow = ip6->ip6_flow & IPV6_FLOWLABEL_MASK; + p = fnv_32_buf(&flow, sizeof(flow), p); + } break; #endif + default: + break; } -out: +done: return (p); } Index: sys/net/if_lagg.c =================================================================== --- sys/net/if_lagg.c +++ sys/net/if_lagg.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include @@ -131,7 +130,6 @@ static void lagg_media_status(struct ifnet *, struct ifmediareq *); static struct lagg_port *lagg_link_active(struct lagg_softc *, struct lagg_port *); -static const void *lagg_gethdr(struct mbuf *, u_int, u_int, void *); /* Simple round robin */ static void lagg_rr_attach(struct lagg_softc *); @@ -490,7 +488,7 @@ sc->flowid_shift = V_def_flowid_shift; /* Hash all layers by default */ - sc->sc_flags = LAGG_F_HASHL2|LAGG_F_HASHL3|LAGG_F_HASHL4; + sc->sc_flags = MBUF_HASHFLAG_L2|MBUF_HASHFLAG_L3|MBUF_HASHFLAG_L4; lagg_proto_attach(sc, LAGG_PROTO_DEFAULT); @@ -1349,7 +1347,13 @@ LAGG_WUNLOCK(sc); break; case SIOCGLAGGFLAGS: - rf->rf_flags = sc->sc_flags; + rf->rf_flags = 0; + if (sc->sc_flags & MBUF_HASHFLAG_L2) + rf->rf_flags |= LAGG_F_HASHL2; + if (sc->sc_flags & MBUF_HASHFLAG_L3) + rf->rf_flags |= LAGG_F_HASHL3; + if (sc->sc_flags & MBUF_HASHFLAG_L4) + rf->rf_flags |= LAGG_F_HASHL4; break; case SIOCSLAGGHASH: error = priv_check(td, PRIV_NET_LAGG); @@ -1360,8 +1364,13 @@ break; } LAGG_WLOCK(sc); - sc->sc_flags &= ~LAGG_F_HASHMASK; - sc->sc_flags |= rf->rf_flags & LAGG_F_HASHMASK; + sc->sc_flags &= ~MBUF_HASHFLAG_MASK; + if (rf->rf_flags & LAGG_F_HASHL2) + sc->sc_flags |= MBUF_HASHFLAG_L2; + if (rf->rf_flags & LAGG_F_HASHL3) + sc->sc_flags |= MBUF_HASHFLAG_L3; + if (rf->rf_flags & LAGG_F_HASHL4) + sc->sc_flags |= MBUF_HASHFLAG_L4; LAGG_WUNLOCK(sc); break; case SIOCGLAGGPORT: @@ -1806,120 +1815,6 @@ return (rval); } -static const void * -lagg_gethdr(struct mbuf *m, u_int off, u_int len, void *buf) -{ - if (m->m_pkthdr.len < (off + len)) { - return (NULL); - } else if (m->m_len < (off + len)) { - m_copydata(m, off, len, buf); - return (buf); - } - return (mtod(m, char *) + off); -} - -uint32_t -lagg_hashmbuf(struct lagg_softc *sc, struct mbuf *m, uint32_t key) -{ - uint16_t etype; - uint32_t p = key; - int off; - struct ether_header *eh; - const struct ether_vlan_header *vlan; -#ifdef INET - const struct ip *ip; - const uint32_t *ports; - int iphlen; -#endif -#ifdef INET6 - const struct ip6_hdr *ip6; - uint32_t flow; -#endif - union { -#ifdef INET - struct ip ip; -#endif -#ifdef INET6 - struct ip6_hdr ip6; -#endif - struct ether_vlan_header vlan; - uint32_t port; - } buf; - - - off = sizeof(*eh); - if (m->m_len < off) - goto out; - eh = mtod(m, struct ether_header *); - etype = ntohs(eh->ether_type); - if (sc->sc_flags & LAGG_F_HASHL2) { - p = fnv_32_buf(&eh->ether_shost, ETHER_ADDR_LEN, p); - p = fnv_32_buf(&eh->ether_dhost, ETHER_ADDR_LEN, p); - } - - /* Special handling for encapsulating VLAN frames */ - if ((m->m_flags & M_VLANTAG) && (sc->sc_flags & LAGG_F_HASHL2)) { - p = fnv_32_buf(&m->m_pkthdr.ether_vtag, - sizeof(m->m_pkthdr.ether_vtag), p); - } else if (etype == ETHERTYPE_VLAN) { - vlan = lagg_gethdr(m, off, sizeof(*vlan), &buf); - if (vlan == NULL) - goto out; - - if (sc->sc_flags & LAGG_F_HASHL2) - p = fnv_32_buf(&vlan->evl_tag, sizeof(vlan->evl_tag), p); - etype = ntohs(vlan->evl_proto); - off += sizeof(*vlan) - sizeof(*eh); - } - - switch (etype) { -#ifdef INET - case ETHERTYPE_IP: - ip = lagg_gethdr(m, off, sizeof(*ip), &buf); - if (ip == NULL) - goto out; - - if (sc->sc_flags & LAGG_F_HASHL3) { - p = fnv_32_buf(&ip->ip_src, sizeof(struct in_addr), p); - p = fnv_32_buf(&ip->ip_dst, sizeof(struct in_addr), p); - } - if (!(sc->sc_flags & LAGG_F_HASHL4)) - break; - switch (ip->ip_p) { - case IPPROTO_TCP: - case IPPROTO_UDP: - case IPPROTO_SCTP: - iphlen = ip->ip_hl << 2; - if (iphlen < sizeof(*ip)) - break; - off += iphlen; - ports = lagg_gethdr(m, off, sizeof(*ports), &buf); - if (ports == NULL) - break; - p = fnv_32_buf(ports, sizeof(*ports), p); - break; - } - break; -#endif -#ifdef INET6 - case ETHERTYPE_IPV6: - if (!(sc->sc_flags & LAGG_F_HASHL3)) - break; - ip6 = lagg_gethdr(m, off, sizeof(*ip6), &buf); - if (ip6 == NULL) - goto out; - - p = fnv_32_buf(&ip6->ip6_src, sizeof(struct in6_addr), p); - p = fnv_32_buf(&ip6->ip6_dst, sizeof(struct in6_addr), p); - flow = ip6->ip6_flow & IPV6_FLOWLABEL_MASK; - p = fnv_32_buf(&flow, sizeof(flow), p); /* IPv6 flow label */ - break; -#endif - } -out: - return (p); -} - int lagg_enqueue(struct ifnet *ifp, struct mbuf *m) { @@ -2160,7 +2055,7 @@ M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) p = m->m_pkthdr.flowid >> sc->flowid_shift; else - p = lagg_hashmbuf(sc, m, lb->lb_key); + p = m_ether_tcpip_hash(sc->sc_flags, m, lb->lb_key); p %= sc->sc_count; lp = lb->lb_ports[p]; Index: sys/ofed/drivers/net/mlx4/en_tx.c =================================================================== --- sys/ofed/drivers/net/mlx4/en_tx.c +++ sys/ofed/drivers/net/mlx4/en_tx.c @@ -49,7 +49,6 @@ #include #include "mlx4_en.h" -#include "utils.h" enum { MAX_INLINE = 104, /* 128 - 16 - 4 - 4 */ @@ -699,10 +698,10 @@ tx_desc->ctrl.fence_size = (real_size / 16) & 0x3f; } -static unsigned long hashrandom; +static uint32_t hashrandom; static void hashrandom_init(void *arg) { - hashrandom = random(); + hashrandom = m_ether_tcpip_hash_init(); } SYSINIT(hashrandom_init, SI_SUB_KLD, SI_ORDER_SECOND, &hashrandom_init, NULL); @@ -724,7 +723,7 @@ if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) queue_index = mb->m_pkthdr.flowid; else - queue_index = mlx4_en_hashmbuf(MLX4_F_HASHL3 | MLX4_F_HASHL4, mb, hashrandom); + queue_index = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | MBUF_HASHFLAG_L4, mb, hashrandom); return ((queue_index % rings_p_up) + (up * rings_p_up)); } Index: sys/ofed/drivers/net/mlx4/utils.h =================================================================== --- sys/ofed/drivers/net/mlx4/utils.h +++ sys/ofed/drivers/net/mlx4/utils.h @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2014 Mellanox Technologies Ltd. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef _MLX4_UTILS_H_ -#define _MLX4_UTILS_H_ - -/* Lagg flags */ -#define MLX4_F_HASHL2 0x00000001 /* hash layer 2 */ -#define MLX4_F_HASHL3 0x00000002 /* hash layer 3 */ -#define MLX4_F_HASHL4 0x00000004 /* hash layer 4 */ -#define MLX4_F_HASHMASK 0x00000007 - -uint32_t mlx4_en_hashmbuf(uint32_t flags, struct mbuf *m, uint32_t key); - -#endif /* _MLX4_UTILS_H_ */ Index: sys/ofed/drivers/net/mlx4/utils.c =================================================================== --- sys/ofed/drivers/net/mlx4/utils.c +++ sys/ofed/drivers/net/mlx4/utils.c @@ -1,189 +0,0 @@ -/* $OpenBSD: if_trunk.c,v 1.30 2007/01/31 06:20:19 reyk Exp $ */ - -/* - * Copyright (c) 2005, 2006 Reyk Floeter - * Copyright (c) 2007 Andrew Thompson - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include "opt_inet.h" -#include "opt_inet6.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(INET) || defined(INET6) -#include -#endif -#ifdef INET -#include -#include -#include -#endif - -#ifdef INET6 -#include -#include -#include -#endif - -#include - -#include "utils.h" - -/* XXX this code should be factored out */ -/* XXX copied from if_lagg.c */ - -static const void * -mlx4_en_gethdr(struct mbuf *m, u_int off, u_int len, void *buf) -{ - if (m->m_pkthdr.len < (off + len)) { - return (NULL); - } else if (m->m_len < (off + len)) { - m_copydata(m, off, len, buf); - return (buf); - } - return (mtod(m, char *) + off); -} - -uint32_t -mlx4_en_hashmbuf(uint32_t flags, struct mbuf *m, uint32_t key) -{ - uint16_t etype; - uint32_t p = key; - int off; - struct ether_header *eh; - const struct ether_vlan_header *vlan; -#ifdef INET - const struct ip *ip; - const uint32_t *ports; - int iphlen; -#endif -#ifdef INET6 - const struct ip6_hdr *ip6; - uint32_t flow; -#endif - union { -#ifdef INET - struct ip ip; -#endif -#ifdef INET6 - struct ip6_hdr ip6; -#endif - struct ether_vlan_header vlan; - uint32_t port; - } buf; - - - off = sizeof(*eh); - if (m->m_len < off) - goto out; - eh = mtod(m, struct ether_header *); - etype = ntohs(eh->ether_type); - if (flags & MLX4_F_HASHL2) { - p = hash32_buf(&eh->ether_shost, ETHER_ADDR_LEN, p); - p = hash32_buf(&eh->ether_dhost, ETHER_ADDR_LEN, p); - } - - /* Special handling for encapsulating VLAN frames */ - if ((m->m_flags & M_VLANTAG) && (flags & MLX4_F_HASHL2)) { - p = hash32_buf(&m->m_pkthdr.ether_vtag, - sizeof(m->m_pkthdr.ether_vtag), p); - } else if (etype == ETHERTYPE_VLAN) { - vlan = mlx4_en_gethdr(m, off, sizeof(*vlan), &buf); - if (vlan == NULL) - goto out; - - if (flags & MLX4_F_HASHL2) - p = hash32_buf(&vlan->evl_tag, sizeof(vlan->evl_tag), p); - etype = ntohs(vlan->evl_proto); - off += sizeof(*vlan) - sizeof(*eh); - } - - switch (etype) { -#ifdef INET - case ETHERTYPE_IP: - ip = mlx4_en_gethdr(m, off, sizeof(*ip), &buf); - if (ip == NULL) - goto out; - - if (flags & MLX4_F_HASHL3) { - p = hash32_buf(&ip->ip_src, sizeof(struct in_addr), p); - p = hash32_buf(&ip->ip_dst, sizeof(struct in_addr), p); - } - if (!(flags & MLX4_F_HASHL4)) - break; - switch (ip->ip_p) { - case IPPROTO_TCP: - case IPPROTO_UDP: - case IPPROTO_SCTP: - iphlen = ip->ip_hl << 2; - if (iphlen < sizeof(*ip)) - break; - off += iphlen; - ports = mlx4_en_gethdr(m, off, sizeof(*ports), &buf); - if (ports == NULL) - break; - p = hash32_buf(ports, sizeof(*ports), p); - break; - } - break; -#endif -#ifdef INET6 - case ETHERTYPE_IPV6: - if (!(flags & MLX4_F_HASHL3)) - break; - ip6 = mlx4_en_gethdr(m, off, sizeof(*ip6), &buf); - if (ip6 == NULL) - goto out; - - p = hash32_buf(&ip6->ip6_src, sizeof(struct in6_addr), p); - p = hash32_buf(&ip6->ip6_dst, sizeof(struct in6_addr), p); - flow = ip6->ip6_flow & IPV6_FLOWLABEL_MASK; - p = hash32_buf(&flow, sizeof(flow), p); /* IPv6 flow label */ - break; -#endif - } -out: - return (p); -} Index: sys/sys/mbuf.h =================================================================== --- sys/sys/mbuf.h +++ sys/sys/mbuf.h @@ -1186,6 +1186,16 @@ ((_m)->m_pkthdr.fibnum) = (_fib); \ } while (0) +/* mbuf hash routines */ + +#define MBUF_HASHFLAG_L2 (1 << 2) +#define MBUF_HASHFLAG_L3 (1 << 3) +#define MBUF_HASHFLAG_L4 (1 << 4) +#define MBUF_HASHFLAG_MASK (MBUF_HASHFLAG_L2 | MBUF_HASHFLAG_L3 | MBUF_HASHFLAG_L4) + +uint32_t m_ether_tcpip_hash_init(void); +uint32_t m_ether_tcpip_hash(const uint32_t, const struct mbuf *, const uint32_t); + #endif /* _KERNEL */ #ifdef MBUF_PROFILING