diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h --- a/sys/netinet/in_pcb.h +++ b/sys/netinet/in_pcb.h @@ -73,7 +73,8 @@ /* * PCB with AF_INET6 null bind'ed laddr can receive AF_INET input packet. * So, AF_INET6 null laddr is also used as AF_INET null laddr, by utilizing - * the following structure. + * the following structure. This requires padding always be zeroed out, + * which is done right after inpcb allocation and stays through its lifetime. */ struct in_addr_4in6 { u_int32_t ia46_pad32[3]; @@ -530,13 +531,36 @@ #define INP_HASH_WLOCK_ASSERT(ipi) mtx_assert(&(ipi)->ipi_hash_lock, \ MA_OWNED) -#define INP_PCBHASH(faddr, lport, fport, mask) \ - (((faddr) ^ ((faddr) >> 16) ^ ntohs((lport) ^ (fport))) & (mask)) -#define INP_PCBPORTHASH(lport, mask) \ - (ntohs((lport)) & (mask)) -#define INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) \ - ((faddr) ^ ((faddr) >> 16) ^ ntohs((lport) ^ (fport))) -#define INP6_PCBHASHKEY(faddr) ((faddr)->s6_addr32[3]) +/* + * Wildcard matching hash is not just a microoptimisation! The hash for + * wildcard IPv4 and wildcard IPv6 must be the same, otherwise AF_INET6 + * wildcard bound pcb won't be able to receive AF_INET connections, while: + * jenkins_hash(&zeroes, 1, s) != jenkins_hash(&zeroes, 4, s) + * See also comment above struct in_addr_4in6. + */ +#define IN_ADDR_JHASH32(addr) \ + ((addr)->s_addr == INADDR_ANY ? V_in_pcbhashseed : \ + jenkins_hash32((&(addr)->s_addr), 1, V_in_pcbhashseed)) +#define IN6_ADDR_JHASH32(addr) \ + (memcmp((addr), &in6addr_any, sizeof(in6addr_any)) == 0 ? \ + V_in_pcbhashseed : \ + jenkins_hash32((addr)->__u6_addr.__u6_addr32, \ + nitems((addr)->__u6_addr.__u6_addr32), V_in_pcbhashseed)) + +#define INP_PCBHASH(faddr, lport, fport, mask) \ + ((IN_ADDR_JHASH32(faddr) ^ ntohs((lport) ^ (fport))) & (mask)) +#define INP6_PCBHASH(faddr, lport, fport, mask) \ + ((IN6_ADDR_JHASH32(faddr) ^ ntohs((lport) ^ (fport))) & (mask)) + +#define INP_PCBHASH_WILD(lport, mask) \ + ((V_in_pcbhashseed ^ ntohs(lport)) & (mask)) + +#define INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) \ + (IN_ADDR_JHASH32(faddr) ^ ntohs((lport) ^ (fport))) +#define INP6_PCBLBGROUP_PKTHASH(faddr, lport, fport) \ + (IN6_ADDR_JHASH32(faddr) ^ ntohs((lport) ^ (fport))) + +#define INP_PCBPORTHASH(lport, mask) (ntohs((lport)) & (mask)) /* * Flags for inp_vflags -- historically version flags only diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c --- a/sys/netinet/in_pcb.c +++ b/sys/netinet/in_pcb.c @@ -49,7 +49,9 @@ #include "opt_rss.h" #include +#include #include +#include #include #include #include @@ -246,6 +248,16 @@ #endif /* INET */ +VNET_DEFINE(uint32_t, in_pcbhashseed); +static void +in_pcbhashseed_init(void) +{ + + V_in_pcbhashseed = arc4random(); +} +VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, + in_pcbhashseed_init, 0); + /* * in_pcb.c: manage the Protocol Control Blocks. * @@ -2085,8 +2097,8 @@ * Look for an unconnected (wildcard foreign addr) PCB that * matches the local address and port we're looking for. */ - head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport, - 0, pcbinfo->ipi_hashmask)]; + head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, + pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { #ifdef INET6 /* XXX inp locking */ @@ -2214,7 +2226,7 @@ if (grp->il_lport != lport) continue; - idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) % + idx = INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) % grp->il_inpcnt; if (grp->il_laddr.s_addr == laddr->s_addr) { if (numa_domain == M_NODOM || @@ -2260,7 +2272,7 @@ * First look for an exact match. */ tmpinp = NULL; - head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport, + head = &pcbinfo->ipi_hashbase[INP_PCBHASH(&faddr, lport, fport, pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { #ifdef INET6 @@ -2315,8 +2327,8 @@ * 4. non-jailed, wild. */ - head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport, - 0, pcbinfo->ipi_hashmask)]; + head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, + pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { #ifdef INET6 /* XXX inp locking */ @@ -2439,7 +2451,6 @@ struct inpcbporthead *pcbporthash; struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; struct inpcbport *phd; - u_int32_t hashkey_faddr; int so_options; INP_WLOCK_ASSERT(inp); @@ -2450,13 +2461,12 @@ #ifdef INET6 if (inp->inp_vflag & INP_IPV6) - hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr); + pcbhash = &pcbinfo->ipi_hashbase[INP6_PCBHASH(&inp->in6p_faddr, + inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; else #endif - hashkey_faddr = inp->inp_faddr.s_addr; - - pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr, - inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; + pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(&inp->inp_faddr, + inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; pcbporthash = &pcbinfo->ipi_porthashbase[ INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)]; @@ -2516,7 +2526,6 @@ { struct inpcbinfo *pcbinfo = inp->inp_pcbinfo; struct inpcbhead *head; - u_int32_t hashkey_faddr; INP_WLOCK_ASSERT(inp); INP_HASH_WLOCK_ASSERT(pcbinfo); @@ -2526,13 +2535,12 @@ #ifdef INET6 if (inp->inp_vflag & INP_IPV6) - hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr); + head = &pcbinfo->ipi_hashbase[INP6_PCBHASH(&inp->in6p_faddr, + inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; else #endif - hashkey_faddr = inp->inp_faddr.s_addr; - - head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr, - inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; + head = &pcbinfo->ipi_hashbase[INP_PCBHASH(&inp->inp_faddr, + inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)]; CK_LIST_REMOVE(inp, inp_hash); CK_LIST_INSERT_HEAD(head, inp, inp_hash); diff --git a/sys/netinet/in_pcb_var.h b/sys/netinet/in_pcb_var.h --- a/sys/netinet/in_pcb_var.h +++ b/sys/netinet/in_pcb_var.h @@ -44,6 +44,9 @@ * Definitions shared between netinet/in_pcb.c and netinet6/in6_pcb.c */ +VNET_DECLARE(uint32_t, in_pcbhashseed); +#define V_in_pcbhashseed VNET(in_pcbhashseed) + bool inp_smr_lock(struct inpcb *, const inp_lookup_t); int in_pcb_lport(struct inpcb *, struct in_addr *, u_short *, struct ucred *, int); diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c --- a/sys/netinet6/in6_pcb.c +++ b/sys/netinet6/in6_pcb.c @@ -75,6 +75,7 @@ #include "opt_route.h" #include "opt_rss.h" +#include #include #include #include @@ -787,8 +788,7 @@ * Look for an unconnected (wildcard foreign addr) PCB that * matches the local address and port we're looking for. */ - head = &pcbinfo->ipi_hashbase[INP_PCBHASH( - INP6_PCBHASHKEY(&in6addr_any), lport, 0, + head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { /* XXX inp locking */ @@ -972,8 +972,8 @@ if (grp->il_lport != lport) continue; - idx = INP_PCBLBGROUP_PKTHASH(INP6_PCBHASHKEY(faddr), lport, - fport) % grp->il_inpcnt; + idx = INP6_PCBLBGROUP_PKTHASH(faddr, lport, fport) % + grp->il_inpcnt; if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr)) { if (numa_domain == M_NODOM || grp->il_numa_domain == numa_domain) { @@ -1015,8 +1015,8 @@ * First look for an exact match. */ tmpinp = NULL; - head = &pcbinfo->ipi_hashbase[INP_PCBHASH( - INP6_PCBHASHKEY(faddr), lport, fport, pcbinfo->ipi_hashmask)]; + head = &pcbinfo->ipi_hashbase[INP6_PCBHASH(faddr, lport, fport, + pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { /* XXX inp locking */ if ((inp->inp_vflag & INP_IPV6) == 0) @@ -1064,8 +1064,7 @@ * 3. non-jailed, non-wild. * 4. non-jailed, wild. */ - head = &pcbinfo->ipi_hashbase[INP_PCBHASH( - INP6_PCBHASHKEY(&in6addr_any), lport, 0, + head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { /* XXX inp locking */