diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c --- a/sys/netinet/in_pcb.c +++ b/sys/netinet/in_pcb.c @@ -139,8 +139,7 @@ static struct inpcb *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport_arg, struct in_addr laddr, u_int lport_arg, - int lookupflags, struct ifnet *ifp, - uint8_t numa_domain); + int lookupflags, uint8_t numa_domain); #define RANGECHK(var, min, max) \ if ((var) < (min)) { (var) = (min); } \ @@ -785,14 +784,14 @@ if (lsa->sa_family == AF_INET) { tmpinp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, lookupflags, - NULL, M_NODOM); + M_NODOM); } #endif #ifdef INET6 if (lsa->sa_family == AF_INET6) { tmpinp = in6_pcblookup_hash_locked(pcbinfo, faddr6, fport, laddr6, lport, lookupflags, - NULL, M_NODOM); + M_NODOM); } #endif } else { @@ -1425,7 +1424,7 @@ if (lport != 0) { oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, - fport, laddr, lport, 0, NULL, M_NODOM); + fport, laddr, lport, 0, M_NODOM); if (oinp != NULL) { if (oinpp != NULL) *oinpp = oinp; @@ -2154,8 +2153,8 @@ static struct inpcb * in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, - const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr, - uint16_t fport, int lookupflags, int domain) + const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr, + uint16_t lport, int domain) { const struct inpcblbgrouphead *hdr; struct inpcblbgroup *grp; @@ -2198,8 +2197,7 @@ in_pcblookup_lb_numa_match(grp, domain)) { local_exact = grp; } - } else if (grp->il_laddr.s_addr == INADDR_ANY && - (lookupflags & INPLOOKUP_WILDCARD) != 0) { + } else if (grp->il_laddr.s_addr == INADDR_ANY) { if (injail) { if (jail_wild == NULL || in_pcblookup_lb_numa_match(grp, domain)) @@ -2226,33 +2224,16 @@ grp->il_inpcnt]); } -/* - * Lookup PCB in hash list, using pcbinfo tables. This variation assumes - * that the caller has either locked the hash list, which usually happens - * for bind(2) operations, or is in SMR section, which happens when sorting - * out incoming packets. - */ static struct inpcb * -in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, - u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, - struct ifnet *ifp, uint8_t numa_domain) +in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr, + u_short fport, struct in_addr laddr, u_short lport) { struct inpcbhead *head; - struct inpcb *inp, *tmpinp; - u_short fport = fport_arg, lport = lport_arg; + struct inpcb *inp, *match; - KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, - ("%s: invalid lookup flags %d", __func__, lookupflags)); - KASSERT(faddr.s_addr != INADDR_ANY, - ("%s: invalid foreign address", __func__)); - KASSERT(laddr.s_addr != INADDR_ANY, - ("%s: invalid local address", __func__)); INP_HASH_LOCK_ASSERT(pcbinfo); - /* - * First look for an exact match. - */ - tmpinp = NULL; + match = NULL; head = &pcbinfo->ipi_hashbase[INP_PCBHASH(&faddr, lport, fport, pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { @@ -2272,105 +2253,132 @@ */ if (prison_flag(inp->inp_cred, PR_IP4)) return (inp); - if (tmpinp == NULL) - tmpinp = inp; + if (match == NULL) + match = inp; } } - if (tmpinp != NULL) - return (tmpinp); + return (match); +} - /* - * Then look for a wildcard match, if requested. - */ - if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { - struct inpcb *local_wild = NULL, *local_exact = NULL; +static struct inpcb * +in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, + u_short fport, struct in_addr laddr, u_short lport) +{ + struct inpcbhead *head; + struct inpcb *inp, *local_wild, *local_exact, *jail_wild; #ifdef INET6 - struct inpcb *local_wild_mapped = NULL; + struct inpcb *local_wild_mapped; #endif - struct inpcb *jail_wild = NULL; - int injail; - /* - * First see if an LB group matches the request before scanning - * all sockets on this port. - */ - inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr, - fport, lookupflags, numa_domain); - if (inp != NULL) - return (inp); + INP_HASH_LOCK_ASSERT(pcbinfo); - /* - * Order of socket selection - we always prefer jails. - * 1. jailed, non-wild. - * 2. jailed, wild. - * 3. non-jailed, non-wild. - * 4. non-jailed, wild. - */ + /* + * Order of socket selection - we always prefer jails. + * 1. jailed, non-wild. + * 2. jailed, wild. + * 3. non-jailed, non-wild. + * 4. non-jailed, wild. + */ + head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, + pcbinfo->ipi_hashmask)]; + local_wild = local_exact = jail_wild = NULL; +#ifdef INET6 + local_wild_mapped = NULL; +#endif + CK_LIST_FOREACH(inp, head, inp_hash) { + bool injail; - head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, - pcbinfo->ipi_hashmask)]; - CK_LIST_FOREACH(inp, head, inp_hash) { #ifdef INET6 - /* XXX inp locking */ - if ((inp->inp_vflag & INP_IPV4) == 0) - continue; + /* XXX inp locking */ + if ((inp->inp_vflag & INP_IPV4) == 0) + continue; #endif - if (inp->inp_faddr.s_addr != INADDR_ANY || - inp->inp_lport != lport) - continue; + if (inp->inp_faddr.s_addr != INADDR_ANY || + inp->inp_lport != lport) + continue; - injail = prison_flag(inp->inp_cred, PR_IP4); - if (injail) { - if (prison_check_ip4_locked( - inp->inp_cred->cr_prison, &laddr) != 0) - continue; - } else { - if (local_exact != NULL) - continue; - } + injail = prison_flag(inp->inp_cred, PR_IP4) != 0; + if (injail) { + if (prison_check_ip4_locked(inp->inp_cred->cr_prison, + &laddr) != 0) + continue; + } else { + if (local_exact != NULL) + continue; + } - if (inp->inp_laddr.s_addr == laddr.s_addr) { - if (injail) - return (inp); - else - local_exact = inp; - } else if (inp->inp_laddr.s_addr == INADDR_ANY) { + if (inp->inp_laddr.s_addr == laddr.s_addr) { + if (injail) + return (inp); + local_exact = inp; + } else if (inp->inp_laddr.s_addr == INADDR_ANY) { #ifdef INET6 - /* XXX inp locking, NULL check */ - if (inp->inp_vflag & INP_IPV6PROTO) - local_wild_mapped = inp; - else + /* XXX inp locking, NULL check */ + if (inp->inp_vflag & INP_IPV6PROTO) + local_wild_mapped = inp; + else #endif - if (injail) - jail_wild = inp; - else - local_wild = inp; - } - } /* LIST_FOREACH */ - if (jail_wild != NULL) - return (jail_wild); - if (local_exact != NULL) - return (local_exact); - if (local_wild != NULL) - return (local_wild); + if (injail) + jail_wild = inp; + else + local_wild = inp; + } + } + if (jail_wild != NULL) + return (jail_wild); + if (local_exact != NULL) + return (local_exact); + if (local_wild != NULL) + return (local_wild); #ifdef INET6 - if (local_wild_mapped != NULL) - return (local_wild_mapped); + if (local_wild_mapped != NULL) + return (local_wild_mapped); #endif - } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */ - return (NULL); } /* - * Lookup PCB in hash list, using pcbinfo tables. This variation locks the - * hash list lock, and will return the inpcb locked (i.e., requires - * INPLOOKUP_LOCKPCB). + * Lookup PCB in hash list, using pcbinfo tables. This variation assumes + * that the caller has either locked the hash list, which usually happens + * for bind(2) operations, or is in SMR section, which happens when sorting + * out incoming packets. */ static struct inpcb * -in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr, +in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr, + u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags, + uint8_t numa_domain) +{ + struct inpcb *inp; + const u_short fport = fport_arg, lport = lport_arg; + + KASSERT((lookupflags & ~INPLOOKUP_WILDCARD) == 0, + ("%s: invalid lookup flags %d", __func__, lookupflags)); + KASSERT(faddr.s_addr != INADDR_ANY, + ("%s: invalid foreign address", __func__)); + KASSERT(laddr.s_addr != INADDR_ANY, + ("%s: invalid local address", __func__)); + INP_HASH_LOCK_ASSERT(pcbinfo); + + inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport); + if (inp != NULL) + return (inp); + + if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { + inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport, &laddr, + lport, numa_domain); + if (inp == NULL) { + inp = in_pcblookup_hash_wild_locked(pcbinfo, faddr, + fport, laddr, lport); + } + } + + return (inp); +} + +static struct inpcb * +in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport, struct in_addr laddr, u_int lport, int lookupflags, - struct ifnet *ifp, uint8_t numa_domain) + uint8_t numa_domain) { struct inpcb *inp; @@ -2381,7 +2389,7 @@ smr_enter(pcbinfo->ipi_smr); inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, - lookupflags & INPLOOKUP_WILDCARD, ifp, numa_domain); + lookupflags & INPLOOKUP_WILDCARD, numa_domain); if (inp != NULL) { if (__predict_false(inp_smr_lock(inp, (lookupflags & INPLOOKUP_LOCKMASK)) == false)) @@ -2398,19 +2406,20 @@ */ struct inpcb * in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport, - struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp) + struct in_addr laddr, u_int lport, int lookupflags, + struct ifnet *ifp __unused) { - return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, - lookupflags, ifp, M_NODOM)); + return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, + lookupflags, M_NODOM)); } struct inpcb * in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport, struct in_addr laddr, u_int lport, int lookupflags, - struct ifnet *ifp, struct mbuf *m) + struct ifnet *ifp __unused, struct mbuf *m) { - return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, - lookupflags, ifp, m->m_pkthdr.numa_domain)); + return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, + lookupflags, m->m_pkthdr.numa_domain)); } #endif /* INET */ diff --git a/sys/netinet6/in6_pcb.h b/sys/netinet6/in6_pcb.h --- a/sys/netinet6/in6_pcb.h +++ b/sys/netinet6/in6_pcb.h @@ -85,7 +85,7 @@ struct inpcb * in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, u_int fport_arg, struct in6_addr *laddr, - u_int lport_arg, int lookupflags, struct ifnet *ifp, uint8_t); + u_int lport_arg, int lookupflags, uint8_t); struct inpcb * in6_pcblookup(struct inpcbinfo *, struct in6_addr *, u_int, struct in6_addr *, u_int, int, diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c --- a/sys/netinet6/in6_pcb.c +++ b/sys/netinet6/in6_pcb.c @@ -452,12 +452,10 @@ return (error); if (in6_pcblookup_hash_locked(pcbinfo, &sin6->sin6_addr, - sin6->sin6_port, - IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) - ? &laddr6.sin6_addr : &inp->in6p_laddr, - inp->inp_lport, 0, NULL, M_NODOM) != NULL) { + sin6->sin6_port, IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr) ? + &laddr6.sin6_addr : &inp->in6p_laddr, inp->inp_lport, 0, + M_NODOM) != NULL) return (EADDRINUSE); - } if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) { if (inp->inp_lport == 0) { /* @@ -895,8 +893,8 @@ static struct inpcb * in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo, - const struct in6_addr *laddr, uint16_t lport, const struct in6_addr *faddr, - uint16_t fport, int lookupflags, uint8_t domain) + const struct in6_addr *faddr, uint16_t fport, const struct in6_addr *laddr, + uint16_t lport, uint8_t domain) { const struct inpcblbgrouphead *hdr; struct inpcblbgroup *grp; @@ -939,8 +937,7 @@ in6_pcblookup_lb_numa_match(grp, domain)) { local_exact = grp; } - } else if (IN6_IS_ADDR_UNSPECIFIED(&grp->il6_laddr) && - (lookupflags & INPLOOKUP_WILDCARD) != 0) { + } else if (IN6_IS_ADDR_UNSPECIFIED(&grp->il6_laddr)) { if (injail) { if (jail_wild == NULL || in6_pcblookup_lb_numa_match(grp, domain)) @@ -967,31 +964,19 @@ grp->il_inpcnt]); } -/* - * Lookup PCB in hash list. Used in in_pcb.c as well as here. - */ -struct inpcb * -in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, - u_int fport_arg, struct in6_addr *laddr, u_int lport_arg, - int lookupflags, struct ifnet *ifp, uint8_t numa_domain) +static struct inpcb * +in6_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, + u_short fport, struct in6_addr *laddr, u_short lport) { struct inpcbhead *head; - struct inpcb *inp, *tmpinp; - u_short fport = fport_arg, lport = lport_arg; - - KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0, - ("%s: invalid lookup flags %d", __func__, lookupflags)); - KASSERT(!IN6_IS_ADDR_UNSPECIFIED(faddr), - ("%s: invalid foreign address", __func__)); - KASSERT(!IN6_IS_ADDR_UNSPECIFIED(laddr), - ("%s: invalid local address", __func__)); + struct inpcb *inp, *match; INP_HASH_LOCK_ASSERT(pcbinfo); /* * First look for an exact match. */ - tmpinp = NULL; + match = NULL; head = &pcbinfo->ipi_hashbase[INP6_PCBHASH(faddr, lport, fport, pcbinfo->ipi_hashmask)]; CK_LIST_FOREACH(inp, head, inp_hash) { @@ -1009,95 +994,111 @@ */ if (prison_flag(inp->inp_cred, PR_IP6)) return (inp); - if (tmpinp == NULL) - tmpinp = inp; + if (match == NULL) + match = inp; } } - if (tmpinp != NULL) - return (tmpinp); + return (match); +} + +static struct inpcb * +in6_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, + struct in6_addr *faddr, u_short fport, struct in6_addr *laddr, + u_short lport) +{ + struct inpcbhead *head; + struct inpcb *inp, *jail_wild, *local_exact, *local_wild; /* - * Then look for a wildcard match, if requested. + * Order of socket selection - we always prefer jails. + * 1. jailed, non-wild. + * 2. jailed, wild. + * 3. non-jailed, non-wild. + * 4. non-jailed, wild. */ - if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { - struct inpcb *local_wild = NULL, *local_exact = NULL; - struct inpcb *jail_wild = NULL; - int injail; + head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, + pcbinfo->ipi_hashmask)]; + local_wild = local_exact = jail_wild = NULL; + CK_LIST_FOREACH(inp, head, inp_hash) { + bool injail; - /* - * First see if an LB group matches the request before scanning - * all sockets on this port. - */ - inp = in6_pcblookup_lbgroup(pcbinfo, laddr, lport, faddr, - fport, lookupflags, numa_domain); - if (inp != NULL) - return (inp); + /* XXX inp locking */ + if ((inp->inp_vflag & INP_IPV6) == 0) + continue; - /* - * Order of socket selection - we always prefer jails. - * 1. jailed, non-wild. - * 2. jailed, wild. - * 3. non-jailed, non-wild. - * 4. non-jailed, wild. - */ - head = &pcbinfo->ipi_hashbase[INP_PCBHASH_WILD(lport, - pcbinfo->ipi_hashmask)]; - CK_LIST_FOREACH(inp, head, inp_hash) { - /* XXX inp locking */ - if ((inp->inp_vflag & INP_IPV6) == 0) - continue; + if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) || + inp->inp_lport != lport) { + continue; + } - if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) || - inp->inp_lport != lport) { + injail = prison_flag(inp->inp_cred, PR_IP6) != 0; + if (injail) { + if (prison_check_ip6_locked( + inp->inp_cred->cr_prison, laddr) != 0) continue; - } + } else { + if (local_exact != NULL) + continue; + } - injail = prison_flag(inp->inp_cred, PR_IP6); - if (injail) { - if (prison_check_ip6_locked( - inp->inp_cred->cr_prison, laddr) != 0) - continue; - } else { - if (local_exact != NULL) - continue; - } + if (IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr)) { + if (injail) + return (inp); + else + local_exact = inp; + } else if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) { + if (injail) + jail_wild = inp; + else + local_wild = inp; + } + } - if (IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr)) { - if (injail) - return (inp); - else - local_exact = inp; - } else if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) { - if (injail) - jail_wild = inp; - else - local_wild = inp; - } - } /* LIST_FOREACH */ + if (jail_wild != NULL) + return (jail_wild); + if (local_exact != NULL) + return (local_exact); + if (local_wild != NULL) + return (local_wild); + return (NULL); +} + +struct inpcb * +in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, + u_int fport_arg, struct in6_addr *laddr, u_int lport_arg, + int lookupflags, uint8_t numa_domain) +{ + struct inpcb *inp; + u_short fport = fport_arg, lport = lport_arg; - if (jail_wild != NULL) - return (jail_wild); - if (local_exact != NULL) - return (local_exact); - if (local_wild != NULL) - return (local_wild); - } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */ + KASSERT((lookupflags & ~INPLOOKUP_WILDCARD) == 0, + ("%s: invalid lookup flags %d", __func__, lookupflags)); + KASSERT(!IN6_IS_ADDR_UNSPECIFIED(faddr), + ("%s: invalid foreign address", __func__)); + KASSERT(!IN6_IS_ADDR_UNSPECIFIED(laddr), + ("%s: invalid local address", __func__)); - /* - * Not found. - */ - return (NULL); + INP_HASH_LOCK_ASSERT(pcbinfo); + + inp = in6_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport); + if (inp != NULL) + return (inp); + + if ((lookupflags & INPLOOKUP_WILDCARD) != 0) { + inp = in6_pcblookup_lbgroup(pcbinfo, faddr, fport, laddr, + lport, numa_domain); + if (inp == NULL) { + inp = in6_pcblookup_hash_wild_locked(pcbinfo, faddr, + fport, laddr, lport); + } + } + return (inp); } -/* - * Lookup PCB in hash list, using pcbinfo tables. This variation locks the - * hash list lock, and will return the inpcb locked (i.e., requires - * INPLOOKUP_LOCKPCB). - */ static struct inpcb * -in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, +in6_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, u_int fport, struct in6_addr *laddr, u_int lport, int lookupflags, - struct ifnet *ifp, uint8_t numa_domain) + uint8_t numa_domain) { struct inpcb *inp; @@ -1108,7 +1109,7 @@ smr_enter(pcbinfo->ipi_smr); inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport, - lookupflags & INPLOOKUP_WILDCARD, ifp, numa_domain); + lookupflags & INPLOOKUP_WILDCARD, numa_domain); if (inp != NULL) { if (__predict_false(inp_smr_lock(inp, (lookupflags & INPLOOKUP_LOCKMASK)) == false)) @@ -1125,19 +1126,20 @@ */ struct inpcb * in6_pcblookup(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, u_int fport, - struct in6_addr *laddr, u_int lport, int lookupflags, struct ifnet *ifp) + struct in6_addr *laddr, u_int lport, int lookupflags, + struct ifnet *ifp __unused) { - return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, - lookupflags, ifp, M_NODOM)); + return (in6_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, + lookupflags, M_NODOM)); } struct inpcb * in6_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, u_int fport, struct in6_addr *laddr, u_int lport, int lookupflags, - struct ifnet *ifp, struct mbuf *m) + struct ifnet *ifp __unused, struct mbuf *m) { - return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport, - lookupflags, ifp, m->m_pkthdr.numa_domain)); + return (in6_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport, + lookupflags, m->m_pkthdr.numa_domain)); } void