diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c --- a/sys/netinet/tcp_input.c +++ b/sys/netinet/tcp_input.c @@ -926,37 +926,6 @@ } INP_LOCK_ASSERT(inp); - if ((inp->inp_flowtype == M_HASHTYPE_NONE) && - !SOLISTENING(inp->inp_socket)) { - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { - inp->inp_flowid = m->m_pkthdr.flowid; - inp->inp_flowtype = M_HASHTYPE_GET(m); -#ifdef RSS - } else { - /* assign flowid by software RSS hash */ -#ifdef INET6 - if (isipv6) { - rss_proto_software_hash_v6(&inp->in6p_faddr, - &inp->in6p_laddr, - inp->inp_fport, - inp->inp_lport, - IPPROTO_TCP, - &inp->inp_flowid, - &inp->inp_flowtype); - } else -#endif /* INET6 */ - { - rss_proto_software_hash_v4(inp->inp_faddr, - inp->inp_laddr, - inp->inp_fport, - inp->inp_lport, - IPPROTO_TCP, - &inp->inp_flowid, - &inp->inp_flowtype); - } -#endif /* RSS */ - } - } #if defined(IPSEC) || defined(IPSEC_SUPPORT) #ifdef INET6 if (isipv6 && IPSEC_ENABLED(ipv6) && @@ -1177,7 +1146,7 @@ * causes. */ if (thflags & TH_RST) { - syncache_chkrst(&inc, th, m, port); + syncache_chkrst(&inc, th, port); goto dropunlock; } /* diff --git a/sys/netinet/tcp_syncache.h b/sys/netinet/tcp_syncache.h --- a/sys/netinet/tcp_syncache.h +++ b/sys/netinet/tcp_syncache.h @@ -43,8 +43,7 @@ struct socket * syncache_add(struct in_conninfo *, struct tcpopt *, struct tcphdr *, struct inpcb *, struct socket *, struct mbuf *, void *, void *, uint8_t, uint16_t); -void syncache_chkrst(struct in_conninfo *, struct tcphdr *, struct mbuf *, - uint16_t); +void syncache_chkrst(struct in_conninfo *, struct tcphdr *, uint16_t); void syncache_badack(struct in_conninfo *, uint16_t); int syncache_pcblist(struct sysctl_req *); @@ -66,6 +65,7 @@ u_int8_t sc_ip_tos; /* TOS / Traffic Class */ u_int8_t sc_requested_s_scale:4, sc_requested_r_scale:4; + uint8_t sc_numa_domain; u_int16_t sc_flags; #if defined(TCP_OFFLOAD) struct toedev *sc_tod; /* entry added by this TOE */ @@ -74,8 +74,8 @@ struct label *sc_label; /* MAC label reference */ struct ucred *sc_cred; /* cred cache for jail checks */ void *sc_tfo_cookie; /* for TCP Fast Open response */ - void *sc_pspare; /* TCP_SIGNATURE */ - u_int32_t sc_spare[2]; /* UTO */ + uint32_t sc_flowtype; /* flowid from SYN packet .. */ + uint32_t sc_flowid; /* .. or calculated by RSS */ }; /* diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c --- a/sys/netinet/tcp_syncache.c +++ b/sys/netinet/tcp_syncache.c @@ -36,6 +36,7 @@ #include "opt_inet.h" #include "opt_inet6.h" #include "opt_ipsec.h" +#include "opt_rss.h" #include #include @@ -121,7 +122,7 @@ static void syncache_drop(struct syncache *, struct syncache_head *); static void syncache_free(struct syncache *); static void syncache_insert(struct syncache *, struct syncache_head *); -static int syncache_respond(struct syncache *, const struct mbuf *, int); +static int syncache_respond(struct syncache *, int); static struct socket *syncache_socket(struct syncache *, struct socket *, struct mbuf *m); static void syncache_timeout(struct syncache *sc, struct syncache_head *sch, @@ -528,7 +529,7 @@ } NET_EPOCH_ENTER(et); - if (syncache_respond(sc, NULL, TH_SYN|TH_ACK) == 0) { + if (syncache_respond(sc, TH_SYN|TH_ACK) == 0) { syncache_timeout(sc, sch, 0); TCPSTAT_INC(tcps_sndacks); TCPSTAT_INC(tcps_sndtotal); @@ -608,8 +609,7 @@ * If required send a challenge ACK. */ void -syncache_chkrst(struct in_conninfo *inc, struct tcphdr *th, struct mbuf *m, - uint16_t port) +syncache_chkrst(struct in_conninfo *inc, struct tcphdr *th, uint16_t port) { struct syncache *sc; struct syncache_head *sch; @@ -695,7 +695,7 @@ "sending challenge ACK\n", s, __func__, th->th_seq, sc->sc_irs + 1, sc->sc_wnd); - if (syncache_respond(sc, m, TH_ACK) == 0) { + if (syncache_respond(sc, TH_ACK) == 0) { TCPSTAT_INC(tcps_sndacks); TCPSTAT_INC(tcps_sndtotal); } else { @@ -827,19 +827,6 @@ #ifdef INET6 } #endif - - /* - * If there's an mbuf and it has a flowid, then let's initialise the - * inp with that particular flowid. - */ - if (m != NULL && M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { - inp->inp_flowid = m->m_pkthdr.flowid; - inp->inp_flowtype = M_HASHTYPE_GET(m); -#ifdef NUMA - inp->inp_numa_domain = m->m_pkthdr.numa_domain; -#endif - } - inp->inp_lport = sc->sc_inc.inc_lport; #ifdef INET6 if (inp->inp_vflag & INP_IPV6PROTO) { @@ -910,6 +897,38 @@ if (ipsec_copy_pcbpolicy(sotoinpcb(lso), inp) != 0) printf("syncache_socket: could not copy policy\n"); #endif + if (sc->sc_flowtype != M_HASHTYPE_NONE) { + inp->inp_flowid = sc->sc_flowid; + inp->inp_flowtype = sc->sc_flowtype; +#ifdef RSS + } else { + /* assign flowid by software RSS hash */ +#ifdef INET6 + if (sc->sc_inc.inc_flags & INC_ISIPV6) { + rss_proto_software_hash_v6(&inp->in6p_faddr, + &inp->in6p_laddr, + inp->inp_fport, + inp->inp_lport, + IPPROTO_TCP, + &inp->inp_flowid, + &inp->inp_flowtype); + } else +#endif /* INET6 */ + { + rss_proto_software_hash_v4(inp->inp_faddr, + inp->inp_laddr, + inp->inp_fport, + inp->inp_lport, + IPPROTO_TCP, + &inp->inp_flowid, + &inp->inp_flowtype); + } +#endif /* RSS */ + } +#ifdef NUMA + inp->inp_numa_domain = sc->sc_numa_domain; +#endif + tp->t_state = TCPS_SYN_RECEIVED; tp->iss = sc->sc_iss; tp->irs = sc->sc_irs; @@ -1144,6 +1163,13 @@ return (-1); /* Do not send RST */ } #endif /* TCP_SIGNATURE */ + if (m != NULL && M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { + sc->sc_flowid = m->m_pkthdr.flowid; + sc->sc_flowtype = M_HASHTYPE_GET(m); + } +#ifdef NUMA + sc->sc_numa_domain = m ? m->m_pkthdr.numa_domain : M_NODOM; +#endif TCPSTATES_INC(TCPS_SYN_RECEIVED); } else { if (sc->sc_port != port) { @@ -1560,7 +1586,7 @@ s, __func__); free(s, M_TCPLOG); } - if (syncache_respond(sc, m, TH_SYN|TH_ACK) == 0) { + if (syncache_respond(sc, TH_SYN|TH_ACK) == 0) { sc->sc_rxmits = 0; syncache_timeout(sc, sch, 1); TCPSTAT_INC(tcps_sndacks); @@ -1731,6 +1757,13 @@ sc->sc_flowlabel = ip6_randomflowlabel(); sc->sc_flowlabel = htonl(sc->sc_flowlabel) & IPV6_FLOWLABEL_MASK; } +#endif + if (m != NULL && M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { + sc->sc_flowid = m->m_pkthdr.flowid; + sc->sc_flowtype = M_HASHTYPE_GET(m); + } +#ifdef NUMA + sc->sc_numa_domain = m ? m->m_pkthdr.numa_domain : M_NODOM; #endif if (locked) SCH_UNLOCK(sch); @@ -1745,7 +1778,7 @@ /* * Do a standard 3-way handshake. */ - if (syncache_respond(sc, m, TH_SYN|TH_ACK) == 0) { + if (syncache_respond(sc, TH_SYN|TH_ACK) == 0) { if (sc != &scs) syncache_insert(sc, sch); /* locks and unlocks sch */ TCPSTAT_INC(tcps_sndacks); @@ -1783,11 +1816,11 @@ } /* - * Send SYN|ACK or ACK to the peer. Either in response to a peer's segment, - * i.e. m0 != NULL, or upon 3WHS ACK timeout, i.e. m0 == NULL. + * Send SYN|ACK or ACK to the peer. Either in response to a peer's segment + * or upon 3WHS ACK timeout. */ static int -syncache_respond(struct syncache *sc, const struct mbuf *m0, int flags) +syncache_respond(struct syncache *sc, int flags) { struct ip *ip = NULL; struct mbuf *m; @@ -1977,15 +2010,11 @@ udp->uh_ulen = htons(ulen); } M_SETFIB(m, sc->sc_inc.inc_fibnum); - /* - * If we have peer's SYN and it has a flowid, then let's assign it to - * our SYN|ACK. ip6_output() and ip_output() will not assign flowid - * to SYN|ACK due to lack of inp here. - */ - if (m0 != NULL && M_HASHTYPE_GET(m0) != M_HASHTYPE_NONE) { - m->m_pkthdr.flowid = m0->m_pkthdr.flowid; - M_HASHTYPE_SET(m, M_HASHTYPE_GET(m0)); - } + m->m_pkthdr.flowid = sc->sc_flowid; + M_HASHTYPE_SET(m, sc->sc_flowtype); +#ifdef NUMA + m->m_pkthdr.numa_domain = sc->sc_numa_domain; +#endif #ifdef INET6 if (sc->sc_inc.inc_flags & INC_ISIPV6) { if (sc->sc_port) {