Index: sys/netinet/tcp_debug.c =================================================================== --- sys/netinet/tcp_debug.c +++ sys/netinet/tcp_debug.c @@ -189,11 +189,11 @@ else printf("%x", seq); printf("@%x, urp=%x", ack, th->th_urp); - flags = th->th_flags; + flags = tcp_get_flags(th); if (flags) { char *cp = "<"; #define pf(f) { \ - if (th->th_flags & TH_##f) { \ + if (tcp_get_flags(th) & TH_##f) { \ printf("%s%s", cp, #f); \ cp = ","; \ } \ Index: sys/netinet/tcp_input.c =================================================================== --- sys/netinet/tcp_input.c +++ sys/netinet/tcp_input.c @@ -548,7 +548,7 @@ void inline cc_ecnpkt_handler(struct tcpcb *tp, struct tcphdr *th, uint8_t iptos) { - cc_ecnpkt_handler_flags(tp, th->th_flags, iptos); + cc_ecnpkt_handler_flags(tp, tcp_get_flags(th), iptos); } /* @@ -799,7 +799,7 @@ optlen = off - sizeof (struct tcphdr); optp = (u_char *)(th + 1); } - thflags = th->th_flags; + thflags = tcp_get_flags(th); /* * Convert TCP protocol specific fields to host format. @@ -1537,7 +1537,7 @@ struct tcphdr tcp_savetcp; short ostate = 0; #endif - thflags = th->th_flags; + thflags = tcp_get_flags(th); inc = &tp->t_inpcb->inp_inc; tp->sackhint.last_sack_ack = 0; sack_changed = 0; @@ -3176,7 +3176,7 @@ if (tp->t_fbyte_out && tp->t_fbyte_in) tp->t_flags2 |= TF2_FBYTES_COMPLETE; } - thflags = th->th_flags & TH_FIN; + thflags = tcp_get_flags(th) & TH_FIN; TCPSTAT_INC(tcps_rcvpack); TCPSTAT_ADD(tcps_rcvbyte, tlen); SOCKBUF_LOCK(&so->so_rcv); @@ -3401,7 +3401,7 @@ } /* Don't bother if destination was broadcast/multicast. */ - if ((th->th_flags & TH_RST) || m->m_flags & (M_BCAST|M_MCAST)) + if ((tcp_get_flags(th) & TH_RST) || m->m_flags & (M_BCAST|M_MCAST)) goto drop; #ifdef INET6 if (mtod(m, struct ip *)->ip_v == 6) { @@ -3431,13 +3431,13 @@ goto drop; /* tcp_respond consumes the mbuf chain. */ - if (th->th_flags & TH_ACK) { + if (tcp_get_flags(th) & TH_ACK) { tcp_respond(tp, mtod(m, void *), th, m, (tcp_seq)0, th->th_ack, TH_RST); } else { - if (th->th_flags & TH_SYN) + if (tcp_get_flags(th) & TH_SYN) tlen++; - if (th->th_flags & TH_FIN) + if (tcp_get_flags(th) & TH_FIN) tlen++; tcp_respond(tp, mtod(m, void *), th, m, th->th_seq+tlen, (tcp_seq)0, TH_RST|TH_ACK); Index: sys/netinet/tcp_lro.c =================================================================== --- sys/netinet/tcp_lro.c +++ sys/netinet/tcp_lro.c @@ -921,7 +921,7 @@ le->next_seq = ntohl(th->th_seq) + tcp_data_len; le->ack_seq = th->th_ack; le->window = th->th_win; - le->flags = (th->th_x2 << 8) | th->th_flags; + le->flags = tcp_get_flags(th); le->needs_merge = 0; /* Setup new data pointers. */ @@ -1033,7 +1033,7 @@ tcp_push_and_replace(lc, le, m); goto again; } - if ((((th->th_x2 << 8) | th->th_flags) & ~(TH_ACK | TH_PUSH)) != 0) { + if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH)) != 0) { /* * Make sure that previously seen segments/ACKs are delivered * before this segment, e.g. FIN. @@ -1077,7 +1077,7 @@ tcp_push_and_replace(lc, le, m); goto again; } - if ((((th->th_x2 << 8) | th->th_flags) & ~(TH_ACK | TH_PUSH)) != 0) { + if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH)) != 0) { tcp_push_and_replace(lc, le, m); goto again; } @@ -1093,7 +1093,7 @@ } /* Try to append the new segment. */ if (__predict_false(ntohl(th->th_seq) != le->next_seq || - ((th->th_flags & TH_ACK) != + ((tcp_get_flags(th) & TH_ACK) != (le->flags & TH_ACK)) || (tcp_data_len == 0 && le->ack_seq == th->th_ack && @@ -1265,14 +1265,14 @@ break; } /* For ACKCMP we only accept ACK, PUSH, ECE and CWR. */ - if ((((th->th_x2 << 8) | th->th_flags) & ~(TH_ACK | TH_PUSH | TH_ECE | TH_CWR)) != 0) + if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH | TH_ECE | TH_CWR)) != 0) ret = false; /* If it has data on it we cannot compress it */ if (m->m_pkthdr.lro_tcp_d_len) ret = false; /* ACK flag must be set. */ - if (!(th->th_flags & TH_ACK)) + if (!(tcp_get_flags(th) & TH_ACK)) ret = false; return (ret); } @@ -1576,7 +1576,7 @@ ae->flags = TSTMP_HDWR; ae->seq = ntohl(th->th_seq); ae->ack = ntohl(th->th_ack); - ae->flags |= (th->th_x2 << 8) | th->th_flags; + ae->flags |= tcp_get_flags(th); if (ts_ptr != NULL) { ae->ts_value = ntohl(ts_ptr[1]); ae->ts_echo = ntohl(ts_ptr[2]); @@ -1831,7 +1831,7 @@ th = pa->tcp; /* Don't process SYN packets. */ - if (__predict_false(th->th_flags & TH_SYN)) + if (__predict_false(tcp_get_flags(th) & TH_SYN)) return (TCP_LRO_CANNOT); /* Get total TCP header length and compute payload length. */ Index: sys/netinet/tcp_output.c =================================================================== --- sys/netinet/tcp_output.c +++ sys/netinet/tcp_output.c @@ -1287,7 +1287,7 @@ bcopy(opt, th + 1, optlen); th->th_off = (sizeof (struct tcphdr) + optlen) >> 2; } - th->th_flags = flags; + tcp_set_flags(th, flags); /* * Calculate receive window. Don't shrink window, * but avoid silly window syndrome. Index: sys/netinet/tcp_reass.c =================================================================== --- sys/netinet/tcp_reass.c +++ sys/netinet/tcp_reass.c @@ -332,7 +332,7 @@ last->tqe_len += tlen; last->tqe_m->m_pkthdr.len += tlen; /* Preserve the FIN bit if its there */ - last->tqe_flags |= (th->th_flags & TH_FIN); + last->tqe_flags |= (tcp_get_flags(th) & TH_FIN); last->tqe_last->m_next = m; last->tqe_last = mlast; last->tqe_mbuf_cnt += lenofoh; @@ -384,7 +384,7 @@ static void tcp_reass_replace(struct tcpcb *tp, struct tseg_qent *q, struct mbuf *m, - tcp_seq seq, int len, struct mbuf *mlast, int mbufoh, uint8_t flags) + tcp_seq seq, int len, struct mbuf *mlast, int mbufoh, uint16_t flags) { /* * Free the data in q, and replace @@ -564,7 +564,7 @@ /* * Check for zero length data. */ - if ((*tlenp == 0) && ((th->th_flags & TH_FIN) == 0)) { + if ((*tlenp == 0) && ((tcp_get_flags(th) & TH_FIN) == 0)) { /* * A zero length segment does no * one any good. We could check @@ -581,7 +581,7 @@ #endif return (0); } else if ((*tlenp == 0) && - (th->th_flags & TH_FIN) && + (tcp_get_flags(th) & TH_FIN) && !TCPS_HAVEESTABLISHED(tp->t_state)) { /* * We have not established, and we @@ -628,7 +628,7 @@ */ last = TAILQ_LAST_FAST(&tp->t_segq, tseg_qent, tqe_q); if (last != NULL) { - if ((th->th_flags & TH_FIN) && + if ((tcp_get_flags(th) & TH_FIN) && SEQ_LT((th->th_seq + *tlenp), (last->tqe_start + last->tqe_len))) { /* * Someone is trying to game us, dump @@ -915,7 +915,7 @@ #ifdef TCP_REASS_COUNTERS counter_u64_add(reass_path7, 1); #endif - tcp_reass_replace(tp, q, m, th->th_seq, *tlenp, mlast, lenofoh, th->th_flags); + tcp_reass_replace(tp, q, m, th->th_seq, *tlenp, mlast, lenofoh, tcp_get_flags(th)); } else { /* * We just need to prepend the data @@ -964,7 +964,7 @@ new_entry: if (th->th_seq == tp->rcv_nxt && TCPS_HAVEESTABLISHED(tp->t_state)) { tp->rcv_nxt += *tlenp; - flags = th->th_flags & TH_FIN; + flags = tcp_get_flags(th) & TH_FIN; TCPSTAT_INC(tcps_rcvoopack); TCPSTAT_ADD(tcps_rcvoobyte, *tlenp); SOCKBUF_LOCK(&so->so_rcv); @@ -1039,7 +1039,7 @@ TCPSTAT_ADD(tcps_rcvoobyte, *tlenp); /* Insert the new segment queue entry into place. */ te->tqe_m = m; - te->tqe_flags = th->th_flags; + te->tqe_flags = tcp_get_flags(th); te->tqe_len = *tlenp; te->tqe_start = th->th_seq; te->tqe_last = mlast; Index: sys/netinet/tcp_stacks/bbr.c =================================================================== --- sys/netinet/tcp_stacks/bbr.c +++ sys/netinet/tcp_stacks/bbr.c @@ -5908,7 +5908,7 @@ static void bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len, - uint32_t seq_out, uint8_t th_flags, int32_t err, uint32_t cts, + uint32_t seq_out, uint16_t th_flags, int32_t err, uint32_t cts, struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc, struct sockbuf *sb) { @@ -7338,7 +7338,7 @@ uint32_t p_maxseg, maxseg, p_acked = 0; INP_WLOCK_ASSERT(tp->t_inpcb); - if (th->th_flags & TH_RST) { + if (tcp_get_flags(th) & TH_RST) { /* We don't log resets */ return (0); } @@ -8283,7 +8283,7 @@ if (tp->t_fbyte_out && tp->t_fbyte_in) tp->t_flags2 |= TF2_FBYTES_COMPLETE; } - thflags = th->th_flags & TH_FIN; + thflags = tcp_get_flags(th) & TH_FIN; KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs); KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); SOCKBUF_LOCK(&so->so_rcv); @@ -11364,7 +11364,7 @@ /* add in our stats */ kern_prefetch(bbr, &prev_state); prev_state = 0; - thflags = th->th_flags; + thflags = tcp_get_flags(th); /* * If this is either a state-changing packet or current state isn't * established, we require a write lock on tcbinfo. Otherwise, we @@ -13451,7 +13451,7 @@ bcopy(opt, th + 1, optlen); th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; } - th->th_flags = flags; + tcp_set_flags(th, flags); /* * Calculate receive window. Don't shrink window, but avoid silly * window syndrome. Index: sys/netinet/tcp_stacks/rack.c =================================================================== --- sys/netinet/tcp_stacks/rack.c +++ sys/netinet/tcp_stacks/rack.c @@ -492,7 +492,7 @@ struct tcphdr *th, int entered_rec, int dup_ack_struck); static void rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len, - uint32_t seq_out, uint8_t th_flags, int32_t err, uint64_t ts, + uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t ts, struct rack_sendmap *hintrsm, uint16_t add_flags, struct mbuf *s_mb, uint32_t s_moff, int hw_tls); static void @@ -7418,7 +7418,7 @@ static void rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len, - uint32_t seq_out, uint8_t th_flags, int32_t err, uint64_t cts, + uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t cts, struct rack_sendmap *hintrsm, uint16_t add_flag, struct mbuf *s_mb, uint32_t s_moff, int hw_tls) { struct tcp_rack *rack; @@ -9653,7 +9653,7 @@ INP_WLOCK_ASSERT(tp->t_inpcb); - if (th->th_flags & TH_RST) { + if (tcp_get_flags(th) & TH_RST) { /* We don't log resets */ return; } @@ -10805,7 +10805,7 @@ if (tp->t_fbyte_out && tp->t_fbyte_in) tp->t_flags2 |= TF2_FBYTES_COMPLETE; } - thflags = th->th_flags & TH_FIN; + thflags = tcp_get_flags(th) & TH_FIN; KMOD_TCPSTAT_ADD(tcps_rcvpack, nsegs); KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); SOCKBUF_LOCK(&so->so_rcv); @@ -13414,7 +13414,7 @@ /* Now fill in the ports */ th->th_sport = tp->t_inpcb->inp_fport; th->th_dport = tp->t_inpcb->inp_lport; - th->th_flags = ae->flags & 0xff; + tcp_set_flags(th, ae->flags); /* Now do we have a timestamp option? */ if (ae->flags & HAS_TSTMP) { u_char *cp; @@ -14247,7 +14247,7 @@ ms_cts = tcp_tv_to_mssectick(tv); nsegs = m->m_pkthdr.lro_nsegs; counter_u64_add(rack_proc_non_comp_ack, 1); - thflags = th->th_flags; + thflags = tcp_get_flags(th); #ifdef TCP_ACCOUNTING sched_pin(); if (thflags & TH_ACK) @@ -14603,7 +14603,7 @@ } rack_clear_rate_sample(rack); if ((rack->forced_ack) && - ((th->th_flags & TH_RST) == 0)) { + ((tcp_get_flags(th) & TH_RST) == 0)) { rack_handle_probe_response(rack, tiwin, us_cts); } /* @@ -16011,7 +16011,7 @@ if ((rsm->r_flags & RACK_HAD_PUSH) && (len == (rsm->r_end - rsm->r_start))) flags |= TH_PUSH; - th->th_flags = flags; + tcp_set_flags(th, flags); th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale)); if (th->th_win == 0) { tp->t_sndzerowin++; @@ -16488,7 +16488,7 @@ sb_offset = tp->snd_max - tp->snd_una; th->th_seq = htonl(tp->snd_max); th->th_ack = htonl(tp->rcv_nxt); - th->th_flags = flags; + tcp_set_flags(th, flags); th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale)); if (th->th_win == 0) { tp->t_sndzerowin++; @@ -16519,7 +16519,7 @@ } if (rack->r_ctl.fsb.rfo_apply_push && (len == rack->r_ctl.fsb.left_to_send)) { - th->th_flags |= TH_PUSH; + tcp_set_flags(th, flags | TH_PUSH); add_flag |= RACK_HAD_PUSH; } if ((m->m_next == NULL) || (len <= 0)){ @@ -18661,7 +18661,7 @@ rack_seq = rsm->r_start; } th->th_ack = htonl(tp->rcv_nxt); - th->th_flags = flags; + tcp_set_flags(th, flags); /* * Calculate receive window. Don't shrink window, but avoid silly * window syndrome. Index: sys/netinet/tcp_subr.c =================================================================== --- sys/netinet/tcp_subr.c +++ sys/netinet/tcp_subr.c @@ -1686,9 +1686,8 @@ th->th_dport = inp->inp_fport; th->th_seq = 0; th->th_ack = 0; - th->th_x2 = 0; th->th_off = 5; - th->th_flags = 0; + tcp_set_flags(th, 0); th->th_win = 0; th->th_urp = 0; th->th_sum = 0; /* in_pseudo() is called later for ipv4 */ @@ -1746,7 +1745,7 @@ uint16_t port; int output_ret; #ifdef INVARIANTS - int thflags = th->th_flags; + int thflags = tcp_get_flags(th); #endif KASSERT(tp != NULL || m != NULL, ("tcp_respond: tp and m both NULL")); @@ -2016,9 +2015,8 @@ #endif nth->th_seq = htonl(seq); nth->th_ack = htonl(ack); - nth->th_x2 = 0; nth->th_off = (sizeof (struct tcphdr) + optlen) >> 2; - nth->th_flags = flags; + tcp_set_flags(nth, flags); if (tp != NULL) nth->th_win = htons((u_short) (win >> tp->rcv_scale)); else @@ -4061,7 +4059,7 @@ } sp = s + strlen(s); if (th) - sprintf(sp, " tcpflags 0x%b", th->th_flags, PRINT_TH_FLAGS); + sprintf(sp, " tcpflags 0x%b", tcp_get_flags(th), PRINT_TH_FLAGS); if (*(s + size - 1) != '\0') panic("%s: string too long", __func__); return (s); Index: sys/netinet/tcp_syncache.c =================================================================== --- sys/netinet/tcp_syncache.c +++ sys/netinet/tcp_syncache.c @@ -627,7 +627,7 @@ * Any RST to our SYN|ACK must not carry ACK, SYN or FIN flags. * See RFC 793 page 65, section SEGMENT ARRIVES. */ - if (th->th_flags & (TH_ACK|TH_SYN|TH_FIN)) { + if (tcp_get_flags(th) & (TH_ACK|TH_SYN|TH_FIN)) { if ((s = tcp_log_addrs(inc, th, NULL, NULL))) log(LOG_DEBUG, "%s; %s: Spurious RST with ACK, SYN or " "FIN flag set, segment ignored\n", s, __func__); @@ -1097,7 +1097,7 @@ bool locked; NET_EPOCH_ASSERT(); - KASSERT((th->th_flags & (TH_RST|TH_ACK|TH_SYN)) == TH_ACK, + KASSERT((tcp_get_flags(th) & (TH_RST|TH_ACK|TH_SYN)) == TH_ACK, ("%s: can handle only ACK", __func__)); if (syncache_cookiesonly()) { @@ -1426,7 +1426,7 @@ bool locked; INP_RLOCK_ASSERT(inp); /* listen socket */ - KASSERT((th->th_flags & (TH_RST|TH_ACK|TH_SYN)) == TH_SYN, + KASSERT((tcp_get_flags(th) & (TH_RST|TH_ACK|TH_SYN)) == TH_SYN, ("%s: unexpected tcp flags", __func__)); /* @@ -1579,7 +1579,7 @@ * Disable ECN if needed. */ if ((sc->sc_flags & SCF_ECN) && - ((th->th_flags & (TH_ECE|TH_CWR)) != (TH_ECE|TH_CWR))) { + ((tcp_get_flags(th) & (TH_ECE|TH_CWR)) != (TH_ECE|TH_CWR))) { sc->sc_flags &= ~SCF_ECN; } #ifdef MAC @@ -1743,7 +1743,7 @@ sc->sc_peer_mss = to->to_mss; /* peer mss may be zero */ if (ltflags & TF_NOOPT) sc->sc_flags |= SCF_NOOPT; - if (((th->th_flags & (TH_ECE|TH_CWR)) == (TH_ECE|TH_CWR)) && + if (((tcp_get_flags(th) & (TH_ECE|TH_CWR)) == (TH_ECE|TH_CWR)) && V_tcp_do_ecn) sc->sc_flags |= SCF_ECN; @@ -1935,15 +1935,14 @@ th->th_seq = htonl(sc->sc_iss + 1); th->th_ack = htonl(sc->sc_irs + 1); th->th_off = sizeof(struct tcphdr) >> 2; - th->th_x2 = 0; - th->th_flags = flags; th->th_win = htons(sc->sc_wnd); th->th_urp = 0; if ((flags & TH_SYN) && (sc->sc_flags & SCF_ECN)) { - th->th_flags |= TH_ECE; + flags |= TH_ECE; TCPSTAT_INC(tcps_ecn_shs); } + tcp_set_flags(th, flags); /* Tack on the TCP options. */ if ((sc->sc_flags & SCF_NOOPT) == 0) { Index: sys/netinet/tcp_timewait.c =================================================================== --- sys/netinet/tcp_timewait.c +++ sys/netinet/tcp_timewait.c @@ -409,7 +409,7 @@ if (tw == NULL) goto drop; - thflags = th->th_flags; + thflags = tcp_get_flags(th); #ifdef INVARIANTS if ((thflags & (TH_SYN | TH_ACK)) == TH_SYN) INP_RLOCK_ASSERT(inp); @@ -475,13 +475,13 @@ * Send RST if UDP port numbers don't match */ if (tw->t_port != m->m_pkthdr.tcp_tun_port) { - if (th->th_flags & TH_ACK) { + if (tcp_get_flags(th) & TH_ACK) { tcp_respond(NULL, mtod(m, void *), th, m, (tcp_seq)0, th->th_ack, TH_RST); } else { - if (th->th_flags & TH_SYN) + if (tcp_get_flags(th) & TH_SYN) tlen++; - if (th->th_flags & TH_FIN) + if (tcp_get_flags(th) & TH_FIN) tlen++; tcp_respond(NULL, mtod(m, void *), th, m, th->th_seq+tlen, (tcp_seq)0, TH_RST|TH_ACK); @@ -692,7 +692,7 @@ th->th_seq = htonl(tw->snd_nxt); th->th_ack = htonl(tw->rcv_nxt); th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; - th->th_flags = flags; + tcp_set_flags(th, flags); th->th_win = htons(tw->last_win); #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE) Index: sys/netinet/tcp_var.h =================================================================== --- sys/netinet/tcp_var.h +++ sys/netinet/tcp_var.h @@ -88,7 +88,7 @@ struct mbuf *tqe_last; /* last mbuf in chain */ tcp_seq tqe_start; /* TCP Sequence number start */ int tqe_len; /* TCP segment data length */ - uint32_t tqe_flags; /* The flags from the th->th_flags */ + uint32_t tqe_flags; /* The flags from tcp_get_flags() */ uint32_t tqe_mbuf_cnt; /* Count of mbuf overhead */ }; TAILQ_HEAD(tsegqe_head, tseg_qent); @@ -1258,6 +1258,19 @@ th->th_urp = htons(th->th_urp); } +static inline uint16_t +tcp_get_flags(const struct tcphdr *th) +{ + return (((uint16_t)th->th_x2 << 8) | th->th_flags); +} + +static inline void +tcp_set_flags(struct tcphdr *th, uint16_t flags) +{ + th->th_x2 = (flags >> 8) & 0x0f; + th->th_flags = flags & 0xff; +} + static inline void tcp_account_for_send(struct tcpcb *tp, uint32_t len, uint8_t is_rxt, uint8_t is_tlp, int hw_tls)