Index: sys/netinet/tcp_hpts.h =================================================================== --- sys/netinet/tcp_hpts.h +++ sys/netinet/tcp_hpts.h @@ -201,6 +201,13 @@ #define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__) void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line); +static inline bool +tcp_in_hpts(struct inpcb *inp) +{ + + return (inp->inp_in_hpts > 0); +} + /* * To insert a TCB on the hpts you *must* be holding the * INP_WLOCK(). The hpts insert code will then acqurire Index: sys/netinet/tcp_hpts.c =================================================================== --- sys/netinet/tcp_hpts.c +++ sys/netinet/tcp_hpts.c @@ -55,7 +55,7 @@ * its top a check to prevent unwanted calls (from user land or the * arrival of incoming ack's). So it would add something like: * - * if (inp->inp_in_hpts) + * if (tcp_in_hpts(inp)) * return; * * to prevent output processing until the time alotted has gone by. Index: sys/netinet/tcp_stacks/bbr.c =================================================================== --- sys/netinet/tcp_stacks/bbr.c +++ sys/netinet/tcp_stacks/bbr.c @@ -747,7 +747,7 @@ uint32_t prev_delay = 0; inp = tp->t_inpcb; - if (inp->inp_in_hpts) { + if (tcp_in_hpts(inp)) { /* A previous call is already set up */ return; } @@ -1058,7 +1058,7 @@ */ wrong_timer: if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) { - if (inp->inp_in_hpts) + if (tcp_in_hpts(inp)) tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); bbr_timer_cancel(bbr, __LINE__, cts); bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val, @@ -1883,7 +1883,7 @@ l->lt_epoch = bbr->r_ctl.rc_lt_epoch; l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain; l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain; - l->inhpts = bbr->rc_inp->inp_in_hpts; + l->inhpts = tcp_in_hpts(bbr->rc_inp); l->ininput = bbr->rc_inp->inp_in_input; l->use_lt_bw = bbr->rc_lt_use_bw; l->pkts_out = bbr->r_ctl.rc_flight_at_input; @@ -3969,7 +3969,7 @@ bbr->rc_tlp_rtx_out = 0; bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate; tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); - if (bbr->rc_inp->inp_in_hpts && + if (tcp_in_hpts(bbr->rc_inp) && ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) { /* * When we enter recovery, we need to restart @@ -5264,7 +5264,7 @@ if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { uint8_t hpts_removed = 0; - if (bbr->rc_inp->inp_in_hpts && + if (tcp_in_hpts(bbr->rc_inp) && (bbr->rc_timer_first == 1)) { /* * If we are canceling timer's when we have the @@ -7972,7 +7972,7 @@ * don't want to transfer forward the time * for our sum's calculations. */ - if (bbr->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(bbr->rc_inp)) { tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); bbr->rc_timer_first = 0; bbr->r_ctl.rc_hpts_flags = 0; @@ -11653,7 +11653,7 @@ */ if ((tp->snd_max == tp->snd_una) && ((tp->t_flags & TF_DELACK) == 0) && - (bbr->rc_inp->inp_in_hpts) && + (tcp_in_hpts(bbr->rc_inp)) && (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { /* * keep alive not needed if we are hptsi @@ -11661,7 +11661,7 @@ */ ; } else { - if (bbr->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(bbr->rc_inp)) { tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && (TSTMP_GT(lcts, bbr->rc_pacer_started))) { @@ -12011,7 +12011,7 @@ } #endif if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && - inp->inp_in_hpts) { + tcp_in_hpts(inp)) { /* * We are on the hpts for some timer but not hptsi output. * Possibly remove from the hpts so we can send/recv etc. @@ -12056,9 +12056,9 @@ if ((bbr->r_timer_override) || (tp->t_state < TCPS_ESTABLISHED)) { /* Timeouts or early states are exempt */ - if (inp->inp_in_hpts) + if (tcp_in_hpts(inp)) tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); - } else if (inp->inp_in_hpts) { + } else if (tcp_in_hpts(inp)) { if ((bbr->r_ctl.rc_last_delay_val) && (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && delay_calc) { Index: sys/netinet/tcp_stacks/rack.c =================================================================== --- sys/netinet/tcp_stacks/rack.c +++ sys/netinet/tcp_stacks/rack.c @@ -2294,7 +2294,7 @@ log.u_bbr.flex5 = rsm->r_start; log.u_bbr.flex6 = rsm->r_end; log.u_bbr.flex8 = mod; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); @@ -2329,7 +2329,7 @@ log.u_bbr.pkts_out = 0; else log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); @@ -2354,7 +2354,7 @@ struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex8 = to_num; log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt; @@ -2393,7 +2393,7 @@ memset(&log.u_bbr, 0, sizeof(log.u_bbr)); log.u_bbr.flex8 = flag; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.cur_del_rate = (uint64_t)prev; log.u_bbr.delRate = (uint64_t)rsm; @@ -2438,7 +2438,7 @@ union tcp_log_stackspecific log; struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = t; log.u_bbr.flex2 = len; @@ -2588,7 +2588,7 @@ struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = line; log.u_bbr.flex2 = tick; @@ -2615,7 +2615,7 @@ union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = slot; if (rack->rack_no_prr) @@ -2661,7 +2661,7 @@ log.u_bbr.flex7 <<= 1; log.u_bbr.flex7 |= rack->r_wanted_output; /* Do we want output */ log.u_bbr.flex8 = rack->rc_in_persist; - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); log.u_bbr.use_lt_bw = rack->r_ent_rec_ns; @@ -2717,7 +2717,7 @@ struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = slot; log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags; @@ -2750,7 +2750,7 @@ union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = line; log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to; @@ -4466,7 +4466,7 @@ rack->r_ctl.rc_app_limited_cnt, 0, 0, 10, __LINE__, NULL, quality); } - if (rack->rc_inp->inp_in_hpts && + if (tcp_in_hpts(rack->rc_inp) && (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { /* * Ok we can't trust the pacer in this case @@ -5678,7 +5678,7 @@ static void rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) { - if (rack->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(rack->rc_inp)) { tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT); rack->r_ctl.rc_hpts_flags = 0; } @@ -5811,7 +5811,7 @@ (tp->t_state == TCPS_LISTEN)) { return; } - if (inp->inp_in_hpts) { + if (tcp_in_hpts(inp)) { /* Already on the pacer */ return; } @@ -7244,7 +7244,7 @@ } if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK; - if (rack->rc_inp->inp_in_hpts && + if (tcp_in_hpts(rack->rc_inp) && ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) { /* * Canceling timer's when we have no output being @@ -9983,7 +9983,7 @@ (entered_recovery == 0)) { rack_update_prr(tp, rack, changed, th_ack); if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) && - ((rack->rc_inp->inp_in_hpts == 0) && + ((tcp_in_hpts(rack->rc_inp) == 0) && ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) { /* * If you are pacing output you don't want @@ -13232,7 +13232,7 @@ * We will force the hpts to be stopped if any, and restart * with the slot set to what was in the saved slot. */ - if (rack->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(rack->rc_inp)) { if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { uint32_t us_cts; @@ -13328,7 +13328,7 @@ } #endif memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr == 0) log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; @@ -14320,7 +14320,7 @@ } #endif memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr == 0) log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; @@ -14679,13 +14679,13 @@ /* We could not send (probably in the hpts but stopped the timer earlier)? */ if ((tp->snd_max == tp->snd_una) && ((tp->t_flags & TF_DELACK) == 0) && - (rack->rc_inp->inp_in_hpts) && + (tcp_in_hpts(rack->rc_inp)) && (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { /* keep alive not needed if we are hptsi output yet */ ; } else { int late = 0; - if (rack->rc_inp->inp_in_hpts) { + if (tcp_in_hpts(rack->rc_inp)) { if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { us_cts = tcp_get_usecs(NULL); if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) { @@ -15611,7 +15611,7 @@ struct timeval tv; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; log.u_bbr.flex1 = error; log.u_bbr.flex2 = flags; @@ -16127,7 +16127,7 @@ union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr) log.u_bbr.flex1 = 0; @@ -16628,7 +16628,7 @@ union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr) log.u_bbr.flex1 = 0; @@ -16867,7 +16867,7 @@ cts = tcp_get_usecs(&tv); ms_cts = tcp_tv_to_mssectick(&tv); if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && - rack->rc_inp->inp_in_hpts) { + tcp_in_hpts(rack->rc_inp)) { /* * We are on the hpts for some timer but not hptsi output. * Remove from the hpts unconditionally. @@ -16893,7 +16893,7 @@ } } if (rack->rc_in_persist) { - if (rack->rc_inp->inp_in_hpts == 0) { + if (tcp_in_hpts(rack->rc_inp) == 0) { /* Timer is not running */ rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); } @@ -16907,9 +16907,9 @@ (delayed) || (tp->t_state < TCPS_ESTABLISHED)) { rack->rc_ack_can_sendout_data = 0; - if (rack->rc_inp->inp_in_hpts) + if (tcp_in_hpts(rack->rc_inp)) tcp_hpts_remove(rack->rc_inp, HPTS_REMOVE_OUTPUT); - } else if (rack->rc_inp->inp_in_hpts) { + } else if (tcp_in_hpts(rack->rc_inp)) { /* * On the hpts you can't pass even if ACKNOW is on, we will * when the hpts fires. @@ -18800,7 +18800,7 @@ union tcp_log_stackspecific log; memset(&log.u_bbr, 0, sizeof(log.u_bbr)); - log.u_bbr.inhpts = rack->rc_inp->inp_in_hpts; + log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); log.u_bbr.ininput = rack->rc_inp->inp_in_input; if (rack->rack_no_prr) log.u_bbr.flex1 = 0;