Index: sys/netinet/cc/cc.h =================================================================== --- sys/netinet/cc/cc.h +++ sys/netinet/cc/cc.h @@ -72,6 +72,15 @@ /* Define the new net.inet.tcp.cc sysctl tree. */ SYSCTL_DECL(_net_inet_tcp_cc); +/* For CC modules that use hystart++ */ +extern uint32_t hystart_lowcwnd; +extern uint32_t hystart_minrtt_thresh; +extern uint32_t hystart_maxrtt_thresh; +extern uint32_t hystart_n_rttsamples; +extern uint32_t hystart_css_growth_div; +extern uint32_t hystart_css_rounds; +extern uint32_t hystart_bblogs; + /* CC housekeeping functions. */ int cc_register_algo(struct cc_algo *add_cc); int cc_deregister_algo(struct cc_algo *remove_cc); Index: sys/netinet/cc/cc.c =================================================================== --- sys/netinet/cc/cc.c +++ sys/netinet/cc/cc.c @@ -84,6 +84,14 @@ #define CC_DEFAULT "newreno" #endif +uint32_t hystart_lowcwnd = 16; +uint32_t hystart_minrtt_thresh = 4000; +uint32_t hystart_maxrtt_thresh = 16000; +uint32_t hystart_n_rttsamples = 8; +uint32_t hystart_css_growth_div = 4; +uint32_t hystart_css_rounds = 5; +uint32_t hystart_bblogs = 0; + MALLOC_DEFINE(M_CC_MEM, "CC Mem", "Congestion Control State memory"); /* @@ -583,6 +591,45 @@ NULL, 0, cc_list_available, "A", "List available congestion control algorithms"); +SYSCTL_NODE(_net_inet_tcp_cc, OID_AUTO, hystartplusplus, + CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, + "New Reno related HyStart++ settings"); + +SYSCTL_UINT(_net_inet_tcp_cc_hystartplusplus, OID_AUTO, lowcwnd, + CTLFLAG_RW, + &hystart_lowcwnd, 16, + "The number of MSS in the CWND before HyStart++ is active"); + +SYSCTL_UINT(_net_inet_tcp_cc_hystartplusplus, OID_AUTO, minrtt_thresh, + CTLFLAG_RW, + &hystart_minrtt_thresh, 4000, + "HyStarts++ minimum RTT thresh used in clamp (in microseconds)"); + +SYSCTL_UINT(_net_inet_tcp_cc_hystartplusplus, OID_AUTO, maxrtt_thresh, + CTLFLAG_RW, + &hystart_maxrtt_thresh, 16000, + "HyStarts++ maximum RTT thresh used in clamp (in microseconds)"); + +SYSCTL_UINT(_net_inet_tcp_cc_hystartplusplus, OID_AUTO, n_rttsamples, + CTLFLAG_RW, + &hystart_n_rttsamples, 8, + "The number of RTT samples that must be seen to consider HyStart++"); + +SYSCTL_UINT(_net_inet_tcp_cc_hystartplusplus, OID_AUTO, css_growth_div, + CTLFLAG_RW, + &hystart_css_growth_div, 4, + "The divisor to the growth when in Hystart++ CSS"); + +SYSCTL_UINT(_net_inet_tcp_cc_hystartplusplus, OID_AUTO, css_rounds, + CTLFLAG_RW, + &hystart_css_rounds, 5, + "The number of rounds HyStart++ lasts in CSS before falling to CA"); + +SYSCTL_UINT(_net_inet_tcp_cc_hystartplusplus, OID_AUTO, bblogs, + CTLFLAG_RW, + &hystart_bblogs, 0, + "Do we enable HyStart++ Black Box logs to be generated if BB logging is on"); + VNET_DEFINE(int, cc_do_abe) = 0; SYSCTL_INT(_net_inet_tcp_cc, OID_AUTO, abe, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(cc_do_abe), 0, Index: sys/netinet/cc/cc_cubic.h =================================================================== --- sys/netinet/cc/cc_cubic.h +++ sys/netinet/cc/cc_cubic.h @@ -78,6 +78,57 @@ */ #define CUBED_ROOT_MAX_ULONG 448845 +/* Flags used in the cubic structure */ +#define CUBICFLAG_CONG_EVENT 0x00000001 /* congestion experienced */ +#define CUBICFLAG_IN_SLOWSTART 0x00000002 /* in slow start */ +#define CUBICFLAG_IN_APPLIMIT 0x00000004 /* application limited */ +#define CUBICFLAG_RTO_EVENT 0x00000008 /* RTO experienced */ +#define CUBICFLAG_HYSTART_ALLOWED 0x00000010 /* Hystart++ is allowed */ +#define CUBICFLAG_HYSTART_ENABLED 0x00000020 /* Hystart++ is enabled */ +#define CUBICFLAG_HYSTART_IN_CSS 0x00000040 /* We are in Hystart++ CSS */ +#define CUBICFLAG_HYSTART_CAN_SH_CWND 0x00000080 /* Can hystart when going CSS -> CA slam the cwnd */ +#define CUBICFLAG_HYSTART_CONS_SSTH 0x00000100 /* Should hystart use the more conservative sstrhesh */ +/* Kernel only bits */ +#ifdef _KERNEL +struct cubic { + /* Cubic K in fixed point form with CUBIC_SHIFT worth of precision. */ + int64_t K; + /* Sum of RTT samples across an epoch in ticks. */ + int64_t sum_rtt_ticks; + /* cwnd at the most recent congestion event. */ + unsigned long max_cwnd; + /* cwnd at the previous congestion event. */ + unsigned long prev_max_cwnd; + /* A copy of prev_max_cwnd. Used for CC_RTO_ERR */ + unsigned long prev_max_cwnd_cp; + /* various flags */ + uint32_t flags; + /* Minimum observed rtt in ticks. */ + int min_rtt_ticks; + /* Mean observed rtt between congestion epochs. */ + int mean_rtt_ticks; + /* ACKs since last congestion event. */ + int epoch_ack_count; + /* Timestamp (in ticks) of arriving in congestion avoidance from last + * congestion event. + */ + int t_last_cong; + /* Timestamp (in ticks) of a previous congestion event. Used for + * CC_RTO_ERR. + */ + int t_last_cong_prev; + uint32_t css_baseline_minrtt; + uint32_t css_current_round_minrtt; + uint32_t css_lastround_minrtt; + uint32_t css_rttsample_count; + uint32_t css_entered_at_round; + uint32_t css_current_round; + uint32_t css_fas_at_css_entry; + uint32_t css_lowrtt_fas; + uint32_t css_last_fas; +}; +#endif + /* Userland only bits. */ #ifndef _KERNEL Index: sys/netinet/cc/cc_cubic.c =================================================================== --- sys/netinet/cc/cc_cubic.c +++ sys/netinet/cc/cc_cubic.c @@ -70,6 +70,8 @@ #include #include #include +#include +#include #include #include #include @@ -85,39 +87,9 @@ static void cubic_ssthresh_update(struct cc_var *ccv, uint32_t maxseg); static void cubic_after_idle(struct cc_var *ccv); static size_t cubic_data_sz(void); - -struct cubic { - /* Cubic K in fixed point form with CUBIC_SHIFT worth of precision. */ - int64_t K; - /* Sum of RTT samples across an epoch in ticks. */ - int64_t sum_rtt_ticks; - /* cwnd at the most recent congestion event. */ - unsigned long max_cwnd; - /* cwnd at the previous congestion event. */ - unsigned long prev_max_cwnd; - /* A copy of prev_max_cwnd. Used for CC_RTO_ERR */ - unsigned long prev_max_cwnd_cp; - /* various flags */ - uint32_t flags; -#define CUBICFLAG_CONG_EVENT 0x00000001 /* congestion experienced */ -#define CUBICFLAG_IN_SLOWSTART 0x00000002 /* in slow start */ -#define CUBICFLAG_IN_APPLIMIT 0x00000004 /* application limited */ -#define CUBICFLAG_RTO_EVENT 0x00000008 /* RTO experienced */ - /* Minimum observed rtt in ticks. */ - int min_rtt_ticks; - /* Mean observed rtt between congestion epochs. */ - int mean_rtt_ticks; - /* ACKs since last congestion event. */ - int epoch_ack_count; - /* Timestamp (in ticks) of arriving in congestion avoidance from last - * congestion event. - */ - int t_last_cong; - /* Timestamp (in ticks) of a previous congestion event. Used for - * CC_RTO_ERR. - */ - int t_last_cong_prev; -}; +static void cubic_newround(struct cc_var *ccv, uint32_t round_cnt); +static void cubic_rttsample(struct cc_var *ccv, uint32_t usec_rtt, + uint32_t rxtcnt, uint32_t fas); struct cc_algo cubic_cc_algo = { .name = "cubic", @@ -129,9 +101,127 @@ .mod_init = cubic_mod_init, .post_recovery = cubic_post_recovery, .after_idle = cubic_after_idle, - .cc_data_sz = cubic_data_sz + .cc_data_sz = cubic_data_sz, + .rttsample = cubic_rttsample, + .newround = cubic_newround }; +static void +cubic_log_hystart_event(struct cc_var *ccv, struct cubic *cubicd, uint8_t mod, uint32_t flex1) +{ + /* + * Types of logs (mod value) + * 1 - rtt_thresh in flex1, checking to see if RTT is to great. + * 2 - rtt is too great, rtt_thresh in flex1. + * 3 - CSS is active incr in flex1 + * 4 - A new round is beginning flex1 is round count + * 5 - A new RTT measurement flex1 is the new measurement. + * 6 - We enter CA ssthresh is also in flex1. + * 7 - Socket option to change hystart executed opt.val in flex1. + * 8 - Back out of CSS into SS, flex1 is the css_baseline_minrtt + */ + struct tcpcb *tp; + + if (hystart_bblogs == 0) + return; + tp = ccv->ccvc.tcp; + if (tp->t_logstate != TCP_LOG_STATE_OFF) { + union tcp_log_stackspecific log; + struct timeval tv; + + memset(&log, 0, sizeof(log)); + log.u_bbr.flex1 = flex1; + log.u_bbr.flex2 = cubicd->css_current_round_minrtt; + log.u_bbr.flex3 = cubicd->css_lastround_minrtt; + log.u_bbr.flex4 = cubicd->css_rttsample_count; + log.u_bbr.flex5 = cubicd->css_entered_at_round; + log.u_bbr.flex6 = cubicd->css_baseline_minrtt; + /* We only need bottom 16 bits of flags */ + log.u_bbr.flex7 = cubicd->flags & 0x0000ffff; + log.u_bbr.flex8 = mod; + log.u_bbr.epoch = cubicd->css_current_round; + log.u_bbr.timeStamp = tcp_get_usecs(&tv); + log.u_bbr.lt_epoch = cubicd->css_fas_at_css_entry; + log.u_bbr.pkts_out = cubicd->css_last_fas; + log.u_bbr.delivered = cubicd->css_lowrtt_fas; + TCP_LOG_EVENTP(tp, NULL, + &tp->t_inpcb->inp_socket->so_rcv, + &tp->t_inpcb->inp_socket->so_snd, + TCP_HYSTART, 0, + 0, &log, false, &tv); + } +} + +static void +cubic_does_slow_start(struct cc_var *ccv, struct cubic *cubicd) +{ + /* + * In slow-start with ABC enabled and no RTO in sight? + * (Must not use abc_l_var > 1 if slow starting after + * an RTO. On RTO, snd_nxt = snd_una, so the + * snd_nxt == snd_max check is sufficient to + * handle this). + * + * XXXLAS: Find a way to signal SS after RTO that + * doesn't rely on tcpcb vars. + */ + u_int cw = CCV(ccv, snd_cwnd); + u_int incr = CCV(ccv, t_maxseg); + uint16_t abc_val; + + cubicd->flags |= CUBICFLAG_IN_SLOWSTART; + if (ccv->flags & CCF_USE_LOCAL_ABC) + abc_val = ccv->labc; + else + abc_val = V_tcp_abc_l_var; + if ((cubicd->flags & CUBICFLAG_HYSTART_ALLOWED) && + (cubicd->flags & CUBICFLAG_HYSTART_ENABLED) && + ((cubicd->flags & CUBICFLAG_HYSTART_IN_CSS) == 0)) { + /* + * Hystart is allowed and still enabled and we are not yet + * in CSS. Lets check to see if we can make a decision on + * if we need to go into CSS. + */ + if ((cubicd->css_rttsample_count >= hystart_n_rttsamples) && + (CCV(ccv, snd_cwnd) > + (hystart_lowcwnd * tcp_fixed_maxseg(ccv->ccvc.tcp)))) { + uint32_t rtt_thresh; + + /* Clamp (minrtt_thresh, lastround/8, maxrtt_thresh) */ + rtt_thresh = (cubicd->css_lastround_minrtt >> 3); + if (rtt_thresh < hystart_minrtt_thresh) + rtt_thresh = hystart_minrtt_thresh; + if (rtt_thresh > hystart_maxrtt_thresh) + rtt_thresh = hystart_maxrtt_thresh; + cubic_log_hystart_event(ccv, cubicd, 1, rtt_thresh); + if (cubicd->css_current_round_minrtt >= (cubicd->css_lastround_minrtt + rtt_thresh)) { + /* Enter CSS */ + cubicd->flags |= CUBICFLAG_HYSTART_IN_CSS; + cubicd->css_fas_at_css_entry = cubicd->css_lowrtt_fas; + cubicd->css_baseline_minrtt = cubicd->css_current_round_minrtt; + cubicd->css_entered_at_round = cubicd->css_current_round; + cubic_log_hystart_event(ccv, cubicd, 2, rtt_thresh); + } + } + } + if (CCV(ccv, snd_nxt) == CCV(ccv, snd_max)) + incr = min(ccv->bytes_this_ack, + ccv->nsegs * abc_val * + CCV(ccv, t_maxseg)); + else + incr = min(ccv->bytes_this_ack, CCV(ccv, t_maxseg)); + + /* Only if Hystart is enabled will the flag get set */ + if (cubicd->flags & CUBICFLAG_HYSTART_IN_CSS) { + incr /= hystart_css_growth_div; + cubic_log_hystart_event(ccv, cubicd, 3, incr); + } + /* ABC is on by default, so incr equals 0 frequently. */ + if (incr > 0) + CCV(ccv, snd_cwnd) = min((cw + incr), + TCP_MAXWIN << CCV(ccv, snd_scale)); +} + static void cubic_ack_received(struct cc_var *ccv, uint16_t type) { @@ -151,8 +241,7 @@ /* Use the logic in NewReno ack_received() for slow start. */ if (CCV(ccv, snd_cwnd) <= CCV(ccv, snd_ssthresh) || cubic_data->min_rtt_ticks == TCPTV_SRTTBASE) { - cubic_data->flags |= CUBICFLAG_IN_SLOWSTART; - newreno_cc_ack_received(ccv, type); + cubic_does_slow_start(ccv, cubic_data); } else { if ((cubic_data->flags & CUBICFLAG_RTO_EVENT) && (cubic_data->flags & CUBICFLAG_IN_SLOWSTART)) { @@ -495,5 +584,77 @@ CCV(ccv, snd_ssthresh) = max(ssthresh, 2 * maxseg); } +static void +cubic_rttsample(struct cc_var *ccv, uint32_t usec_rtt, uint32_t rxtcnt, uint32_t fas) +{ + struct cubic *cubicd; + + cubicd = ccv->cc_data; + if (rxtcnt > 1) { + /* + * Only look at RTT's that are non-ambiguous. + */ + return; + } + cubicd->css_rttsample_count++; + cubicd->css_last_fas = fas; + if (cubicd->css_current_round_minrtt > usec_rtt) { + cubicd->css_current_round_minrtt = usec_rtt; + cubicd->css_lowrtt_fas = cubicd->css_last_fas; + } + if ((cubicd->flags & CUBICFLAG_HYSTART_IN_CSS) && + (cubicd->css_rttsample_count >= hystart_n_rttsamples) && + (cubicd->css_baseline_minrtt > cubicd->css_current_round_minrtt)) { + /* + * We were in CSS and the RTT is now less, we + * entered CSS erroneously. + */ + cubicd->flags &= ~CUBICFLAG_HYSTART_IN_CSS; + cubic_log_hystart_event(ccv, cubicd, 8, cubicd->css_baseline_minrtt); + cubicd->css_baseline_minrtt = 0xffffffff; + } + cubic_log_hystart_event(ccv, cubicd, 5, usec_rtt); +} + +static void +cubic_newround(struct cc_var *ccv, uint32_t round_cnt) +{ + struct cubic *cubicd; + + cubicd = ccv->cc_data; + /* We have entered a new round */ + cubicd->css_lastround_minrtt = cubicd->css_current_round_minrtt; + cubicd->css_current_round_minrtt = 0xffffffff; + cubicd->css_rttsample_count = 0; + cubicd->css_current_round = round_cnt; + if ((cubicd->flags & CUBICFLAG_HYSTART_IN_CSS) && + ((round_cnt - cubicd->css_entered_at_round) >= hystart_css_rounds)) { + /* Enter CA */ + if (cubicd->flags & CUBICFLAG_HYSTART_CAN_SH_CWND) { + /* + * We engage more than snd_ssthresh, engage + * the brakes!! Though we will stay in SS to + * creep back up again, so lets leave CSS active + * and give us hystart_css_rounds more rounds. + */ + if (cubicd->flags & CUBICFLAG_HYSTART_CONS_SSTH) { + CCV(ccv, snd_ssthresh) = ((cubicd->css_lowrtt_fas + cubicd->css_fas_at_css_entry) / 2); + } else { + CCV(ccv, snd_ssthresh) = cubicd->css_lowrtt_fas; + } + CCV(ccv, snd_cwnd) = cubicd->css_fas_at_css_entry; + cubicd->css_entered_at_round = round_cnt; + } else { + CCV(ccv, snd_ssthresh) = CCV(ccv, snd_cwnd); + /* Turn off the CSS flag */ + cubicd->flags &= ~CUBICFLAG_HYSTART_IN_CSS; + /* Disable use of CSS in the future except long idle */ + cubicd->flags &= ~CUBICFLAG_HYSTART_ENABLED; + } + cubic_log_hystart_event(ccv, cubicd, 6, CCV(ccv, snd_ssthresh)); + } + cubic_log_hystart_event(ccv, cubicd, 4, round_cnt); +} + DECLARE_CC_MODULE(cubic, &cubic_cc_algo); MODULE_VERSION(cubic, 2); Index: sys/netinet/cc/cc_newreno.c =================================================================== --- sys/netinet/cc/cc_newreno.c +++ sys/netinet/cc/cc_newreno.c @@ -116,14 +116,6 @@ .cc_data_sz = newreno_data_sz, }; -static uint32_t hystart_lowcwnd = 16; -static uint32_t hystart_minrtt_thresh = 4000; -static uint32_t hystart_maxrtt_thresh = 16000; -static uint32_t hystart_n_rttsamples = 8; -static uint32_t hystart_css_growth_div = 4; -static uint32_t hystart_css_rounds = 5; -static uint32_t hystart_bblogs = 0; - static void newreno_log_hystart_event(struct cc_var *ccv, struct newreno *nreno, uint8_t mod, uint32_t flex1) { @@ -615,45 +607,5 @@ &VNET_NAME(newreno_beta_ecn), 3, &newreno_beta_handler, "IU", "New Reno beta ecn, specified as number between 1 and 100"); -SYSCTL_NODE(_net_inet_tcp_cc_newreno, OID_AUTO, hystartplusplus, - CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, - "New Reno related HyStart++ settings"); - -SYSCTL_UINT(_net_inet_tcp_cc_newreno_hystartplusplus, OID_AUTO, lowcwnd, - CTLFLAG_RW, - &hystart_lowcwnd, 16, - "The number of MSS in the CWND before HyStart++ is active"); - -SYSCTL_UINT(_net_inet_tcp_cc_newreno_hystartplusplus, OID_AUTO, minrtt_thresh, - CTLFLAG_RW, - &hystart_minrtt_thresh, 4000, - "HyStarts++ minimum RTT thresh used in clamp (in microseconds)"); - -SYSCTL_UINT(_net_inet_tcp_cc_newreno_hystartplusplus, OID_AUTO, maxrtt_thresh, - CTLFLAG_RW, - &hystart_maxrtt_thresh, 16000, - "HyStarts++ maximum RTT thresh used in clamp (in microseconds)"); - -SYSCTL_UINT(_net_inet_tcp_cc_newreno_hystartplusplus, OID_AUTO, n_rttsamples, - CTLFLAG_RW, - &hystart_n_rttsamples, 8, - "The number of RTT samples that must be seen to consider HyStart++"); - -SYSCTL_UINT(_net_inet_tcp_cc_newreno_hystartplusplus, OID_AUTO, css_growth_div, - CTLFLAG_RW, - &hystart_css_growth_div, 4, - "The divisor to the growth when in Hystart++ CSS"); - -SYSCTL_UINT(_net_inet_tcp_cc_newreno_hystartplusplus, OID_AUTO, css_rounds, - CTLFLAG_RW, - &hystart_css_rounds, 5, - "The number of rounds HyStart++ lasts in CSS before falling to CA"); - -SYSCTL_UINT(_net_inet_tcp_cc_newreno_hystartplusplus, OID_AUTO, bblogs, - CTLFLAG_RW, - &hystart_bblogs, 0, - "Do we enable HyStart++ Black Box logs to be generated if BB logging is on"); - - DECLARE_CC_MODULE(newreno, &newreno_cc_algo); MODULE_VERSION(newreno, 2);