Page MenuHomeFreeBSD
Differential D33035 Diff 102444 sys/netinet/tcp_stacks/rack.c

Changeset View

Standalone View

View Options

sys/netinet/tcp_stacks/rack.c

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 12,852 Lines • ▼ Show 20 Lines#endif
* also change the values of t_srtt and t_rttvar, if * also change the values of t_srtt and t_rttvar, if
* they are non-zero. They are kept with a 5 * they are non-zero. They are kept with a 5
* bit decimal so we have to carefully convert * bit decimal so we have to carefully convert
* these to get the full precision. * these to get the full precision.
*/ */
rack_convert_rtts(tp); rack_convert_rtts(tp);
tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow); tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);
if (rack_do_hystart) { if (rack_do_hystart) {
struct sockopt sopt; tp->ccv->flags |= CCF_HYSTART_ALLOWED;
struct cc_newreno_opts opt; if (rack_do_hystart > 1)
rscheffUnsubmitted
Done
Inline Actions

another magic number here...

rscheff: another magic number here...
rrsAuthorUnsubmitted
Done
Inline Actions

sure I can get these turned into defines....

rrs: sure I can get these turned into defines....
tp->ccv->flags |= CCF_HYSTART_CAN_SH_CWND;
sopt.sopt_valsize = sizeof(struct cc_newreno_opts); if (rack_do_hystart > 2)
sopt.sopt_dir = SOPT_SET; tp->ccv->flags |= CCF_HYSTART_CONS_SSTH;
opt.name = CC_NEWRENO_ENABLE_HYSTART;
opt.val = rack_do_hystart;
if (CC_ALGO(tp)->ctl_output != NULL)
(void)CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
} }
if (rack_def_profile) if (rack_def_profile)
rack_set_profile(rack, rack_def_profile); rack_set_profile(rack, rack_def_profile);
/* Cancel the GP measurement in progress */ /* Cancel the GP measurement in progress */
tp->t_flags &= ~TF_GPUTINPROG; tp->t_flags &= ~TF_GPUTINPROG;
if (SEQ_GT(tp->snd_max, tp->iss)) if (SEQ_GT(tp->snd_max, tp->iss))
snt = tp->snd_max - tp->iss; snt = tp->snd_max - tp->iss;
else else
▲ Show 20 Lines • Show All 632 Lines • ▼ Show 20 Lines if (rack->probe_not_answered == 0) {
if (rack_apply_rtt_with_reduced_conf) { if (rack_apply_rtt_with_reduced_conf) {
rack_apply_updated_usrtt(rack, us_rtt, us_cts); rack_apply_updated_usrtt(rack, us_rtt, us_cts);
tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 0, NULL, 1); tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 0, NULL, 1);
} }
} }
} }
} }
static int static int
rack_do_compressed_ack_processing(struct tcpcb *tp, struct socket *so, struct mbuf *m, int nxt_pkt, struct timeval *tv) rack_do_compressed_ack_processing(struct tcpcb *tp, struct socket *so, struct mbuf *m, int nxt_pkt, struct timeval *tv)
{ {
/* /*
* Handle a "special" compressed ack mbuf. Each incoming * Handle a "special" compressed ack mbuf. Each incoming
* ack has only four possible dispositions: * ack has only four possible dispositions:
* *
* A) It moves the cum-ack forward * A) It moves the cum-ack forward
▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines#ifdef TCP_ACCOUNTING
/* Account for the acks */ /* Account for the acks */
if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((ae->ack - high_seq) + segsiz - 1) / segsiz); tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((ae->ack - high_seq) + segsiz - 1) / segsiz);
} }
counter_u64_add(tcp_cnt_counters[CNT_OF_ACKS_IN], counter_u64_add(tcp_cnt_counters[CNT_OF_ACKS_IN],
(((ae->ack - high_seq) + segsiz - 1) / segsiz)); (((ae->ack - high_seq) + segsiz - 1) / segsiz));
#endif #endif
high_seq = ae->ack; high_seq = ae->ack;
if (SEQ_GEQ(high_seq, rack->r_ctl.roundends)) { if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
union tcp_log_stackspecific log;
struct timeval tv;
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.flex1 = high_seq;
log.u_bbr.flex2 = rack->r_ctl.roundends;
log.u_bbr.flex3 = rack->r_ctl.current_round;
log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround;
log.u_bbr.flex8 = 8;
tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
0, &log, false, NULL, NULL, 0, &tv);
}
/*
* The draft (v3) calls for us to use SEQ_GEQ, but that
* causes issues when we are just going app limited. Lets
* instead use SEQ_GT <or> where its equal but more data
* is outstanding.
*/
if ((SEQ_GT(high_seq, rack->r_ctl.roundends)) ||
((high_seq == rack->r_ctl.roundends) &&
SEQ_GT(tp->snd_max, tp->snd_una))) {
rack->r_ctl.current_round++; rack->r_ctl.current_round++;
rack->r_ctl.roundends = tp->snd_max; rack->r_ctl.roundends = tp->snd_max;
if (CC_ALGO(tp)->newround != NULL) { if (CC_ALGO(tp)->newround != NULL) {
CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round); CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);
} }
} }
/* Setup our act_rcv_time */ /* Setup our act_rcv_time */
if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) { if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) {
▲ Show 20 Lines • Show All 411 Lines • ▼ Show 20 Lines#endif
int32_t thflags, retval, did_out = 0; int32_t thflags, retval, did_out = 0;
int32_t way_out = 0; int32_t way_out = 0;
/* /*
* cts - is the current time from tv (caller gets ts) in microseconds. * cts - is the current time from tv (caller gets ts) in microseconds.
* ms_cts - is the current time from tv in milliseconds. * ms_cts - is the current time from tv in milliseconds.
* us_cts - is the time that LRO or hardware actually got the packet in microseconds. * us_cts - is the time that LRO or hardware actually got the packet in microseconds.
*/ */
uint32_t cts, us_cts, ms_cts; uint32_t cts, us_cts, ms_cts;
uint32_t tiwin; uint32_t tiwin, high_seq;
struct timespec ts; struct timespec ts;
struct tcpopt to; struct tcpopt to;
struct tcp_rack *rack; struct tcp_rack *rack;
struct rack_sendmap *rsm; struct rack_sendmap *rsm;
int32_t prev_state = 0; int32_t prev_state = 0;
#ifdef TCP_ACCOUNTING #ifdef TCP_ACCOUNTING
int ack_val_set = 0xf; int ack_val_set = 0xf;
#endif #endif
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines if (bytes > sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
*/ */
rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/, rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
rack->r_ctl.rc_gp_srtt /*flex1*/, rack->r_ctl.rc_gp_srtt /*flex1*/,
tp->gput_seq, tp->gput_seq,
0, 0, 18, __LINE__, NULL, 0); 0, 0, 18, __LINE__, NULL, 0);
tp->t_flags &= ~TF_GPUTINPROG; tp->t_flags &= ~TF_GPUTINPROG;
} }
} }
high_seq = th->th_ack;
if (tp->t_logstate != TCP_LOG_STATE_OFF) { if (tp->t_logstate != TCP_LOG_STATE_OFF) {
union tcp_log_stackspecific log; union tcp_log_stackspecific log;
struct timeval ltv; struct timeval ltv;
#ifdef NETFLIX_HTTP_LOGGING #ifdef NETFLIX_HTTP_LOGGING
struct http_sendfile_track *http_req; struct http_sendfile_track *http_req;
if (SEQ_GT(th->th_ack, tp->snd_una)) { if (SEQ_GT(th->th_ack, tp->snd_una)) {
http_req = tcp_http_find_req_for_seq(tp, (th->th_ack-1)); http_req = tcp_http_find_req_for_seq(tp, (th->th_ack-1));
▲ Show 20 Lines • Show All 310 Lines • ▼ Show 20 Linesdo_output_now:
if (tcp_output(tp) < 0) if (tcp_output(tp) < 0)
return (1); return (1);
did_out = 1; did_out = 1;
} }
rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
rack_free_trim(rack); rack_free_trim(rack);
} }
/* Update any rounds needed */ /* Update any rounds needed */
if (SEQ_GEQ(tp->snd_una, rack->r_ctl.roundends)) { if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
union tcp_log_stackspecific log;
struct timeval tv;
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.flex1 = high_seq;
log.u_bbr.flex2 = rack->r_ctl.roundends;
log.u_bbr.flex3 = rack->r_ctl.current_round;
log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround;
log.u_bbr.flex8 = 9;
tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
0, &log, false, NULL, NULL, 0, &tv);
}
/*
* The draft (v3) calls for us to use SEQ_GEQ, but that
* causes issues when we are just going app limited. Lets
* instead use SEQ_GT <or> where its equal but more data
* is outstanding.
*/
if ((SEQ_GT(tp->snd_una, rack->r_ctl.roundends)) ||
((tp->snd_una == rack->r_ctl.roundends) && SEQ_GT(tp->snd_max, tp->snd_una))) {
rack->r_ctl.current_round++; rack->r_ctl.current_round++;
rack->r_ctl.roundends = tp->snd_max; rack->r_ctl.roundends = tp->snd_max;
if (CC_ALGO(tp)->newround != NULL) { if (CC_ALGO(tp)->newround != NULL) {
CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round); CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);
} }
} }
if ((nxt_pkt == 0) && if ((nxt_pkt == 0) &&
((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) && ((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
▲ Show 20 Lines • Show All 5,560 Lines • ▼ Show 20 Lines case TCP_RACK_MIN_TO:
break; break;
case TCP_RACK_EARLY_SEG: case TCP_RACK_EARLY_SEG:
/* If early recovery max segments */ /* If early recovery max segments */
RACK_OPTS_INC(tcp_rack_early_seg); RACK_OPTS_INC(tcp_rack_early_seg);
rack->r_ctl.rc_early_recovery_segs = optval; rack->r_ctl.rc_early_recovery_segs = optval;
break; break;
case TCP_RACK_ENABLE_HYSTART: case TCP_RACK_ENABLE_HYSTART:
{ {
struct sockopt sopt; if (optval) {
struct cc_newreno_opts opt; tp->ccv->flags |= CCF_HYSTART_ALLOWED;
if (rack_do_hystart > RACK_HYSTART_ON)
sopt.sopt_valsize = sizeof(struct cc_newreno_opts); tp->ccv->flags |= CCF_HYSTART_CAN_SH_CWND;
sopt.sopt_dir = SOPT_SET; if (rack_do_hystart > RACK_HYSTART_ON_W_SC)
opt.name = CC_NEWRENO_ENABLE_HYSTART; tp->ccv->flags |= CCF_HYSTART_CONS_SSTH;
opt.val = optval; } else {
if (CC_ALGO(tp)->ctl_output != NULL) tp->ccv->flags &= ~(CCF_HYSTART_ALLOWED|CCF_HYSTART_CAN_SH_CWND|CCF_HYSTART_CONS_SSTH);
error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
else
error = EINVAL;
} }
}
break; break;
case TCP_RACK_REORD_THRESH: case TCP_RACK_REORD_THRESH:
/* RACK reorder threshold (shift amount) */ /* RACK reorder threshold (shift amount) */
RACK_OPTS_INC(tcp_rack_reord_thresh); RACK_OPTS_INC(tcp_rack_reord_thresh);
if ((optval > 0) && (optval < 31)) if ((optval > 0) && (optval < 31))
rack->r_ctl.rc_reorder_shift = optval; rack->r_ctl.rc_reorder_shift = optval;
else else
error = EINVAL; error = EINVAL;
▲ Show 20 Lines • Show All 452 Lines • ▼ Show 20 Lines if (rack->rc_rack_tmr_std_based) {
optval |= 1; optval |= 1;
} }
if (rack->rc_rack_use_dsack) { if (rack->rc_rack_use_dsack) {
optval |= 2; optval |= 2;
} }
break; break;
case TCP_RACK_ENABLE_HYSTART: case TCP_RACK_ENABLE_HYSTART:
{ {
struct sockopt sopt; if (tp->ccv->flags & CCF_HYSTART_ALLOWED) {
struct cc_newreno_opts opt; optval = RACK_HYSTART_ON;
rscheffUnsubmitted
Done
Inline Actions

Can we not have hard coded magic numbers for optval?

rscheff: Can we not have hard coded magic numbers for optval?
rrsAuthorUnsubmitted
Done
Inline Actions

sure I can do that ;)

rrs: sure I can do that ;)
if (tp->ccv->flags & CCF_HYSTART_CAN_SH_CWND)
sopt.sopt_valsize = sizeof(struct cc_newreno_opts); optval = RACK_HYSTART_ON_W_SC;
sopt.sopt_dir = SOPT_GET; if (tp->ccv->flags & CCF_HYSTART_CONS_SSTH)
opt.name = CC_NEWRENO_ENABLE_HYSTART; optval = RACK_HYSTART_ON_W_SC_C;
if (CC_ALGO(tp)->ctl_output != NULL) } else {
error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt); optval = RACK_HYSTART_OFF;
else }
error = EINVAL;
optval = opt.val;
} }
break; break;
case TCP_FAST_RSM_HACK: case TCP_FAST_RSM_HACK:
optval = rack->fast_rsm_hack; optval = rack->fast_rsm_hack;
break; break;
case TCP_DEFER_OPTIONS: case TCP_DEFER_OPTIONS:
optval = rack->defer_options; optval = rack->defer_options;
break; break;
▲ Show 20 Lines • Show All 284 Lines • Show Last 20 Lines