Differential D33035 Diff 99238 sys/netinet/tcp_stacks/rack.c

Changeset View

Standalone View

sys/netinet/tcp_stacks/rack.c

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 12,847 Lines • ▼ Show 20 Lines	#endif
* also change the values of t_srtt and t_rttvar, if		* also change the values of t_srtt and t_rttvar, if
* they are non-zero. They are kept with a 5		* they are non-zero. They are kept with a 5
* bit decimal so we have to carefully convert		* bit decimal so we have to carefully convert
* these to get the full precision.		* these to get the full precision.
*/		*/
rack_convert_rtts(tp);		rack_convert_rtts(tp);
tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);		tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);
if (rack_do_hystart) {		if (rack_do_hystart) {
struct sockopt sopt;		tp->ccv->flags \|= CCF_HYSTART_ALLOWED;
struct cc_newreno_opts opt;		if (rack_do_hystart > 1)
		rscheffUnsubmitted Done Inline Actions another magic number here... rscheff: another magic number here...
		rrsAuthorUnsubmitted Done Inline Actions sure I can get these turned into defines.... rrs: sure I can get these turned into defines....
		tp->ccv->flags \|= CCF_HYSTART_CAN_SH_CWND;
sopt.sopt_valsize = sizeof(struct cc_newreno_opts);		if (rack_do_hystart > 2)
sopt.sopt_dir = SOPT_SET;		tp->ccv->flags \|= CCF_HYSTART_CONS_SSTH;
opt.name = CC_NEWRENO_ENABLE_HYSTART;
opt.val = rack_do_hystart;
if (CC_ALGO(tp)->ctl_output != NULL)
(void)CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
}		}
if (rack_def_profile)		if (rack_def_profile)
rack_set_profile(rack, rack_def_profile);		rack_set_profile(rack, rack_def_profile);
/* Cancel the GP measurement in progress */		/* Cancel the GP measurement in progress */
tp->t_flags &= ~TF_GPUTINPROG;		tp->t_flags &= ~TF_GPUTINPROG;
if (SEQ_GT(tp->snd_max, tp->iss))		if (SEQ_GT(tp->snd_max, tp->iss))
snt = tp->snd_max - tp->iss;		snt = tp->snd_max - tp->iss;
else		else
▲ Show 20 Lines • Show All 628 Lines • ▼ Show 20 Lines	if (rack->probe_not_answered == 0) {
if (rack_apply_rtt_with_reduced_conf) {		if (rack_apply_rtt_with_reduced_conf) {
rack_apply_updated_usrtt(rack, us_rtt, us_cts);		rack_apply_updated_usrtt(rack, us_rtt, us_cts);
tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 0, NULL, 1);		tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 0, NULL, 1);
}		}
}		}
}		}
}		}


static int		static int
rack_do_compressed_ack_processing(struct tcpcb tp, struct socket so, struct mbuf m, int nxt_pkt, struct timeval tv)		rack_do_compressed_ack_processing(struct tcpcb tp, struct socket so, struct mbuf m, int nxt_pkt, struct timeval tv)
{		{
/*		/*
* Handle a "special" compressed ack mbuf. Each incoming		* Handle a "special" compressed ack mbuf. Each incoming
* ack has only four possible dispositions:		* ack has only four possible dispositions:
*		*
* A) It moves the cum-ack forward		* A) It moves the cum-ack forward
▲ Show 20 Lines • Show All 253 Lines • ▼ Show 20 Lines	#ifdef TCP_ACCOUNTING
/* Account for the acks */		/* Account for the acks */
if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {		if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((ae->ack - high_seq) + segsiz - 1) / segsiz);		tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((ae->ack - high_seq) + segsiz - 1) / segsiz);
}		}
counter_u64_add(tcp_cnt_counters[CNT_OF_ACKS_IN],		counter_u64_add(tcp_cnt_counters[CNT_OF_ACKS_IN],
(((ae->ack - high_seq) + segsiz - 1) / segsiz));		(((ae->ack - high_seq) + segsiz - 1) / segsiz));
#endif		#endif
high_seq = ae->ack;		high_seq = ae->ack;
if (SEQ_GEQ(high_seq, rack->r_ctl.roundends)) {		if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
		union tcp_log_stackspecific log;
		struct timeval tv;

		memset(&log.u_bbr, 0, sizeof(log.u_bbr));
		log.u_bbr.timeStamp = tcp_get_usecs(&tv);
		log.u_bbr.flex1 = high_seq;
		log.u_bbr.flex2 = rack->r_ctl.roundends;
		log.u_bbr.flex3 = rack->r_ctl.current_round;
		log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround;
		log.u_bbr.flex8 = 8;
		tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
		0, &log, false, NULL, NULL, 0, &tv);
		}
		if (SEQ_GT(high_seq, rack->r_ctl.roundends)) {
rack->r_ctl.current_round++;		rack->r_ctl.current_round++;
rack->r_ctl.roundends = tp->snd_max;		rack->r_ctl.roundends = tp->snd_max;
if (CC_ALGO(tp)->newround != NULL) {		if (CC_ALGO(tp)->newround != NULL) {
CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);		CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);
}		}
}		}
/* Setup our act_rcv_time */		/* Setup our act_rcv_time */
if ((ae->flags & TSTMP_LRO) \|\| (ae->flags & TSTMP_HDWR)) {		if ((ae->flags & TSTMP_LRO) \|\| (ae->flags & TSTMP_HDWR)) {
▲ Show 20 Lines • Show All 407 Lines • ▼ Show 20 Lines	#endif
int32_t thflags, retval, did_out = 0;		int32_t thflags, retval, did_out = 0;
int32_t way_out = 0;		int32_t way_out = 0;
/*		/*
* cts - is the current time from tv (caller gets ts) in microseconds.		* cts - is the current time from tv (caller gets ts) in microseconds.
* ms_cts - is the current time from tv in milliseconds.		* ms_cts - is the current time from tv in milliseconds.
* us_cts - is the time that LRO or hardware actually got the packet in microseconds.		* us_cts - is the time that LRO or hardware actually got the packet in microseconds.
*/		*/
uint32_t cts, us_cts, ms_cts;		uint32_t cts, us_cts, ms_cts;
uint32_t tiwin;		uint32_t tiwin, high_seq;
struct timespec ts;		struct timespec ts;
struct tcpopt to;		struct tcpopt to;
struct tcp_rack *rack;		struct tcp_rack *rack;
struct rack_sendmap *rsm;		struct rack_sendmap *rsm;
int32_t prev_state = 0;		int32_t prev_state = 0;
#ifdef TCP_ACCOUNTING		#ifdef TCP_ACCOUNTING
int ack_val_set = 0xf;		int ack_val_set = 0xf;
#endif		#endif
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines	if (bytes > sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
*/		*/
rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /flex2/,		rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /flex2/,
rack->r_ctl.rc_gp_srtt /flex1/,		rack->r_ctl.rc_gp_srtt /flex1/,
tp->gput_seq,		tp->gput_seq,
0, 0, 18, __LINE__, NULL, 0);		0, 0, 18, __LINE__, NULL, 0);
tp->t_flags &= ~TF_GPUTINPROG;		tp->t_flags &= ~TF_GPUTINPROG;
}		}
}		}
		high_seq = th->th_ack;
if (tp->t_logstate != TCP_LOG_STATE_OFF) {		if (tp->t_logstate != TCP_LOG_STATE_OFF) {
union tcp_log_stackspecific log;		union tcp_log_stackspecific log;
struct timeval ltv;		struct timeval ltv;
#ifdef NETFLIX_HTTP_LOGGING		#ifdef NETFLIX_HTTP_LOGGING
struct http_sendfile_track *http_req;		struct http_sendfile_track *http_req;

if (SEQ_GT(th->th_ack, tp->snd_una)) {		if (SEQ_GT(th->th_ack, tp->snd_una)) {
http_req = tcp_http_find_req_for_seq(tp, (th->th_ack-1));		http_req = tcp_http_find_req_for_seq(tp, (th->th_ack-1));
▲ Show 20 Lines • Show All 340 Lines • ▼ Show 20 Lines
do_output_now:		do_output_now:
did_out = 1;		did_out = 1;
(void)tp->t_fb->tfb_tcp_output(tp);		(void)tp->t_fb->tfb_tcp_output(tp);
}		}
rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);		rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
rack_free_trim(rack);		rack_free_trim(rack);
}		}
/* Update any rounds needed */		/* Update any rounds needed */
if (SEQ_GEQ(tp->snd_una, rack->r_ctl.roundends)) {		if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
		union tcp_log_stackspecific log;
		struct timeval tv;

		memset(&log.u_bbr, 0, sizeof(log.u_bbr));
		log.u_bbr.timeStamp = tcp_get_usecs(&tv);
		log.u_bbr.flex1 = high_seq;
		log.u_bbr.flex2 = rack->r_ctl.roundends;
		log.u_bbr.flex3 = rack->r_ctl.current_round;
		log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround;
		log.u_bbr.flex8 = 9;
		tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
		0, &log, false, NULL, NULL, 0, &tv);
		}
		if (SEQ_GT(tp->snd_una, rack->r_ctl.roundends)) {
rack->r_ctl.current_round++;		rack->r_ctl.current_round++;
rack->r_ctl.roundends = tp->snd_max;		rack->r_ctl.roundends = tp->snd_max;
if (CC_ALGO(tp)->newround != NULL) {		if (CC_ALGO(tp)->newround != NULL) {
CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);		CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);
}		}
}		}
if ((nxt_pkt == 0) &&		if ((nxt_pkt == 0) &&
((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&		((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
▲ Show 20 Lines • Show All 5,564 Lines • ▼ Show 20 Lines	case TCP_RACK_MIN_TO:
break;		break;
case TCP_RACK_EARLY_SEG:		case TCP_RACK_EARLY_SEG:
/* If early recovery max segments */		/* If early recovery max segments */
RACK_OPTS_INC(tcp_rack_early_seg);		RACK_OPTS_INC(tcp_rack_early_seg);
rack->r_ctl.rc_early_recovery_segs = optval;		rack->r_ctl.rc_early_recovery_segs = optval;
break;		break;
case TCP_RACK_ENABLE_HYSTART:		case TCP_RACK_ENABLE_HYSTART:
{		{
struct sockopt sopt;		if (optval) {
struct cc_newreno_opts opt;		tp->ccv->flags \|= CCF_HYSTART_ALLOWED;
		if (rack_do_hystart > RACK_HYSTART_ON)
sopt.sopt_valsize = sizeof(struct cc_newreno_opts);		tp->ccv->flags \|= CCF_HYSTART_CAN_SH_CWND;
sopt.sopt_dir = SOPT_SET;		if (rack_do_hystart > RACK_HYSTART_ON_W_SC)
opt.name = CC_NEWRENO_ENABLE_HYSTART;		tp->ccv->flags \|= CCF_HYSTART_CONS_SSTH;
opt.val = optval;		} else {
if (CC_ALGO(tp)->ctl_output != NULL)		tp->ccv->flags &= ~(CCF_HYSTART_ALLOWED\|CCF_HYSTART_CAN_SH_CWND\|CCF_HYSTART_CONS_SSTH);
error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
else
error = EINVAL;
}		}
		}
break;		break;
case TCP_RACK_REORD_THRESH:		case TCP_RACK_REORD_THRESH:
/* RACK reorder threshold (shift amount) */		/* RACK reorder threshold (shift amount) */
RACK_OPTS_INC(tcp_rack_reord_thresh);		RACK_OPTS_INC(tcp_rack_reord_thresh);
if ((optval > 0) && (optval < 31))		if ((optval > 0) && (optval < 31))
rack->r_ctl.rc_reorder_shift = optval;		rack->r_ctl.rc_reorder_shift = optval;
else		else
error = EINVAL;		error = EINVAL;
▲ Show 20 Lines • Show All 430 Lines • ▼ Show 20 Lines	if (rack->rc_rack_tmr_std_based) {
optval \|= 1;		optval \|= 1;
}		}
if (rack->rc_rack_use_dsack) {		if (rack->rc_rack_use_dsack) {
optval \|= 2;		optval \|= 2;
}		}
break;		break;
case TCP_RACK_ENABLE_HYSTART:		case TCP_RACK_ENABLE_HYSTART:
{		{
struct sockopt sopt;		if (tp->ccv->flags & CCF_HYSTART_ALLOWED) {
struct cc_newreno_opts opt;		optval = RACK_HYSTART_ON;
		rscheffUnsubmitted Done Inline Actions Can we not have hard coded magic numbers for optval? rscheff: Can we not have hard coded magic numbers for optval?
		rrsAuthorUnsubmitted Done Inline Actions sure I can do that ;) rrs: sure I can do that ;)
		if (tp->ccv->flags & CCF_HYSTART_CAN_SH_CWND)
sopt.sopt_valsize = sizeof(struct cc_newreno_opts);		optval = RACK_HYSTART_ON_W_SC;
sopt.sopt_dir = SOPT_GET;		if (tp->ccv->flags & CCF_HYSTART_CONS_SSTH)
opt.name = CC_NEWRENO_ENABLE_HYSTART;		optval = RACK_HYSTART_ON_W_SC_C;
if (CC_ALGO(tp)->ctl_output != NULL)		} else {
error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);		optval = RACK_HYSTART_OFF;
else		}
error = EINVAL;
optval = opt.val;
}		}
break;		break;
case TCP_FAST_RSM_HACK:		case TCP_FAST_RSM_HACK:
optval = rack->fast_rsm_hack;		optval = rack->fast_rsm_hack;
break;		break;
case TCP_DEFER_OPTIONS:		case TCP_DEFER_OPTIONS:
optval = rack->defer_options;		optval = rack->defer_options;
break;		break;
▲ Show 20 Lines • Show All 293 Lines • Show Last 20 Lines