Differential D40229 Diff 122372 sys/netinet/tcp_stacks/rack.c

Changeset View

Standalone View

sys/netinet/tcp_stacks/rack.c

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 553 Lines • ▼ Show 20 Lines
static int		static int
rack_do_syn_recv(struct mbuf m, struct tcphdr th,		rack_do_syn_recv(struct mbuf m, struct tcphdr th,
struct socket so, struct tcpcb tp, struct tcpopt *to, int32_t drop_hdrlen,		struct socket so, struct tcpcb tp, struct tcpopt *to, int32_t drop_hdrlen,
int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);		int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
static int		static int
rack_do_syn_sent(struct mbuf m, struct tcphdr th,		rack_do_syn_sent(struct mbuf m, struct tcphdr th,
struct socket so, struct tcpcb tp, struct tcpopt *to, int32_t drop_hdrlen,		struct socket so, struct tcpcb tp, struct tcpopt *to, int32_t drop_hdrlen,
int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);		int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
static void rack_chk_http_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts);		static void rack_chk_req_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts);
struct rack_sendmap *		struct rack_sendmap *
tcp_rack_output(struct tcpcb tp, struct tcp_rack rack,		tcp_rack_output(struct tcpcb tp, struct tcp_rack rack,
uint32_t tsused);		uint32_t tsused);
static void tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt,		static void tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt,
uint32_t len, uint32_t us_tim, int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt);		uint32_t len, uint32_t us_tim, int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt);
static void		static void
tcp_rack_partialack(struct tcpcb *tp);		tcp_rack_partialack(struct tcpcb *tp);
static int		static int
▲ Show 20 Lines • Show All 1,391 Lines • ▼ Show 20 Lines	else if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh)
return (rack->r_ctl.rc_fixed_pacing_rate_ss);		return (rack->r_ctl.rc_fixed_pacing_rate_ss);
else		else
return (rack->r_ctl.rc_fixed_pacing_rate_ca);		return (rack->r_ctl.rc_fixed_pacing_rate_ca);
}		}

static void		static void
rack_log_hybrid_bw(struct tcp_rack *rack, uint32_t seq, uint64_t cbw, uint64_t tim,		rack_log_hybrid_bw(struct tcp_rack *rack, uint32_t seq, uint64_t cbw, uint64_t tim,
uint64_t data, uint8_t mod, uint16_t aux,		uint64_t data, uint8_t mod, uint16_t aux,
struct http_sendfile_track *cur)		struct tcp_sendfile_track *cur)
{		{
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
int do_log = 0;		int do_log = 0;

/*		/*
* The rate cap one is noisy and only should come out when normal BB logging		* The rate cap one is noisy and only should come out when normal BB logging
* is enabled, the other logs (not RATE_CAP and NOT CAP_CALC) only come out		* is enabled, the other logs (not RATE_CAP and NOT CAP_CALC) only come out
* once per chunk and make up the BBpoint that can be turned on by the client.		* once per chunk and make up the BBpoint that can be turned on by the client.
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines	if (cur != NULL) {
log.u_bbr.pkts_out = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff);		log.u_bbr.pkts_out = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff);
}		}
/* first_send = <lt_epoch \| epoch> */		/* first_send = <lt_epoch \| epoch> */
log.u_bbr.epoch = (uint32_t)(cur->first_send & 0x00000000ffffffff);		log.u_bbr.epoch = (uint32_t)(cur->first_send & 0x00000000ffffffff);
log.u_bbr.lt_epoch = (uint32_t)((cur->first_send >> 32) & 0x00000000ffffffff);		log.u_bbr.lt_epoch = (uint32_t)((cur->first_send >> 32) & 0x00000000ffffffff);
/* localtime = <delivered \| applimited>*/		/* localtime = <delivered \| applimited>*/
log.u_bbr.applimited = (uint32_t)(cur->localtime & 0x00000000ffffffff);		log.u_bbr.applimited = (uint32_t)(cur->localtime & 0x00000000ffffffff);
log.u_bbr.delivered = (uint32_t)((cur->localtime >> 32) & 0x00000000ffffffff);		log.u_bbr.delivered = (uint32_t)((cur->localtime >> 32) & 0x00000000ffffffff);
off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_http_info[0]);		off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
log.u_bbr.bbr_substate = (uint8_t)(off / sizeof(struct http_sendfile_track));		log.u_bbr.bbr_substate = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
log.u_bbr.flex4 = (uint32_t)(rack->rc_tp->t_sndbytes - cur->sent_at_fs);		log.u_bbr.flex4 = (uint32_t)(rack->rc_tp->t_sndbytes - cur->sent_at_fs);
log.u_bbr.flex5 = (uint32_t)(rack->rc_tp->t_snd_rxt_bytes - cur->rxt_at_fs);		log.u_bbr.flex5 = (uint32_t)(rack->rc_tp->t_snd_rxt_bytes - cur->rxt_at_fs);
log.u_bbr.flex7 = (uint16_t)cur->hybrid_flags;		log.u_bbr.flex7 = (uint16_t)cur->hybrid_flags;
} else {		} else {
log.u_bbr.flex7 = 0xffff;		log.u_bbr.flex7 = 0xffff;
log.u_bbr.cur_del_rate = 0xffffffffffffffff;		log.u_bbr.cur_del_rate = 0xffffffffffffffff;
}		}
/*		/*
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
if (rack->rc_catch_up && rack->rc_hybrid_mode &&		if (rack->rc_catch_up && rack->rc_hybrid_mode &&
(rack->r_ctl.rc_last_sft != NULL)) {		(rack->r_ctl.rc_last_sft != NULL)) {
/*		/*
* We have a dynamic cap. The original target		* We have a dynamic cap. The original target
* is in bw_rate_cap, but we need to look at		* is in bw_rate_cap, but we need to look at
* how long it is until we hit the deadline.		* how long it is until we hit the deadline.
*/		*/
struct http_sendfile_track *ent;		struct tcp_sendfile_track *ent;

ent = rack->r_ctl.rc_last_sft;		ent = rack->r_ctl.rc_last_sft;
microuptime(&tv);		microuptime(&tv);
timenow = tcp_tv_to_lusectick(&tv);		timenow = tcp_tv_to_lusectick(&tv);
if (timenow >= ent->deadline) {		if (timenow >= ent->deadline) {
/* No time left we do DGP only */		/* No time left we do DGP only */
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,		rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
0, 0, 0, HYBRID_LOG_OUTOFTIME, 0, ent);		0, 0, 0, HYBRID_LOG_OUTOFTIME, 0, ent);
Show All 10 Lines	if (timeleft < HPTS_MSEC_IN_SEC) {
return;		return;
}		}
/*		/*
* Now lets find the amount of data left to send.		* Now lets find the amount of data left to send.
*		*
* Now ideally we want to use the end_seq to figure out how much more		* Now ideally we want to use the end_seq to figure out how much more
* but it might not be possible (only if we have the TRACK_FG_COMP on the entry..		* but it might not be possible (only if we have the TRACK_FG_COMP on the entry..
*/		*/
if (ent->flags & TCP_HTTP_TRACK_FLG_COMP) {		if (ent->flags & TCP_TRK_TRACK_FLG_COMP) {
if (SEQ_GT(ent->end_seq, rack->rc_tp->snd_una))		if (SEQ_GT(ent->end_seq, rack->rc_tp->snd_una))
lenleft = ent->end_seq - rack->rc_tp->snd_una;		lenleft = ent->end_seq - rack->rc_tp->snd_una;
else {		else {
/* TSNH, we should catch it at the send */		/* TSNH, we should catch it at the send */
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,		rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
0, timeleft, 0, HYBRID_LOG_CAPERROR, 0, ent);		0, timeleft, 0, HYBRID_LOG_CAPERROR, 0, ent);
rack->r_ctl.bw_rate_cap = 0;		rack->r_ctl.bw_rate_cap = 0;
return;		return;
▲ Show 20 Lines • Show All 6,194 Lines • ▼ Show 20 Lines	if (len == 0) {
return;		return;
}		}
if (IN_FASTRECOVERY(tp->t_flags)) {		if (IN_FASTRECOVERY(tp->t_flags)) {
rack->r_ctl.rc_prr_out += len;		rack->r_ctl.rc_prr_out += len;
}		}
/* First question is it a retransmission or new? */		/* First question is it a retransmission or new? */
if (seq_out == snd_max) {		if (seq_out == snd_max) {
/* Its new */		/* Its new */
rack_chk_http_and_hybrid_on_out(rack, seq_out, len, cts);		rack_chk_req_and_hybrid_on_out(rack, seq_out, len, cts);
again:		again:
rsm = rack_alloc(rack);		rsm = rack_alloc(rack);
if (rsm == NULL) {		if (rsm == NULL) {
/*		/*
* Hmm out of memory and the tcb got destroyed while		* Hmm out of memory and the tcb got destroyed while
* we tried to wait.		* we tried to wait.
*/		*/
return;		return;
▲ Show 20 Lines • Show All 3,171 Lines • ▼ Show 20 Lines	if (tp->snd_max == tp->snd_una) {
rack->rc_dragged_bottom = 1;		rack->rc_dragged_bottom = 1;
rack_increase_bw_mul(rack, -1, 0, 0, 1);		rack_increase_bw_mul(rack, -1, 0, 0, 1);
}		}
}		}

#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
static void		static void
rack_log_hybrid(struct tcp_rack *rack, uint32_t seq,		rack_log_hybrid(struct tcp_rack *rack, uint32_t seq,
struct http_sendfile_track *cur, uint8_t mod, int line, int err)		struct tcp_sendfile_track *cur, uint8_t mod, int line, int err)
{		{
int do_log;		int do_log;

do_log = tcp_bblogging_on(rack->rc_tp);		do_log = tcp_bblogging_on(rack->rc_tp);
if (do_log == 0) {		if (do_log == 0) {
if ((do_log = tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING) )== 0)		if ((do_log = tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING) )== 0)
return;		return;
/* We only allow the three below with point logging on */		/* We only allow the three below with point logging on */
Show All 24 Lines	if (cur != NULL) {
log.u_bbr.rttProp = cur->timestamp;		log.u_bbr.rttProp = cur->timestamp;
log.u_bbr.cur_del_rate = cur->cspr;		log.u_bbr.cur_del_rate = cur->cspr;
log.u_bbr.bw_inuse = cur->start;		log.u_bbr.bw_inuse = cur->start;
log.u_bbr.applimited = (uint32_t)(cur->end & 0x00000000ffffffff);		log.u_bbr.applimited = (uint32_t)(cur->end & 0x00000000ffffffff);
log.u_bbr.delivered = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff) ;		log.u_bbr.delivered = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff) ;
log.u_bbr.epoch = (uint32_t)(cur->deadline & 0x00000000ffffffff);		log.u_bbr.epoch = (uint32_t)(cur->deadline & 0x00000000ffffffff);
log.u_bbr.lt_epoch = (uint32_t)((cur->deadline >> 32) & 0x00000000ffffffff) ;		log.u_bbr.lt_epoch = (uint32_t)((cur->deadline >> 32) & 0x00000000ffffffff) ;
log.u_bbr.bbr_state = 1;		log.u_bbr.bbr_state = 1;
off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_http_info[0]);		off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
log.u_bbr.use_lt_bw = (uint8_t)(off / sizeof(struct http_sendfile_track));		log.u_bbr.use_lt_bw = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
} else {		} else {
log.u_bbr.flex2 = err;		log.u_bbr.flex2 = err;
}		}
/*		/*
* Fill in flex7 to be CHD (catchup\|hybrid\|DGP)		* Fill in flex7 to be CHD (catchup\|hybrid\|DGP)
*/		*/
log.u_bbr.flex7 = rack->rc_catch_up;		log.u_bbr.flex7 = rack->rc_catch_up;
log.u_bbr.flex7 <<= 1;		log.u_bbr.flex7 <<= 1;
Show All 15 Lines	rack_log_hybrid(struct tcp_rack *rack, uint32_t seq,
}		}
}		}
#endif		#endif

#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
static void		static void
rack_set_dgp_hybrid_mode(struct tcp_rack *rack, tcp_seq seq, uint32_t len)		rack_set_dgp_hybrid_mode(struct tcp_rack *rack, tcp_seq seq, uint32_t len)
{		{
struct http_sendfile_track *rc_cur;		struct tcp_sendfile_track *rc_cur;
struct tcpcb *tp;		struct tcpcb *tp;
int err = 0;		int err = 0;

rc_cur = tcp_http_find_req_for_seq(rack->rc_tp, seq);		rc_cur = tcp_req_find_req_for_seq(rack->rc_tp, seq);
if (rc_cur == NULL) {		if (rc_cur == NULL) {
/* If not in the beginning what about the end piece */		/* If not in the beginning what about the end piece */
rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);		rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);
rc_cur = tcp_http_find_req_for_seq(rack->rc_tp, (seq + len - 1));		rc_cur = tcp_req_find_req_for_seq(rack->rc_tp, (seq + len - 1));
} else {		} else {
err = 12345;		err = 12345;
}		}
/* If we find no parameters we are in straight DGP mode */		/* If we find no parameters we are in straight DGP mode */
if(rc_cur == NULL) {		if(rc_cur == NULL) {
/* None found for this seq, just DGP for now */		/* None found for this seq, just DGP for now */
rack->r_ctl.client_suggested_maxseg = 0;		rack->r_ctl.client_suggested_maxseg = 0;
rack->rc_catch_up = 0;		rack->rc_catch_up = 0;
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines	rack_set_dgp_hybrid_mode(struct tcp_rack *rack, tcp_seq seq, uint32_t len)
}		}
rack_log_hybrid(rack, seq, rc_cur, HYBRID_LOG_RULES_APP, __LINE__, 0);		rack_log_hybrid(rack, seq, rc_cur, HYBRID_LOG_RULES_APP, __LINE__, 0);
/* Remember it for next time and for CU mode */		/* Remember it for next time and for CU mode */
rack->r_ctl.rc_last_sft = rc_cur;		rack->r_ctl.rc_last_sft = rc_cur;
}		}
#endif		#endif

static void		static void
rack_chk_http_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts)		rack_chk_req_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts)
{		{
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
struct http_sendfile_track *ent;		struct tcp_sendfile_track *ent;

ent = rack->r_ctl.rc_last_sft;		ent = rack->r_ctl.rc_last_sft;
if ((ent == NULL) \|\|		if ((ent == NULL) \|\|
(ent->flags == TCP_HTTP_TRACK_FLG_EMPTY) \|\|		(ent->flags == TCP_TRK_TRACK_FLG_EMPTY) \|\|
(SEQ_GEQ(seq, ent->end_seq))) {		(SEQ_GEQ(seq, ent->end_seq))) {
/* Time to update the track. */		/* Time to update the track. */
rack_set_dgp_hybrid_mode(rack, seq, len);		rack_set_dgp_hybrid_mode(rack, seq, len);
ent = rack->r_ctl.rc_last_sft;		ent = rack->r_ctl.rc_last_sft;
}		}
/* Out of all */		/* Out of all */
if (ent == NULL) {		if (ent == NULL) {
return;		return;
}		}
if (SEQ_LT(ent->end_seq, (seq + len))) {		if (SEQ_LT(ent->end_seq, (seq + len))) {
/*		/*
* This is the case where our end_seq guess		* This is the case where our end_seq guess
* was wrong. This is usually due to TLS having		* was wrong. This is usually due to TLS having
* more bytes then our guess. It could also be the		* more bytes then our guess. It could also be the
* case that the client sent in two requests closely		* case that the client sent in two requests closely
* and the SB is full of both so we are sending part		* and the SB is full of both so we are sending part
* of each (end\|beg). In such a case lets move this		* of each (end\|beg). In such a case lets move this
* guys end to match the end of this send. That		* guys end to match the end of this send. That
* way it will complete when all of it is acked.		* way it will complete when all of it is acked.
*/		*/
ent->end_seq = (seq + len);		ent->end_seq = (seq + len);
rack_log_hybrid_bw(rack, seq, len, 0, 0, HYBRID_LOG_EXTEND, 0, ent);		rack_log_hybrid_bw(rack, seq, len, 0, 0, HYBRID_LOG_EXTEND, 0, ent);
}		}
/* Now validate we have set the send time of this one */		/* Now validate we have set the send time of this one */
if ((ent->flags & TCP_HTTP_TRACK_FLG_FSND) == 0) {		if ((ent->flags & TCP_TRK_TRACK_FLG_FSND) == 0) {
ent->flags \|= TCP_HTTP_TRACK_FLG_FSND;		ent->flags \|= TCP_TRK_TRACK_FLG_FSND;
ent->first_send = cts;		ent->first_send = cts;
ent->sent_at_fs = rack->rc_tp->t_sndbytes;		ent->sent_at_fs = rack->rc_tp->t_sndbytes;
ent->rxt_at_fs = rack->rc_tp->t_snd_rxt_bytes;		ent->rxt_at_fs = rack->rc_tp->t_snd_rxt_bytes;
}		}
#endif		#endif
}		}

static void		static void
▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines	#endif
rsm = tqhash_next(rack->r_ctl.tqh, rsm);		rsm = tqhash_next(rack->r_ctl.tqh, rsm);
if (rsm == NULL)		if (rsm == NULL)
break;		break;
}		}
}		}

#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
static inline void		static inline void
rack_http_check_for_comp(struct tcp_rack *rack, tcp_seq th_ack)		rack_req_check_for_comp(struct tcp_rack *rack, tcp_seq th_ack)
{		{
struct http_sendfile_track *ent;		struct tcp_sendfile_track *ent;
int i;		int i;

if ((rack->rc_hybrid_mode == 0) &&		if ((rack->rc_hybrid_mode == 0) &&
(tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING) == 0)) {		(tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING) == 0)) {
/*		/*
* Just do normal completions hybrid pacing is not on		* Just do normal completions hybrid pacing is not on
* and CLDL is off as well.		* and CLDL is off as well.
*/		*/
tcp_http_check_for_comp(rack->rc_tp, th_ack);		tcp_req_check_for_comp(rack->rc_tp, th_ack);
return;		return;
}		}
/*		/*
* Originally I was just going to find the th_ack associated		* Originally I was just going to find the th_ack associated
* with an entry. But then I realized a large strech ack could		* with an entry. But then I realized a large strech ack could
* in theory ack two or more requests at once. So instead we		* in theory ack two or more requests at once. So instead we
* need to find all entries that are completed by th_ack not		* need to find all entries that are completed by th_ack not
* just a single entry and do our logging.		* just a single entry and do our logging.
*/		*/
ent = tcp_http_find_a_req_that_is_completed_by(rack->rc_tp, th_ack, &i);		ent = tcp_req_find_a_req_that_is_completed_by(rack->rc_tp, th_ack, &i);
while (ent != NULL) {		while (ent != NULL) {
/*		/*
* We may be doing hybrid pacing or CLDL and need more details possibly		* We may be doing hybrid pacing or CLDL and need more details possibly
* so we do it manually instead of calling		* so we do it manually instead of calling
* tcp_http_check_for_comp()		* tcp_req_check_for_comp()
*/		*/
uint64_t laa, tim, data, cbw, ftim;		uint64_t laa, tim, data, cbw, ftim;

/* Ok this ack frees it */		/* Ok this ack frees it */
rack_log_hybrid(rack, th_ack,		rack_log_hybrid(rack, th_ack,
ent, HYBRID_LOG_REQ_COMP, __LINE__, 0);		ent, HYBRID_LOG_REQ_COMP, __LINE__, 0);
/* calculate the time based on the ack arrival */		/* calculate the time based on the ack arrival */
data = ent->end - ent->start;		data = ent->end - ent->start;
laa = tcp_tv_to_lusectick(&rack->r_ctl.act_rcv_time);		laa = tcp_tv_to_lusectick(&rack->r_ctl.act_rcv_time);
if (ent->flags & TCP_HTTP_TRACK_FLG_FSND) {		if (ent->flags & TCP_TRK_TRACK_FLG_FSND) {
if (ent->first_send > ent->localtime)		if (ent->first_send > ent->localtime)
ftim = ent->first_send;		ftim = ent->first_send;
else		else
ftim = ent->localtime;		ftim = ent->localtime;
} else {		} else {
/* TSNH */		/* TSNH */
ftim = ent->localtime;		ftim = ent->localtime;
}		}
Show All 10 Lines	while (ent != NULL) {
/*		/*
* Check to see if we are freeing what we are pointing to send wise		* Check to see if we are freeing what we are pointing to send wise
* if so be sure to NULL the pointer so we know we are no longer		* if so be sure to NULL the pointer so we know we are no longer
* set to anything.		* set to anything.
*/		*/
if (ent == rack->r_ctl.rc_last_sft)		if (ent == rack->r_ctl.rc_last_sft)
rack->r_ctl.rc_last_sft = NULL;		rack->r_ctl.rc_last_sft = NULL;
/* Generate the log that the tcp_netflix call would have */		/* Generate the log that the tcp_netflix call would have */
tcp_http_log_req_info(rack->rc_tp, ent,		tcp_req_log_req_info(rack->rc_tp, ent,
i, TCP_HTTP_REQ_LOG_FREED, 0, 0);		i, TCP_TRK_REQ_LOG_FREED, 0, 0);
/* Free it and see if there is another one */		/* Free it and see if there is another one */
tcp_http_free_a_slot(rack->rc_tp, ent);		tcp_req_free_a_slot(rack->rc_tp, ent);
ent = tcp_http_find_a_req_that_is_completed_by(rack->rc_tp, th_ack, &i);		ent = tcp_req_find_a_req_that_is_completed_by(rack->rc_tp, th_ack, &i);
}		}
}		}
#endif		#endif


/*		/*
* Return value of 1, we do not need to call rack_process_data().		* Return value of 1, we do not need to call rack_process_data().
* return value of 0, rack_process_data can be called.		* return value of 0, rack_process_data can be called.
▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines	if (acked) {
rack->r_ctl.rc_tlp_cnt_out = 0;		rack->r_ctl.rc_tlp_cnt_out = 0;
/*		/*
* If it is the RXT timer we want to		* If it is the RXT timer we want to
* stop it, so we can restart a TLP.		* stop it, so we can restart a TLP.
*/		*/
if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)		if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);		rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
rack_http_check_for_comp(rack, th->th_ack);		rack_req_check_for_comp(rack, th->th_ack);
#endif		#endif
}		}
/*		/*
* If we have a timestamp reply, update smoothed round trip time. If		* If we have a timestamp reply, update smoothed round trip time. If
* no timestamp is present but transmit timer is running and timed		* no timestamp is present but transmit timer is running and timed
* sequence number was acked, update smoothed round trip time. Since		* sequence number was acked, update smoothed round trip time. Since
* we now have an rtt measurement, cancel the timer backoff (cf.,		* we now have an rtt measurement, cancel the timer backoff (cf.,
* Phil Karn's retransmit alg.). Recompute the initial retransmit		* Phil Karn's retransmit alg.). Recompute the initial retransmit
▲ Show 20 Lines • Show All 841 Lines • ▼ Show 20 Lines	if (acked) {
/*		/*
* If it is the RXT timer we want to		* If it is the RXT timer we want to
* stop it, so we can restart a TLP.		* stop it, so we can restart a TLP.
*/		*/
if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)		if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);		rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);

#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
rack_http_check_for_comp(rack, th->th_ack);		rack_req_check_for_comp(rack, th->th_ack);
#endif		#endif
}		}
/*		/*
* Let the congestion control algorithm update congestion control		* Let the congestion control algorithm update congestion control
* related information. This typically means increasing the		* related information. This typically means increasing the
* congestion window.		* congestion window.
*/		*/
if (tp->snd_wnd < ctf_outstanding(tp)) {		if (tp->snd_wnd < ctf_outstanding(tp)) {
▲ Show 20 Lines • Show All 2,570 Lines • ▼ Show 20 Lines	if (tcp_bblogging_on(rack->rc_tp)) {
struct timeval ltv;		struct timeval ltv;
char tcp_hdr_buf[60];		char tcp_hdr_buf[60];
struct tcphdr *th;		struct tcphdr *th;
struct timespec ts;		struct timespec ts;
uint32_t orig_snd_una;		uint32_t orig_snd_una;
uint8_t xx = 0;		uint8_t xx = 0;

#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
struct http_sendfile_track *http_req;		struct tcp_sendfile_track *tcp_req;

if (SEQ_GT(ae->ack, tp->snd_una)) {		if (SEQ_GT(ae->ack, tp->snd_una)) {
http_req = tcp_http_find_req_for_seq(tp, (ae->ack-1));		tcp_req = tcp_req_find_req_for_seq(tp, (ae->ack-1));
} else {		} else {
http_req = tcp_http_find_req_for_seq(tp, ae->ack);		tcp_req = tcp_req_find_req_for_seq(tp, ae->ack);
}		}
#endif		#endif
memset(&log.u_bbr, 0, sizeof(log.u_bbr));		memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);		log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
if (rack->rack_no_prr == 0)		if (rack->rack_no_prr == 0)
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;		log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
else		else
log.u_bbr.flex1 = 0;		log.u_bbr.flex1 = 0;
Show All 25 Lines	if (ae->flags & TSTMP_HDWR) {
ltv.tv_sec = ts.tv_sec;		ltv.tv_sec = ts.tv_sec;
ltv.tv_usec = ts.tv_nsec / 1000;		ltv.tv_usec = ts.tv_nsec / 1000;
log.u_bbr.flex5 = tcp_tv_to_usectick(&ltv);		log.u_bbr.flex5 = tcp_tv_to_usectick(&ltv);
}		}
log.u_bbr.timeStamp = tcp_get_usecs(&ltv);		log.u_bbr.timeStamp = tcp_get_usecs(&ltv);
/* Log the rcv time */		/* Log the rcv time */
log.u_bbr.delRate = ae->timestamp;		log.u_bbr.delRate = ae->timestamp;
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
log.u_bbr.applimited = tp->t_http_closed;		log.u_bbr.applimited = tp->t_tcpreq_closed;
log.u_bbr.applimited <<= 8;		log.u_bbr.applimited <<= 8;
log.u_bbr.applimited \|= tp->t_http_open;		log.u_bbr.applimited \|= tp->t_tcpreq_open;
log.u_bbr.applimited <<= 8;		log.u_bbr.applimited <<= 8;
log.u_bbr.applimited \|= tp->t_http_req;		log.u_bbr.applimited \|= tp->t_tcpreq_req;
if (http_req) {		if (tcp_req) {
/* Copy out any client req info */		/* Copy out any client req info */
/* seconds */		/* seconds */
log.u_bbr.pkt_epoch = (http_req->localtime / HPTS_USEC_IN_SEC);		log.u_bbr.pkt_epoch = (tcp_req->localtime / HPTS_USEC_IN_SEC);
/* useconds */		/* useconds */
log.u_bbr.delivered = (http_req->localtime % HPTS_USEC_IN_SEC);		log.u_bbr.delivered = (tcp_req->localtime % HPTS_USEC_IN_SEC);
log.u_bbr.rttProp = http_req->timestamp;		log.u_bbr.rttProp = tcp_req->timestamp;
log.u_bbr.cur_del_rate = http_req->start;		log.u_bbr.cur_del_rate = tcp_req->start;
if (http_req->flags & TCP_HTTP_TRACK_FLG_OPEN) {		if (tcp_req->flags & TCP_TRK_TRACK_FLG_OPEN) {
log.u_bbr.flex8 \|= 1;		log.u_bbr.flex8 \|= 1;
} else {		} else {
log.u_bbr.flex8 \|= 2;		log.u_bbr.flex8 \|= 2;
log.u_bbr.bw_inuse = http_req->end;		log.u_bbr.bw_inuse = tcp_req->end;
}		}
log.u_bbr.flex6 = http_req->start_seq;		log.u_bbr.flex6 = tcp_req->start_seq;
if (http_req->flags & TCP_HTTP_TRACK_FLG_COMP) {		if (tcp_req->flags & TCP_TRK_TRACK_FLG_COMP) {
log.u_bbr.flex8 \|= 4;		log.u_bbr.flex8 \|= 4;
log.u_bbr.epoch = http_req->end_seq;		log.u_bbr.epoch = tcp_req->end_seq;
}		}
}		}
#endif		#endif
memset(tcp_hdr_buf, 0, sizeof(tcp_hdr_buf));		memset(tcp_hdr_buf, 0, sizeof(tcp_hdr_buf));
th = (struct tcphdr *)tcp_hdr_buf;		th = (struct tcphdr *)tcp_hdr_buf;
th->th_seq = ae->seq;		th->th_seq = ae->seq;
th->th_ack = ae->ack;		th->th_ack = ae->ack;
th->th_win = ae->win;		th->th_win = ae->win;
▲ Show 20 Lines • Show All 371 Lines • ▼ Show 20 Lines	#endif
rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;		rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;
rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;		rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;
} else {		} else {
rack->r_ctl.act_rcv_time = *tv;		rack->r_ctl.act_rcv_time = *tv;
}		}
rack_process_to_cumack(tp, rack, ae->ack, cts, to,		rack_process_to_cumack(tp, rack, ae->ack, cts, to,
tcp_tv_to_lusectick(&rack->r_ctl.act_rcv_time));		tcp_tv_to_lusectick(&rack->r_ctl.act_rcv_time));
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
rack_http_check_for_comp(rack, high_seq);		rack_req_check_for_comp(rack, high_seq);
#endif		#endif
if (rack->rc_dsack_round_seen) {		if (rack->rc_dsack_round_seen) {
/* Is the dsack round over? */		/* Is the dsack round over? */
if (SEQ_GEQ(ae->ack, rack->r_ctl.dsack_round_end)) {		if (SEQ_GEQ(ae->ack, rack->r_ctl.dsack_round_end)) {
/* Yes it is */		/* Yes it is */
rack->rc_dsack_round_seen = 0;		rack->rc_dsack_round_seen = 0;
rack_log_dsack_event(rack, 3, __LINE__, 0, 0);		rack_log_dsack_event(rack, 3, __LINE__, 0, 0);
}		}
▲ Show 20 Lines • Show All 620 Lines • ▼ Show 20 Lines	if (bytes > sbavail(&tptosocket(tp)->so_snd)) {
tp->t_flags &= ~TF_GPUTINPROG;		tp->t_flags &= ~TF_GPUTINPROG;
}		}
}		}
high_seq = th->th_ack;		high_seq = th->th_ack;
if (tcp_bblogging_on(rack->rc_tp)) {		if (tcp_bblogging_on(rack->rc_tp)) {
union tcp_log_stackspecific log;		union tcp_log_stackspecific log;
struct timeval ltv;		struct timeval ltv;
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
struct http_sendfile_track *http_req;		struct tcp_sendfile_track *tcp_req;

if (SEQ_GT(th->th_ack, tp->snd_una)) {		if (SEQ_GT(th->th_ack, tp->snd_una)) {
http_req = tcp_http_find_req_for_seq(tp, (th->th_ack-1));		tcp_req = tcp_req_find_req_for_seq(tp, (th->th_ack-1));
} else {		} else {
http_req = tcp_http_find_req_for_seq(tp, th->th_ack);		tcp_req = tcp_req_find_req_for_seq(tp, th->th_ack);
}		}
#endif		#endif
memset(&log.u_bbr, 0, sizeof(log.u_bbr));		memset(&log.u_bbr, 0, sizeof(log.u_bbr));
log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);		log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
if (rack->rack_no_prr == 0)		if (rack->rack_no_prr == 0)
log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;		log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
else		else
log.u_bbr.flex1 = 0;		log.u_bbr.flex1 = 0;
Show All 24 Lines	if (m->m_flags & M_TSTMP) {
ltv.tv_sec = ts.tv_sec;		ltv.tv_sec = ts.tv_sec;
ltv.tv_usec = ts.tv_nsec / 1000;		ltv.tv_usec = ts.tv_nsec / 1000;
log.u_bbr.flex5 = tcp_tv_to_usectick(&ltv);		log.u_bbr.flex5 = tcp_tv_to_usectick(&ltv);
}		}
log.u_bbr.timeStamp = tcp_get_usecs(&ltv);		log.u_bbr.timeStamp = tcp_get_usecs(&ltv);
/* Log the rcv time */		/* Log the rcv time */
log.u_bbr.delRate = m->m_pkthdr.rcv_tstmp;		log.u_bbr.delRate = m->m_pkthdr.rcv_tstmp;
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
log.u_bbr.applimited = tp->t_http_closed;		log.u_bbr.applimited = tp->t_tcpreq_closed;
log.u_bbr.applimited <<= 8;		log.u_bbr.applimited <<= 8;
log.u_bbr.applimited \|= tp->t_http_open;		log.u_bbr.applimited \|= tp->t_tcpreq_open;
log.u_bbr.applimited <<= 8;		log.u_bbr.applimited <<= 8;
log.u_bbr.applimited \|= tp->t_http_req;		log.u_bbr.applimited \|= tp->t_tcpreq_req;
if (http_req) {		if (tcp_req) {
/* Copy out any client req info */		/* Copy out any client req info */
/* seconds */		/* seconds */
log.u_bbr.pkt_epoch = (http_req->localtime / HPTS_USEC_IN_SEC);		log.u_bbr.pkt_epoch = (tcp_req->localtime / HPTS_USEC_IN_SEC);
/* useconds */		/* useconds */
log.u_bbr.delivered = (http_req->localtime % HPTS_USEC_IN_SEC);		log.u_bbr.delivered = (tcp_req->localtime % HPTS_USEC_IN_SEC);
log.u_bbr.rttProp = http_req->timestamp;		log.u_bbr.rttProp = tcp_req->timestamp;
log.u_bbr.cur_del_rate = http_req->start;		log.u_bbr.cur_del_rate = tcp_req->start;
if (http_req->flags & TCP_HTTP_TRACK_FLG_OPEN) {		if (tcp_req->flags & TCP_TRK_TRACK_FLG_OPEN) {
log.u_bbr.flex8 \|= 1;		log.u_bbr.flex8 \|= 1;
} else {		} else {
log.u_bbr.flex8 \|= 2;		log.u_bbr.flex8 \|= 2;
log.u_bbr.bw_inuse = http_req->end;		log.u_bbr.bw_inuse = tcp_req->end;
}		}
log.u_bbr.flex6 = http_req->start_seq;		log.u_bbr.flex6 = tcp_req->start_seq;
if (http_req->flags & TCP_HTTP_TRACK_FLG_COMP) {		if (tcp_req->flags & TCP_TRK_TRACK_FLG_COMP) {
log.u_bbr.flex8 \|= 4;		log.u_bbr.flex8 \|= 4;
log.u_bbr.epoch = http_req->end_seq;		log.u_bbr.epoch = tcp_req->end_seq;
}		}
}		}
#endif		#endif
TCP_LOG_EVENTP(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0,		TCP_LOG_EVENTP(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0,
tlen, &log, true, &ltv);		tlen, &log, true, &ltv);
}		}
/* Remove ack required flag if set, we have one */		/* Remove ack required flag if set, we have one */
if (thflags & TH_ACK)		if (thflags & TH_ACK)
▲ Show 20 Lines • Show All 5,853 Lines • ▼ Show 20 Lines	rack_add_deferred_option(struct tcp_rack *rack, int sopt_name, uint64_t loptval)
TAILQ_INSERT_TAIL(&rack->r_ctl.opt_list, dol, next);		TAILQ_INSERT_TAIL(&rack->r_ctl.opt_list, dol, next);
return (1);		return (1);
}		}

static int		static int
process_hybrid_pacing(struct tcp_rack rack, struct tcp_hybrid_req hybrid)		process_hybrid_pacing(struct tcp_rack rack, struct tcp_hybrid_req hybrid)
{		{
#ifdef TCP_REQUEST_TRK		#ifdef TCP_REQUEST_TRK
struct http_sendfile_track *sft;		struct tcp_sendfile_track *sft;
struct timeval tv;		struct timeval tv;
tcp_seq seq;		tcp_seq seq;
int err;		int err;

microuptime(&tv);		microuptime(&tv);

/*		/*
* If BB logging is not on we need to look at the DTL flag.		* If BB logging is not on we need to look at the DTL flag.
Show All 9 Lines	if (tcp_bblogging_on(rack->rc_tp) == 0) {
}		}
}		}
/* Make sure no fixed rate is on */		/* Make sure no fixed rate is on */
rack->use_fixed_rate = 0;		rack->use_fixed_rate = 0;
rack->r_ctl.rc_fixed_pacing_rate_rec = 0;		rack->r_ctl.rc_fixed_pacing_rate_rec = 0;
rack->r_ctl.rc_fixed_pacing_rate_ca = 0;		rack->r_ctl.rc_fixed_pacing_rate_ca = 0;
rack->r_ctl.rc_fixed_pacing_rate_ss = 0;		rack->r_ctl.rc_fixed_pacing_rate_ss = 0;
/* Now allocate or find our entry that will have these settings */		/* Now allocate or find our entry that will have these settings */
sft = tcp_http_alloc_req_full(rack->rc_tp, &hybrid->req, tcp_tv_to_lusectick(&tv), 0);		sft = tcp_req_alloc_req_full(rack->rc_tp, &hybrid->req, tcp_tv_to_lusectick(&tv), 0);
if (sft == NULL) {		if (sft == NULL) {
rack->rc_tp->tcp_hybrid_error++;		rack->rc_tp->tcp_hybrid_error++;
/* no space, where would it have gone? */		/* no space, where would it have gone? */
seq = rack->rc_tp->snd_una + rack->rc_tp->t_inpcb.inp_socket->so_snd.sb_ccc;		seq = rack->rc_tp->snd_una + rack->rc_tp->t_inpcb.inp_socket->so_snd.sb_ccc;
rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_ROOM, __LINE__, 0);		rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_ROOM, __LINE__, 0);
return (ENOSPC);		return (ENOSPC);
}		}
/* The seq will be snd_una + everything in the buffer */		/* The seq will be snd_una + everything in the buffer */
▲ Show 20 Lines • Show All 1,477 Lines • Show Last 20 Lines