Differential D39210 Diff 119274 sys/netinet/tcp_stacks/bbr.c

Changeset View

Standalone View

sys/netinet/tcp_stacks/bbr.c

This file is larger than 256 KB, so syntax highlighting is disabled by default.

Show First 20 Lines • Show All 494 Lines • ▼ Show 20 Lines
static void		static void
bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot,		bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot,
uint32_t del_by, uint32_t cts, uint32_t sloton,		uint32_t del_by, uint32_t cts, uint32_t sloton,
uint32_t prev_delay);		uint32_t prev_delay);
static void		static void
bbr_enter_persist(struct tcpcb tp, struct tcp_bbr bbr, uint32_t cts,		bbr_enter_persist(struct tcpcb tp, struct tcp_bbr bbr, uint32_t cts,
int32_t line);		int32_t line);
static void		static void
bbr_stop_all_timers(struct tcpcb *tp);		bbr_stop_all_timers(struct tcpcb tp, struct tcp_bbr bbr);
static void		static void
bbr_exit_probe_rtt(struct tcpcb tp, struct tcp_bbr bbr, uint32_t cts);		bbr_exit_probe_rtt(struct tcpcb tp, struct tcp_bbr bbr, uint32_t cts);
static void		static void
bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts);		bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts);
static void		static void
bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts);		bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts);
static void		static void
bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,		bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
▲ Show 20 Lines • Show All 1,453 Lines • ▼ Show 20 Lines	TCP_LOG_EVENTP(bbr->rc_tp, NULL,
BBR_LOG_ENTREC, 0,		BBR_LOG_ENTREC, 0,
0, &log, false, &bbr->rc_tv);		0, &log, false, &bbr->rc_tv);
}		}
}		}

static void		static void
bbr_log_msgsize_fail(struct tcp_bbr bbr, struct tcpcb tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts)		bbr_log_msgsize_fail(struct tcp_bbr bbr, struct tcpcb tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts)
{		{
if (tcp_bblogging_on(bbr->rc_tp)) {		if (tcp_bblogging_on(tp)) {
union tcp_log_stackspecific log;		union tcp_log_stackspecific log;

bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
log.u_bbr.flex1 = tso;		log.u_bbr.flex1 = tso;
log.u_bbr.flex2 = maxseg;		log.u_bbr.flex2 = maxseg;
log.u_bbr.flex3 = mtu;		log.u_bbr.flex3 = mtu;
log.u_bbr.flex4 = csum_flags;		log.u_bbr.flex4 = csum_flags;
TCP_LOG_EVENTP(tp, NULL,		TCP_LOG_EVENTP(tp, NULL,
▲ Show 20 Lines • Show All 682 Lines • ▼ Show 20 Lines	bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts,
}		}
}		}

static void		static void
bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason,		bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason,
uint32_t newbw, uint32_t obw, uint32_t diff,		uint32_t newbw, uint32_t obw, uint32_t diff,
uint32_t tim)		uint32_t tim)
{		{
if (tcp_bblogging_on(bbr->rc_tp)) {		if (/bbr_verbose_logging && /tcp_bblogging_on(bbr->rc_tp)) {
union tcp_log_stackspecific log;		union tcp_log_stackspecific log;

bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
log.u_bbr.flex1 = reason;		log.u_bbr.flex1 = reason;
log.u_bbr.flex2 = newbw;		log.u_bbr.flex2 = newbw;
log.u_bbr.flex3 = obw;		log.u_bbr.flex3 = obw;
log.u_bbr.flex4 = diff;		log.u_bbr.flex4 = diff;
log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;		log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;
Show All 11 Lines	TCP_LOG_EVENTP(bbr->rc_tp, NULL,
BBR_LOG_BWSAMP, 0,		BBR_LOG_BWSAMP, 0,
0, &log, false, &bbr->rc_tv);		0, &log, false, &bbr->rc_tv);
}		}
}		}

static inline void		static inline void
bbr_log_progress_event(struct tcp_bbr bbr, struct tcpcb tp, uint32_t tick, int event, int line)		bbr_log_progress_event(struct tcp_bbr bbr, struct tcpcb tp, uint32_t tick, int event, int line)
{		{
if (tcp_bblogging_on(bbr->rc_tp)) {		if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
union tcp_log_stackspecific log;		union tcp_log_stackspecific log;

bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
log.u_bbr.flex1 = line;		log.u_bbr.flex1 = line;
log.u_bbr.flex2 = tick;		log.u_bbr.flex2 = tick;
log.u_bbr.flex3 = tp->t_maxunacktime;		log.u_bbr.flex3 = tp->t_maxunacktime;
log.u_bbr.flex4 = tp->t_acktime;		log.u_bbr.flex4 = tp->t_acktime;
log.u_bbr.flex8 = event;		log.u_bbr.flex8 = event;
▲ Show 20 Lines • Show All 3,567 Lines • ▼ Show 20 Lines	if (bbr->rc_prtt_set_ts) {

bbr->rc_prtt_set_ts = 0;		bbr->rc_prtt_set_ts = 0;
rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);		rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
if (rtt > rtt_prop)		if (rtt > rtt_prop)
filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);		filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);
else		else
apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);		apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
}		}
		#ifdef STATS
		stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rtt));
		#endif
if (bbr->rc_ack_was_delayed)		if (bbr->rc_ack_was_delayed)
rtt += bbr->r_ctl.rc_ack_hdwr_delay;		rtt += bbr->r_ctl.rc_ack_hdwr_delay;

if (rtt < bbr->r_ctl.rc_lowest_rtt)		if (rtt < bbr->r_ctl.rc_lowest_rtt)
bbr->r_ctl.rc_lowest_rtt = rtt;		bbr->r_ctl.rc_lowest_rtt = rtt;
bbr_log_rtt_sample(bbr, rtt, tsin);		bbr_log_rtt_sample(bbr, rtt, tsin);
if (bbr->r_init_rtt) {		if (bbr->r_init_rtt) {
/*		/*
▲ Show 20 Lines • Show All 3,553 Lines • ▼ Show 20 Lines	if (ctf_progress_timeout_check(tp, true)) {
return (1);		return (1);
}		}
}		}
return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,		return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
tiwin, thflags, nxt_pkt));		tiwin, thflags, nxt_pkt));
}		}

static void		static void
bbr_stop_all_timers(struct tcpcb *tp)		bbr_stop_all_timers(struct tcpcb tp, struct tcp_bbr bbr)
{		{
struct tcp_bbr *bbr;

/*		/*
* Assure no timers are running.		* Assure no timers are running.
*/		*/
if (tcp_timer_active(tp, TT_PERSIST)) {		if (tcp_timer_active(tp, TT_PERSIST)) {
/* We enter in persists, set the flag appropriately */		/* We enter in persists, set the flag appropriately */
bbr = (struct tcp_bbr *)tp->t_fb_ptr;
bbr->rc_in_persist = 1;		bbr->rc_in_persist = 1;
}		}
}		}

static void		static void
bbr_google_mode_on(struct tcp_bbr *bbr)		bbr_google_mode_on(struct tcp_bbr *bbr)
{		{
bbr->rc_use_google = 1;		bbr->rc_use_google = 1;
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines	reset_time_small(&bbr->r_ctl.rc_rttprop,
(bbr_filter_len_sec * USECS_IN_SECOND));		(bbr_filter_len_sec * USECS_IN_SECOND));
tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));		tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
}		}
/*		/*
* Return 0 on success, non-zero on failure		* Return 0 on success, non-zero on failure
* which indicates the error (usually no memory).		* which indicates the error (usually no memory).
*/		*/
static int		static int
bbr_init(struct tcpcb *tp)		bbr_init(struct tcpcb tp, void *ptr)
{		{
struct inpcb *inp = tptoinpcb(tp);		struct inpcb *inp = tptoinpcb(tp);
struct tcp_bbr *bbr = NULL;		struct tcp_bbr *bbr = NULL;
uint32_t cts;		uint32_t cts;

tp->t_fb_ptr = uma_zalloc(bbr_pcb_zone, (M_NOWAIT \| M_ZERO));		*ptr = uma_zalloc(bbr_pcb_zone, (M_NOWAIT \| M_ZERO));
if (tp->t_fb_ptr == NULL) {		if (*ptr == NULL) {
/*		/*
* We need to allocate memory but cant. The INP and INP_INFO		* We need to allocate memory but cant. The INP and INP_INFO
* locks and they are recursive (happens during setup. So a		* locks and they are recursive (happens during setup. So a
* scheme to drop the locks fails :(		* scheme to drop the locks fails :(
*		*
*/		*/
return (ENOMEM);		return (ENOMEM);
}		}
bbr = (struct tcp_bbr *)tp->t_fb_ptr;		bbr = (struct tcp_bbr )ptr;
bbr->rtt_valid = 0;		bbr->rtt_valid = 0;
inp->inp_flags2 \|= INP_CANNOT_DO_ECN;		inp->inp_flags2 \|= INP_CANNOT_DO_ECN;
inp->inp_flags2 \|= INP_SUPPORTS_MBUFQ;		inp->inp_flags2 \|= INP_SUPPORTS_MBUFQ;
		/* Take off any undesired flags */
		inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
		inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
		inp->inp_flags2 &= ~INP_MBUF_ACKCMP;
		inp->inp_flags2 &= ~INP_MBUF_L_ACKS;

TAILQ_INIT(&bbr->r_ctl.rc_map);		TAILQ_INIT(&bbr->r_ctl.rc_map);
TAILQ_INIT(&bbr->r_ctl.rc_free);		TAILQ_INIT(&bbr->r_ctl.rc_free);
TAILQ_INIT(&bbr->r_ctl.rc_tmap);		TAILQ_INIT(&bbr->r_ctl.rc_tmap);
bbr->rc_tp = tp;		bbr->rc_tp = tp;
bbr->rc_inp = inp;		bbr->rc_inp = inp;
cts = tcp_get_usecs(&bbr->rc_tv);		cts = tcp_get_usecs(&bbr->rc_tv);
tp->t_acktime = 0;		tp->t_acktime = 0;
bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;		bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines	#ifdef NETFLIX_PEAKRATE
tp->t_peakrate_thr = tp->t_maxpeakrate;		tp->t_peakrate_thr = tp->t_maxpeakrate;
#endif		#endif
if (tp->snd_una != tp->snd_max) {		if (tp->snd_una != tp->snd_max) {
/* Create a send map for the current outstanding data */		/* Create a send map for the current outstanding data */
struct bbr_sendmap *rsm;		struct bbr_sendmap *rsm;

rsm = bbr_alloc(bbr);		rsm = bbr_alloc(bbr);
if (rsm == NULL) {		if (rsm == NULL) {
uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);		uma_zfree(bbr_pcb_zone, *ptr);
tp->t_fb_ptr = NULL;		*ptr = NULL;
return (ENOMEM);		return (ENOMEM);
}		}
rsm->r_rtt_not_allowed = 1;		rsm->r_rtt_not_allowed = 1;
rsm->r_tim_lastsent[0] = cts;		rsm->r_tim_lastsent[0] = cts;
rsm->r_rtr_cnt = 1;		rsm->r_rtr_cnt = 1;
rsm->r_rtr_bytes = 0;		rsm->r_rtr_bytes = 0;
rsm->r_start = tp->snd_una;		rsm->r_start = tp->snd_una;
rsm->r_end = tp->snd_max;		rsm->r_end = tp->snd_max;
Show All 36 Lines	#endif
/* announce the settings and state */		/* announce the settings and state */
bbr_log_settings_change(bbr, BBR_RECOVERY_LOWRTT);		bbr_log_settings_change(bbr, BBR_RECOVERY_LOWRTT);
tcp_bbr_tso_size_check(bbr, cts);		tcp_bbr_tso_size_check(bbr, cts);
/*		/*
* Now call the generic function to start a timer. This will place		* Now call the generic function to start a timer. This will place
* the TCB on the hptsi wheel if a timer is needed with appropriate		* the TCB on the hptsi wheel if a timer is needed with appropriate
* flags.		* flags.
*/		*/
bbr_stop_all_timers(tp);		bbr_stop_all_timers(tp, bbr);
		/*
		* Validate the timers are not in usec, if they are convert.
		* BBR should in theory move to USEC and get rid of a
		* lot of the TICKS_2 calls.. but for now we stay
		* with tick timers.
		*/
		tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
		TCPT_RANGESET(tp->t_rxtcur,
		((tp->t_srtt >> 2) + tp->t_rttvar) >> 1,
		tp->t_rttmin, TCPTV_REXMTMAX);
bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0);		bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0);
return (0);		return (0);
}		}

/*		/*
* Return 0 if we can accept the connection. Return		* Return 0 if we can accept the connection. Return
* non-zero if we can't handle the connection. A EAGAIN		* non-zero if we can't handle the connection. A EAGAIN
* means you need to wait until the connection is up.		* means you need to wait until the connection is up.
Show All 27 Lines	bbr_handoff_ok(struct tcpcb *tp)
*/		*/
return (EINVAL);		return (EINVAL);
}		}

static void		static void
bbr_fini(struct tcpcb *tp, int32_t tcb_is_purged)		bbr_fini(struct tcpcb *tp, int32_t tcb_is_purged)
{		{
if (tp->t_fb_ptr) {		if (tp->t_fb_ptr) {
struct inpcb *inp = tptoinpcb(tp);
uint32_t calc;		uint32_t calc;
struct tcp_bbr *bbr;		struct tcp_bbr *bbr;
struct bbr_sendmap *rsm;		struct bbr_sendmap *rsm;

bbr = (struct tcp_bbr *)tp->t_fb_ptr;		bbr = (struct tcp_bbr *)tp->t_fb_ptr;
if (bbr->r_ctl.crte)		if (bbr->r_ctl.crte)
tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);		tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
bbr_log_flowend(bbr);		bbr_log_flowend(bbr);
bbr->rc_tp = NULL;		bbr->rc_tp = NULL;
/* Backout any flags2 we applied */
inp->inp_flags2 &= ~INP_CANNOT_DO_ECN;
inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
if (bbr->bbr_hdrw_pacing)		if (bbr->bbr_hdrw_pacing)
counter_u64_add(bbr_flows_whdwr_pacing, -1);		counter_u64_add(bbr_flows_whdwr_pacing, -1);
else		else
counter_u64_add(bbr_flows_nohdwr_pacing, -1);		counter_u64_add(bbr_flows_nohdwr_pacing, -1);
if (bbr->r_ctl.crte != NULL) {		if (bbr->r_ctl.crte != NULL) {
tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);		tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
bbr->r_ctl.crte = NULL;		bbr->r_ctl.crte = NULL;
}		}
▲ Show 20 Lines • Show All 1,651 Lines • ▼ Show 20 Lines	#endif
struct inpcb *inp;		struct inpcb *inp;
struct sockbuf *sb;		struct sockbuf *sb;
uint32_t hpts_calling;		uint32_t hpts_calling;
#ifdef INET6		#ifdef INET6
struct ip6_hdr *ip6 = NULL;		struct ip6_hdr *ip6 = NULL;
int32_t isipv6;		int32_t isipv6;
#endif		#endif
uint8_t app_limited = BBR_JR_SENT_DATA;		uint8_t app_limited = BBR_JR_SENT_DATA;
uint8_t filled_all = 0;
bbr = (struct tcp_bbr *)tp->t_fb_ptr;		bbr = (struct tcp_bbr *)tp->t_fb_ptr;
/* We take a cache hit here */		/* We take a cache hit here */
memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));		memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));
cts = tcp_tv_to_usectick(&bbr->rc_tv);		cts = tcp_tv_to_usectick(&bbr->rc_tv);
inp = bbr->rc_inp;		inp = bbr->rc_inp;
so = inp->inp_socket;		so = inp->inp_socket;
sb = &so->so_snd;		sb = &so->so_snd;
if (tp->t_nic_ktls_xmit)		if (tp->t_nic_ktls_xmit)
▲ Show 20 Lines • Show All 1,292 Lines • ▼ Show 20 Lines	#ifdef BBR_INVARIANTS
}		}
#endif		#endif
m->m_next = tcp_m_copym(		m->m_next = tcp_m_copym(
mb, moff, &len,		mb, moff, &len,
if_hw_tsomaxsegcount,		if_hw_tsomaxsegcount,
if_hw_tsomaxsegsize, msb,		if_hw_tsomaxsegsize, msb,
((rsm == NULL) ? hw_tls : 0)		((rsm == NULL) ? hw_tls : 0)
#ifdef NETFLIX_COPY_ARGS		#ifdef NETFLIX_COPY_ARGS
, &filled_all		, NULL, NULL
#endif		#endif
);		);
if (len <= maxseg) {		if (len <= maxseg) {
/*		/*
* Must have ran out of mbufs for the copy		* Must have ran out of mbufs for the copy
* shorten it to no longer need tso. Lets		* shorten it to no longer need tso. Lets
* not put on sendalot since we are low on		* not put on sendalot since we are low on
* mbufs.		* mbufs.
▲ Show 20 Lines • Show All 295 Lines • ▼ Show 20 Lines	KASSERT(len + hdrlen == m_length(m, NULL),
__func__, len, hdrlen, m_length(m, NULL)));		__func__, len, hdrlen, m_length(m, NULL)));

#ifdef TCP_HHOOK		#ifdef TCP_HHOOK
/* Run HHOOK_TC_ESTABLISHED_OUT helper hooks. */		/* Run HHOOK_TC_ESTABLISHED_OUT helper hooks. */
hhook_run_tcp_est_out(tp, th, &to, len, tso);		hhook_run_tcp_est_out(tp, th, &to, len, tso);
#endif		#endif

/* Log to the black box */		/* Log to the black box */
if (tcp_bblogging_on(bbr->rc_tp)) {		if (tcp_bblogging_on(tp)) {
union tcp_log_stackspecific log;		union tcp_log_stackspecific log;

bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);		bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
/* Record info on type of transmission */		/* Record info on type of transmission */
log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;		log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;
log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);		log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);
log.u_bbr.flex3 = maxseg;		log.u_bbr.flex3 = maxseg;
log.u_bbr.flex4 = delay_calc;		log.u_bbr.flex4 = delay_calc;
/* Encode filled_all into the upper flex5 bit */
log.u_bbr.flex5 = bbr->rc_past_init_win;		log.u_bbr.flex5 = bbr->rc_past_init_win;
log.u_bbr.flex5 <<= 1;		log.u_bbr.flex5 <<= 1;
log.u_bbr.flex5 \|= bbr->rc_no_pacing;		log.u_bbr.flex5 \|= bbr->rc_no_pacing;
log.u_bbr.flex5 <<= 29;		log.u_bbr.flex5 <<= 29;
if (filled_all)
log.u_bbr.flex5 \|= 0x80000000;
log.u_bbr.flex5 \|= tp->t_maxseg;		log.u_bbr.flex5 \|= tp->t_maxseg;
log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;		log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;
log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) \| bbr_state_val(bbr);		log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) \| bbr_state_val(bbr);
/* lets poke in the low and the high here for debugging */		/* lets poke in the low and the high here for debugging */
log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;		log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
if (rsm \|\| sack_rxmit) {		if (rsm \|\| sack_rxmit) {
if (doing_tlp)		if (doing_tlp)
log.u_bbr.flex8 = 2;		log.u_bbr.flex8 = 2;
▲ Show 20 Lines • Show All 567 Lines • ▼ Show 20 Lines
static int		static int
bbr_pru_options(struct tcpcb *tp, int flags)		bbr_pru_options(struct tcpcb *tp, int flags)
{		{
if (flags & PRUS_OOB)		if (flags & PRUS_OOB)
return (EOPNOTSUPP);		return (EOPNOTSUPP);
return (0);		return (0);
}		}

		static void
		bbr_switch_failed(struct tcpcb *tp)
		{
		/*
		* If a switch fails we only need to
		* make sure mbuf_queuing is still in place.
		* We also need to make sure we are still in
		* ticks granularity (though we should probably
		* change bbr to go to USECs).
		*
		* For timers we need to see if we are still in the
		* pacer (if our flags are up) if so we are good, if
		* not we need to get back into the pacer.
		*/
		struct inpcb *inp = tptoinpcb(tp);
		struct timeval tv;
		uint32_t cts;
		uint32_t toval;
		struct tcp_bbr *bbr;
		struct hpts_diag diag;

		inp->inp_flags2 \|= INP_CANNOT_DO_ECN;
		inp->inp_flags2 \|= INP_SUPPORTS_MBUFQ;
		tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
		if (inp->inp_in_hpts) {
		return;
		}
		bbr = (struct tcp_bbr *)tp->t_fb_ptr;
		cts = tcp_get_usecs(&tv);
		if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
		if (TSTMP_GT(bbr->rc_pacer_started, cts)) {
		toval = bbr->rc_pacer_started - cts;
		} else {
		/* one slot please */
		toval = HPTS_TICKS_PER_SLOT;
		}
		} else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
		if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
		toval = bbr->r_ctl.rc_timer_exp - cts;
		} else {
		/* one slot please */
		toval = HPTS_TICKS_PER_SLOT;
		}
		} else
		toval = HPTS_TICKS_PER_SLOT;
		(void)tcp_hpts_insert_diag(inp, HPTS_USEC_TO_SLOTS(toval),
		__LINE__, &diag);
		bbr_log_hpts_diag(bbr, cts, &diag);
		}

struct tcp_function_block __tcp_bbr = {		struct tcp_function_block __tcp_bbr = {
.tfb_tcp_block_name = __XSTRING(STACKNAME),		.tfb_tcp_block_name = __XSTRING(STACKNAME),
.tfb_tcp_output = bbr_output,		.tfb_tcp_output = bbr_output,
.tfb_do_queued_segments = ctf_do_queued_segments,		.tfb_do_queued_segments = ctf_do_queued_segments,
.tfb_do_segment_nounlock = bbr_do_segment_nounlock,		.tfb_do_segment_nounlock = bbr_do_segment_nounlock,
.tfb_tcp_do_segment = bbr_do_segment,		.tfb_tcp_do_segment = bbr_do_segment,
.tfb_tcp_ctloutput = bbr_ctloutput,		.tfb_tcp_ctloutput = bbr_ctloutput,
.tfb_tcp_fb_init = bbr_init,		.tfb_tcp_fb_init = bbr_init,
.tfb_tcp_fb_fini = bbr_fini,		.tfb_tcp_fb_fini = bbr_fini,
.tfb_tcp_timer_stop_all = bbr_stopall,		.tfb_tcp_timer_stop_all = bbr_stopall,
.tfb_tcp_rexmit_tmr = bbr_remxt_tmr,		.tfb_tcp_rexmit_tmr = bbr_remxt_tmr,
.tfb_tcp_handoff_ok = bbr_handoff_ok,		.tfb_tcp_handoff_ok = bbr_handoff_ok,
.tfb_tcp_mtu_chg = bbr_mtu_chg,		.tfb_tcp_mtu_chg = bbr_mtu_chg,
.tfb_pru_options = bbr_pru_options,		.tfb_pru_options = bbr_pru_options,
		.tfb_switch_failed = bbr_switch_failed,
.tfb_flags = TCP_FUNC_OUTPUT_CANDROP,		.tfb_flags = TCP_FUNC_OUTPUT_CANDROP,
};		};

/*		/*
* bbr_ctloutput() must drop the inpcb lock before performing copyin on		* bbr_ctloutput() must drop the inpcb lock before performing copyin on
* socket option arguments. When it re-acquires the lock after the copy, it		* socket option arguments. When it re-acquires the lock after the copy, it
* has to revalidate that the connection is still valid for the socket		* has to revalidate that the connection is still valid for the socket
* option.		* option.
▲ Show 20 Lines • Show All 674 Lines • Show Last 20 Lines