Differential D22479 Diff 64673 netinet/tcp_stacks/bbr.c

Changeset View

Standalone View

netinet/tcp_stacks/bbr.c

	/*-	/*-
	* Copyright (c) 2016-2019	* Copyright (c) 2016-9
	* Netflix Inc.	* Netflix Inc.
	* All rights reserved.	* All rights reserved.
	*	*
Context not available.
	#include "opt_ratelimit.h"	#include "opt_ratelimit.h"
	#include "opt_kern_tls.h"	#include "opt_kern_tls.h"
	#include <sys/param.h>	#include <sys/param.h>
		#include <sys/arb.h>
	#include <sys/module.h>	#include <sys/module.h>
	#include <sys/kernel.h>	#include <sys/kernel.h>
	#ifdef TCP_HHOOK	#ifdef TCP_HHOOK
Context not available.
	#endif	#endif
	#include <sys/sysctl.h>	#include <sys/sysctl.h>
	#include <sys/systm.h>	#include <sys/systm.h>
		#ifdef STATS
	#include <sys/qmath.h>	#include <sys/qmath.h>
	#include <sys/tree.h>	#include <sys/tree.h>
	#ifdef NETFLIX_STATS
	#include <sys/stats.h> /* Must come after qmath.h and tree.h */	#include <sys/stats.h> /* Must come after qmath.h and tree.h */
	#endif	#endif
	#include <sys/refcount.h>	#include <sys/refcount.h>
Context not available.
	static int32_t bbr_hardware_pacing_limit = 8000;	static int32_t bbr_hardware_pacing_limit = 8000;
	static int32_t bbr_quanta = 3; /* How much extra quanta do we get? */	static int32_t bbr_quanta = 3; /* How much extra quanta do we get? */
	static int32_t bbr_no_retran = 0;	static int32_t bbr_no_retran = 0;
	static int32_t bbr_tcp_map_entries_limit = 1500;
	static int32_t bbr_tcp_map_split_limit = 256;


	static int32_t bbr_error_base_paceout = 10000; /* usec to pace */	static int32_t bbr_error_base_paceout = 10000; /* usec to pace */
	static int32_t bbr_max_net_error_cnt = 10;	static int32_t bbr_max_net_error_cnt = 10;
	/* Should the following be dynamic too -- loss wise */	/* Should the following be dynamic too -- loss wise */
Context not available.
	static struct bbr_sendmap *	static struct bbr_sendmap *
	bbr_alloc_full_limit(struct tcp_bbr *bbr)	bbr_alloc_full_limit(struct tcp_bbr *bbr)
	{	{
	if ((bbr_tcp_map_entries_limit > 0) &&	if ((V_tcp_map_entries_limit > 0) &&
	(bbr->r_ctl.rc_num_maps_alloced >= bbr_tcp_map_entries_limit)) {	(bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
	BBR_STAT_INC(bbr_alloc_limited);	BBR_STAT_INC(bbr_alloc_limited);
	if (!bbr->alloc_limit_reported) {	if (!bbr->alloc_limit_reported) {
	bbr->alloc_limit_reported = 1;	bbr->alloc_limit_reported = 1;
Context not available.

	if (limit_type) {	if (limit_type) {
	/* currently there is only one limit type */	/* currently there is only one limit type */
	if (bbr_tcp_map_split_limit > 0 &&	if (V_tcp_map_split_limit > 0 &&
	bbr->r_ctl.rc_num_split_allocs >= bbr_tcp_map_split_limit) {	bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {
	BBR_STAT_INC(bbr_split_limited);	BBR_STAT_INC(bbr_split_limited);
	if (!bbr->alloc_limit_reported) {	if (!bbr->alloc_limit_reported) {
	bbr->alloc_limit_reported = 1;	bbr->alloc_limit_reported = 1;
Context not available.
	uint32_t cwnd, target_cwnd, saved_bytes, maxseg;	uint32_t cwnd, target_cwnd, saved_bytes, maxseg;
	int32_t meth;	int32_t meth;

	#ifdef NETFLIX_STATS	#ifdef STATS
	if ((tp->t_flags & TF_GPUTINPROG) &&	if ((tp->t_flags & TF_GPUTINPROG) &&
	SEQ_GEQ(th->th_ack, tp->gput_ack)) {	SEQ_GEQ(th->th_ack, tp->gput_ack)) {
	/*	/*
Context not available.
	}	}
	TCPSTAT_INC(tcps_rttupdated);	TCPSTAT_INC(tcps_rttupdated);
	tp->t_rttupdated++;	tp->t_rttupdated++;
	#ifdef NETFLIX_STATS	#ifdef STATS
	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));
	#endif	#endif
	/*	/*
Context not available.
	return (0);	return (0);
	}	}
	}	}

	#endif	#endif
	if (DELAY_ACK(tp, bbr, nsegs) \|\| tfo_syn) {	if (DELAY_ACK(tp, bbr, nsegs) \|\| tfo_syn) {
	bbr->bbr_segs_rcvd += max(1, nsegs);	bbr->bbr_segs_rcvd += max(1, nsegs);
Context not available.
	* reassembly queue and we have enough buffer space to take it.	* reassembly queue and we have enough buffer space to take it.
	*/	*/
	nsegs = max(1, m->m_pkthdr.lro_nsegs);	nsegs = max(1, m->m_pkthdr.lro_nsegs);

	#ifdef NETFLIX_SB_LIMITS	#ifdef NETFLIX_SB_LIMITS
	if (so->so_rcv.sb_shlim) {	if (so->so_rcv.sb_shlim) {
	mcnt = m_memcnt(m);	mcnt = m_memcnt(m);
Context not available.
	newsize, so, NULL))	newsize, so, NULL))
	so->so_rcv.sb_flags &= ~SB_AUTOSIZE;	so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
	m_adj(m, drop_hdrlen); /* delayed header drop */	m_adj(m, drop_hdrlen); /* delayed header drop */

	#ifdef NETFLIX_SB_LIMITS	#ifdef NETFLIX_SB_LIMITS
	appended =	appended =
	#endif	#endif
Context not available.
	* the scale is zero.	* the scale is zero.
	*/	*/
	tiwin = th->th_win << tp->snd_scale;	tiwin = th->th_win << tp->snd_scale;
	#ifdef NETFLIX_STATS	#ifdef STATS
	stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);	stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
	#endif	#endif
	/*	/*
Context not available.
	if ((tp->t_flags & TF_FORCEDATA) && len == 1) {	if ((tp->t_flags & TF_FORCEDATA) && len == 1) {
	/* Window probe */	/* Window probe */
	TCPSTAT_INC(tcps_sndprobe);	TCPSTAT_INC(tcps_sndprobe);
	#ifdef NETFLIX_STATS	#ifdef STATS
	stats_voi_update_abs_u32(tp->t_stats,	stats_voi_update_abs_u32(tp->t_stats,
	VOI_TCP_RETXPB, len);	VOI_TCP_RETXPB, len);
	#endif	#endif
Context not available.
	tp->t_sndrexmitpack++;	tp->t_sndrexmitpack++;
	TCPSTAT_INC(tcps_sndrexmitpack);	TCPSTAT_INC(tcps_sndrexmitpack);
	TCPSTAT_ADD(tcps_sndrexmitbyte, len);	TCPSTAT_ADD(tcps_sndrexmitbyte, len);
	#ifdef NETFLIX_STATS	#ifdef STATS
	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
	len);	len);
	#endif	#endif
Context not available.
	/* Place in 17's the total sent */	/* Place in 17's the total sent */
	counter_u64_add(bbr_state_resend[17], len);	counter_u64_add(bbr_state_resend[17], len);
	counter_u64_add(bbr_state_lost[17], len);	counter_u64_add(bbr_state_lost[17], len);
	#ifdef NETFLIX_STATS	#ifdef STATS
	stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,	stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
	len);	len);
	#endif	#endif
Context not available.
	* as long as we are not retransmiting.	* as long as we are not retransmiting.
	*/	*/
	if ((rsm == NULL) &&	if ((rsm == NULL) &&
	(bbr_tcp_map_entries_limit > 0) &&	(V_tcp_map_entries_limit > 0) &&
	(bbr->r_ctl.rc_num_maps_alloced >= bbr_tcp_map_entries_limit)) {	(bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
	BBR_STAT_INC(bbr_alloc_limited);	BBR_STAT_INC(bbr_alloc_limited);
	if (!bbr->alloc_limit_reported) {	if (!bbr->alloc_limit_reported) {
	bbr->alloc_limit_reported = 1;	bbr->alloc_limit_reported = 1;
Context not available.
	SOCKBUF_UNLOCK(&so->so_snd);	SOCKBUF_UNLOCK(&so->so_snd);
	return (EHOSTUNREACH);	return (EHOSTUNREACH);
	}	}

	hdrlen += sizeof(struct udphdr);	hdrlen += sizeof(struct udphdr);
	}	}
	#endif	#endif
Context not available.
	bbr_start_hpts_timer(bbr, tp, cts, 11, slot, 0);	bbr_start_hpts_timer(bbr, tp, cts, 11, slot, 0);
	return (error);	return (error);
	}	}
	#ifdef NETFLIX_STATS	#ifdef STATS
	} else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&	} else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&
	len &&	len &&
	(rsm == NULL) &&	(rsm == NULL) &&
Context not available.