Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/tcp_stacks/bbr.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 5,902 Lines • ▼ Show 20 Lines | if (old_tso != new_tso) { | ||||
bbr->r_ctl.rc_pace_max_segs = new_tso; | bbr->r_ctl.rc_pace_max_segs = new_tso; | ||||
} | } | ||||
/* We have hardware pacing! */ | /* We have hardware pacing! */ | ||||
bbr_adjust_for_hw_pacing(bbr, cts); | bbr_adjust_for_hw_pacing(bbr, cts); | ||||
} | } | ||||
static void | static void | ||||
bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len, | bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len, | ||||
uint32_t seq_out, uint8_t th_flags, int32_t err, uint32_t cts, | uint32_t seq_out, uint16_t th_flags, int32_t err, uint32_t cts, | ||||
struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc, | struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc, | ||||
struct sockbuf *sb) | struct sockbuf *sb) | ||||
{ | { | ||||
struct bbr_sendmap *rsm, *nrsm; | struct bbr_sendmap *rsm, *nrsm; | ||||
register uint32_t snd_max, snd_una; | register uint32_t snd_max, snd_una; | ||||
uint32_t pacing_time; | uint32_t pacing_time; | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 1,413 Lines • ▼ Show 20 Lines | bbr_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, | ||||
struct bbr_sendmap *rsm; | struct bbr_sendmap *rsm; | ||||
struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1]; | struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1]; | ||||
register uint32_t th_ack; | register uint32_t th_ack; | ||||
int32_t i, j, k, new_sb, num_sack_blks = 0; | int32_t i, j, k, new_sb, num_sack_blks = 0; | ||||
uint32_t cts, acked, ack_point, sack_changed = 0; | uint32_t cts, acked, ack_point, sack_changed = 0; | ||||
uint32_t p_maxseg, maxseg, p_acked = 0; | uint32_t p_maxseg, maxseg, p_acked = 0; | ||||
INP_WLOCK_ASSERT(tp->t_inpcb); | INP_WLOCK_ASSERT(tp->t_inpcb); | ||||
if (th->th_flags & TH_RST) { | if (tcp_get_flags(th) & TH_RST) { | ||||
/* We don't log resets */ | /* We don't log resets */ | ||||
return (0); | return (0); | ||||
} | } | ||||
bbr = (struct tcp_bbr *)tp->t_fb_ptr; | bbr = (struct tcp_bbr *)tp->t_fb_ptr; | ||||
cts = bbr->r_ctl.rc_rcvtime; | cts = bbr->r_ctl.rc_rcvtime; | ||||
rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); | rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); | ||||
changed = 0; | changed = 0; | ||||
▲ Show 20 Lines • Show All 928 Lines • ▼ Show 20 Lines | #endif | ||||
((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && | ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && | ||||
(tp->t_fbyte_in == 0)) { | (tp->t_fbyte_in == 0)) { | ||||
tp->t_fbyte_in = ticks; | tp->t_fbyte_in = ticks; | ||||
if (tp->t_fbyte_in == 0) | if (tp->t_fbyte_in == 0) | ||||
tp->t_fbyte_in = 1; | tp->t_fbyte_in = 1; | ||||
if (tp->t_fbyte_out && tp->t_fbyte_in) | if (tp->t_fbyte_out && tp->t_fbyte_in) | ||||
tp->t_flags2 |= TF2_FBYTES_COMPLETE; | tp->t_flags2 |= TF2_FBYTES_COMPLETE; | ||||
} | } | ||||
thflags = th->th_flags & TH_FIN; | thflags = tcp_get_flags(th) & TH_FIN; | ||||
KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs); | KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs); | ||||
KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); | KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); | ||||
SOCKBUF_LOCK(&so->so_rcv); | SOCKBUF_LOCK(&so->so_rcv); | ||||
if (so->so_rcv.sb_state & SBS_CANTRCVMORE) | if (so->so_rcv.sb_state & SBS_CANTRCVMORE) | ||||
m_freem(m); | m_freem(m); | ||||
else | else | ||||
#ifdef NETFLIX_SB_LIMITS | #ifdef NETFLIX_SB_LIMITS | ||||
appended = | appended = | ||||
▲ Show 20 Lines • Show All 3,064 Lines • ▼ Show 20 Lines | bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, | ||||
int32_t prev_state; | int32_t prev_state; | ||||
uint32_t lost; | uint32_t lost; | ||||
nsegs = max(1, m->m_pkthdr.lro_nsegs); | nsegs = max(1, m->m_pkthdr.lro_nsegs); | ||||
bbr = (struct tcp_bbr *)tp->t_fb_ptr; | bbr = (struct tcp_bbr *)tp->t_fb_ptr; | ||||
/* add in our stats */ | /* add in our stats */ | ||||
kern_prefetch(bbr, &prev_state); | kern_prefetch(bbr, &prev_state); | ||||
prev_state = 0; | prev_state = 0; | ||||
thflags = th->th_flags; | thflags = tcp_get_flags(th); | ||||
/* | /* | ||||
* If this is either a state-changing packet or current state isn't | * If this is either a state-changing packet or current state isn't | ||||
* established, we require a write lock on tcbinfo. Otherwise, we | * established, we require a write lock on tcbinfo. Otherwise, we | ||||
* allow the tcbinfo to be in either alocked or unlocked, as the | * allow the tcbinfo to be in either alocked or unlocked, as the | ||||
* caller may have unnecessarily acquired a write lock due to a | * caller may have unnecessarily acquired a write lock due to a | ||||
* race. | * race. | ||||
*/ | */ | ||||
INP_WLOCK_ASSERT(tp->t_inpcb); | INP_WLOCK_ASSERT(tp->t_inpcb); | ||||
▲ Show 20 Lines • Show All 2,070 Lines • ▼ Show 20 Lines | if (sack_rxmit == 0) { | ||||
th->th_seq = htonl(rsm->r_start); | th->th_seq = htonl(rsm->r_start); | ||||
bbr_seq = rsm->r_start; | bbr_seq = rsm->r_start; | ||||
} | } | ||||
th->th_ack = htonl(tp->rcv_nxt); | th->th_ack = htonl(tp->rcv_nxt); | ||||
if (optlen) { | if (optlen) { | ||||
bcopy(opt, th + 1, optlen); | bcopy(opt, th + 1, optlen); | ||||
th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; | th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; | ||||
} | } | ||||
th->th_flags = flags; | tcp_set_flags(th, flags); | ||||
/* | /* | ||||
* Calculate receive window. Don't shrink window, but avoid silly | * Calculate receive window. Don't shrink window, but avoid silly | ||||
* window syndrome. | * window syndrome. | ||||
*/ | */ | ||||
if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) && | if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) && | ||||
recwin < maxseg))) | recwin < maxseg))) | ||||
recwin = 0; | recwin = 0; | ||||
if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) && | if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) && | ||||
▲ Show 20 Lines • Show All 1,448 Lines • Show Last 20 Lines |