Changeset View
Changeset View
Standalone View
Standalone View
head/sys/netinet/tcp_stacks/bbr.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show All 32 Lines | |||||
#include <sys/cdefs.h> | #include <sys/cdefs.h> | ||||
__FBSDID("$FreeBSD$"); | __FBSDID("$FreeBSD$"); | ||||
#include "opt_inet.h" | #include "opt_inet.h" | ||||
#include "opt_inet6.h" | #include "opt_inet6.h" | ||||
#include "opt_ipsec.h" | #include "opt_ipsec.h" | ||||
#include "opt_tcpdebug.h" | #include "opt_tcpdebug.h" | ||||
#include "opt_ratelimit.h" | #include "opt_ratelimit.h" | ||||
#include "opt_kern_tls.h" | |||||
#include <sys/param.h> | #include <sys/param.h> | ||||
#include <sys/arb.h> | #include <sys/arb.h> | ||||
#include <sys/module.h> | #include <sys/module.h> | ||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/libkern.h> | #include <sys/libkern.h> | ||||
#ifdef TCP_HHOOK | #ifdef TCP_HHOOK | ||||
#include <sys/hhook.h> | #include <sys/hhook.h> | ||||
#endif | #endif | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/mbuf.h> | #include <sys/mbuf.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/socket.h> | #include <sys/socket.h> | ||||
#include <sys/socketvar.h> | #include <sys/socketvar.h> | ||||
#ifdef KERN_TLS | |||||
#include <sys/ktls.h> | |||||
#endif | |||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/systm.h> | #include <sys/systm.h> | ||||
#ifdef STATS | #ifdef STATS | ||||
#include <sys/qmath.h> | #include <sys/qmath.h> | ||||
#include <sys/tree.h> | #include <sys/tree.h> | ||||
#include <sys/stats.h> /* Must come after qmath.h and tree.h */ | #include <sys/stats.h> /* Must come after qmath.h and tree.h */ | ||||
#endif | #endif | ||||
#include <sys/refcount.h> | #include <sys/refcount.h> | ||||
▲ Show 20 Lines • Show All 4,529 Lines • ▼ Show 20 Lines | bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) | ||||
/* Did we somehow get into persists? */ | /* Did we somehow get into persists? */ | ||||
if (bbr->rc_in_persist) { | if (bbr->rc_in_persist) { | ||||
return (0); | return (0); | ||||
} | } | ||||
if (bbr->r_state && (bbr->r_state != tp->t_state)) | if (bbr->r_state && (bbr->r_state != tp->t_state)) | ||||
bbr_set_state(tp, bbr, 0); | bbr_set_state(tp, bbr, 0); | ||||
BBR_STAT_INC(bbr_tlp_tot); | BBR_STAT_INC(bbr_tlp_tot); | ||||
maxseg = tp->t_maxseg - bbr->rc_last_options; | maxseg = tp->t_maxseg - bbr->rc_last_options; | ||||
#ifdef KERN_TLS | |||||
if (bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) { | |||||
/* | /* | ||||
* For hardware TLS we do *not* want to send | |||||
* new data. | |||||
*/ | |||||
goto need_retran; | |||||
} | |||||
#endif | |||||
/* | |||||
* A TLP timer has expired. We have been idle for 2 rtts. So we now | * A TLP timer has expired. We have been idle for 2 rtts. So we now | ||||
* need to figure out how to force a full MSS segment out. | * need to figure out how to force a full MSS segment out. | ||||
*/ | */ | ||||
so = tp->t_inpcb->inp_socket; | so = tp->t_inpcb->inp_socket; | ||||
avail = sbavail(&so->so_snd); | avail = sbavail(&so->so_snd); | ||||
out = ctf_outstanding(tp); | out = ctf_outstanding(tp); | ||||
if (out > tp->snd_wnd) { | if (out > tp->snd_wnd) { | ||||
/* special case, we need a retransmission */ | /* special case, we need a retransmission */ | ||||
▲ Show 20 Lines • Show All 1,176 Lines • ▼ Show 20 Lines | tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts) | ||||
* per-tcb-divisor = 1000 | * per-tcb-divisor = 1000 | ||||
* per-tcb-floor = 1 | * per-tcb-floor = 1 | ||||
* | * | ||||
* This will get you "google bbr" behavior with respect to tso size. | * This will get you "google bbr" behavior with respect to tso size. | ||||
* | * | ||||
* Note we do set anything TSO size until we are past the initial | * Note we do set anything TSO size until we are past the initial | ||||
* window. Before that we gnerally use either a single MSS | * window. Before that we gnerally use either a single MSS | ||||
* or we use the full IW size (so we burst a IW at a time) | * or we use the full IW size (so we burst a IW at a time) | ||||
* Also note that Hardware-TLS is special and does alternate | |||||
* things to minimize PCI Bus Bandwidth use. | |||||
*/ | */ | ||||
if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) { | if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) { | ||||
maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; | maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; | ||||
} else { | } else { | ||||
maxseg = BBR_MIN_SEG - bbr->rc_last_options; | maxseg = BBR_MIN_SEG - bbr->rc_last_options; | ||||
} | } | ||||
#ifdef KERN_TLS | |||||
if (bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) { | |||||
tls_seg = ctf_get_opt_tls_size(bbr->rc_inp->inp_socket, bbr->rc_tp->snd_wnd); | |||||
bbr->r_ctl.rc_pace_min_segs = (tls_seg + bbr->rc_last_options); | |||||
} | |||||
#endif | |||||
old_tso = bbr->r_ctl.rc_pace_max_segs; | old_tso = bbr->r_ctl.rc_pace_max_segs; | ||||
if (bbr->rc_past_init_win == 0) { | if (bbr->rc_past_init_win == 0) { | ||||
/* | /* | ||||
* Not enough data has been acknowledged to make a | * Not enough data has been acknowledged to make a | ||||
* judgement unless we are hardware TLS. Set up | * judgement. Set up the initial TSO based on if we | ||||
* the initial TSO based on if we are sending a | * are sending a full IW at once or not. | ||||
* full IW at once or not. | |||||
*/ | */ | ||||
if (bbr->rc_use_google) | if (bbr->rc_use_google) | ||||
bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2); | bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2); | ||||
else if (bbr->bbr_init_win_cheat) | else if (bbr->bbr_init_win_cheat) | ||||
bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp); | bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp); | ||||
else | else | ||||
bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options; | bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options; | ||||
if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg) | if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg) | ||||
bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg; | bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg; | ||||
#ifdef KERN_TLS | |||||
if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) && tls_seg) { | |||||
/* | |||||
* For hardware TLS we set our min to the tls_seg size. | |||||
*/ | |||||
bbr->r_ctl.rc_pace_max_segs = tls_seg; | |||||
bbr->r_ctl.rc_pace_min_segs = tls_seg + bbr->rc_last_options; | |||||
} | |||||
#endif | |||||
if (bbr->r_ctl.rc_pace_max_segs == 0) { | if (bbr->r_ctl.rc_pace_max_segs == 0) { | ||||
bbr->r_ctl.rc_pace_max_segs = maxseg; | bbr->r_ctl.rc_pace_max_segs = maxseg; | ||||
} | } | ||||
bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0); | bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0); | ||||
#ifdef KERN_TLS | |||||
if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) == 0) | |||||
#endif | |||||
bbr_adjust_for_hw_pacing(bbr, cts); | bbr_adjust_for_hw_pacing(bbr, cts); | ||||
return; | return; | ||||
} | } | ||||
/** | /** | ||||
* Now lets set the TSO goal based on our delivery rate in | * Now lets set the TSO goal based on our delivery rate in | ||||
* bytes per second. Note we only do this if | * bytes per second. Note we only do this if | ||||
* we have acked at least the initial cwnd worth of data. | * we have acked at least the initial cwnd worth of data. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | if (IN_RECOVERY(bbr->rc_tp->t_flags) && | ||||
*/ | */ | ||||
if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg)) | if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg)) | ||||
new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg); | new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg); | ||||
} | } | ||||
if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor))) | if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor))) | ||||
new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor; | new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor; | ||||
if (new_tso > PACE_MAX_IP_BYTES) | if (new_tso > PACE_MAX_IP_BYTES) | ||||
new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg); | new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg); | ||||
/* Enforce an utter maximum if we are not HW-TLS */ | /* Enforce an utter maximum. */ | ||||
#ifdef KERN_TLS | |||||
if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) == 0) | |||||
#endif | |||||
if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) { | if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) { | ||||
new_tso = bbr->r_ctl.bbr_utter_max * maxseg; | new_tso = bbr->r_ctl.bbr_utter_max * maxseg; | ||||
} | } | ||||
#ifdef KERN_TLS | |||||
if (tls_seg) { | |||||
/* | |||||
* Lets move the output size | |||||
* up to 1 or more TLS record sizes. | |||||
*/ | |||||
uint32_t temp; | |||||
temp = roundup(new_tso, tls_seg); | |||||
new_tso = temp; | |||||
/* Back down if needed to under a full frame */ | |||||
while (new_tso > PACE_MAX_IP_BYTES) | |||||
new_tso -= tls_seg; | |||||
} | |||||
#endif | |||||
if (old_tso != new_tso) { | if (old_tso != new_tso) { | ||||
/* Only log changes */ | /* Only log changes */ | ||||
bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0); | bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0); | ||||
bbr->r_ctl.rc_pace_max_segs = new_tso; | bbr->r_ctl.rc_pace_max_segs = new_tso; | ||||
} | } | ||||
#ifdef KERN_TLS | /* We have hardware pacing! */ | ||||
if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) && | |||||
tls_seg) { | |||||
bbr->r_ctl.rc_pace_min_segs = tls_seg + bbr->rc_last_options; | |||||
} else | |||||
#endif | |||||
/* We have hardware pacing and not hardware TLS! */ | |||||
bbr_adjust_for_hw_pacing(bbr, cts); | bbr_adjust_for_hw_pacing(bbr, cts); | ||||
} | } | ||||
static void | static void | ||||
bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len, | bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len, | ||||
uint32_t seq_out, uint8_t th_flags, int32_t err, uint32_t cts, | uint32_t seq_out, uint8_t th_flags, int32_t err, uint32_t cts, | ||||
struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc, | struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc, | ||||
struct sockbuf *sb) | struct sockbuf *sb) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 6,102 Lines • ▼ Show 20 Lines | #endif | ||||
int32_t hw_tls; | int32_t hw_tls; | ||||
#if defined(IPSEC) || defined(IPSEC_SUPPORT) | #if defined(IPSEC) || defined(IPSEC_SUPPORT) | ||||
unsigned ipsec_optlen = 0; | unsigned ipsec_optlen = 0; | ||||
#endif | #endif | ||||
volatile int32_t sack_rxmit; | volatile int32_t sack_rxmit; | ||||
struct bbr_sendmap *rsm = NULL; | struct bbr_sendmap *rsm = NULL; | ||||
int32_t tso, mtu; | int32_t tso, mtu; | ||||
int force_tso = 0; | |||||
struct tcpopt to; | struct tcpopt to; | ||||
int32_t slot = 0; | int32_t slot = 0; | ||||
struct inpcb *inp; | struct inpcb *inp; | ||||
struct sockbuf *sb; | struct sockbuf *sb; | ||||
uint32_t hpts_calling; | uint32_t hpts_calling; | ||||
#ifdef INET6 | #ifdef INET6 | ||||
struct ip6_hdr *ip6 = NULL; | struct ip6_hdr *ip6 = NULL; | ||||
int32_t isipv6; | int32_t isipv6; | ||||
#endif | #endif | ||||
uint8_t app_limited = BBR_JR_SENT_DATA; | uint8_t app_limited = BBR_JR_SENT_DATA; | ||||
uint8_t filled_all = 0; | uint8_t filled_all = 0; | ||||
bbr = (struct tcp_bbr *)tp->t_fb_ptr; | bbr = (struct tcp_bbr *)tp->t_fb_ptr; | ||||
/* We take a cache hit here */ | /* We take a cache hit here */ | ||||
memcpy(&bbr->rc_tv, tv, sizeof(struct timeval)); | memcpy(&bbr->rc_tv, tv, sizeof(struct timeval)); | ||||
cts = tcp_tv_to_usectick(&bbr->rc_tv); | cts = tcp_tv_to_usectick(&bbr->rc_tv); | ||||
inp = bbr->rc_inp; | inp = bbr->rc_inp; | ||||
so = inp->inp_socket; | so = inp->inp_socket; | ||||
sb = &so->so_snd; | sb = &so->so_snd; | ||||
#ifdef KERN_TLS | |||||
if (sb->sb_flags & SB_TLS_IFNET) | if (sb->sb_flags & SB_TLS_IFNET) | ||||
hw_tls = 1; | hw_tls = 1; | ||||
else | else | ||||
#endif | |||||
hw_tls = 0; | hw_tls = 0; | ||||
kern_prefetch(sb, &maxseg); | kern_prefetch(sb, &maxseg); | ||||
maxseg = tp->t_maxseg - bbr->rc_last_options; | maxseg = tp->t_maxseg - bbr->rc_last_options; | ||||
if (bbr_minseg(bbr) < maxseg) { | if (bbr_minseg(bbr) < maxseg) { | ||||
tcp_bbr_tso_size_check(bbr, cts); | tcp_bbr_tso_size_check(bbr, cts); | ||||
} | } | ||||
/* Remove any flags that indicate we are pacing on the inp */ | /* Remove any flags that indicate we are pacing on the inp */ | ||||
pace_max_segs = bbr->r_ctl.rc_pace_max_segs; | pace_max_segs = bbr->r_ctl.rc_pace_max_segs; | ||||
▲ Show 20 Lines • Show All 289 Lines • ▼ Show 20 Lines | if (rsm->r_flags & BBR_HAS_SYN) { | ||||
/* Retransmitting SYN */ | /* Retransmitting SYN */ | ||||
rsm = NULL; | rsm = NULL; | ||||
SOCKBUF_LOCK(sb); | SOCKBUF_LOCK(sb); | ||||
goto send; | goto send; | ||||
} | } | ||||
} else | } else | ||||
len = rsm->r_end - rsm->r_start; | len = rsm->r_end - rsm->r_start; | ||||
if ((bbr->rc_resends_use_tso == 0) && | if ((bbr->rc_resends_use_tso == 0) && | ||||
#ifdef KERN_TLS | |||||
((sb->sb_flags & SB_TLS_IFNET) == 0) && | |||||
#endif | |||||
(len > maxseg)) { | (len > maxseg)) { | ||||
len = maxseg; | len = maxseg; | ||||
more_to_rxt = 1; | more_to_rxt = 1; | ||||
} | } | ||||
sb_offset = rsm->r_start - tp->snd_una; | sb_offset = rsm->r_start - tp->snd_una; | ||||
if (len > 0) { | if (len > 0) { | ||||
sack_rxmit = 1; | sack_rxmit = 1; | ||||
KMOD_TCPSTAT_INC(tcps_sack_rexmits); | KMOD_TCPSTAT_INC(tcps_sack_rexmits); | ||||
▲ Show 20 Lines • Show All 757 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
maxseg = tp->t_maxseg - (ipoptlen + optlen); | maxseg = tp->t_maxseg - (ipoptlen + optlen); | ||||
p_maxseg = min(maxseg, pace_max_segs); | p_maxseg = min(maxseg, pace_max_segs); | ||||
/* | /* | ||||
* Adjust data length if insertion of options will bump the packet | * Adjust data length if insertion of options will bump the packet | ||||
* length beyond the t_maxseg length. Clear the FIN bit because we | * length beyond the t_maxseg length. Clear the FIN bit because we | ||||
* cut off the tail of the segment. | * cut off the tail of the segment. | ||||
*/ | */ | ||||
#ifdef KERN_TLS | |||||
/* force TSO for so TLS offload can get mss */ | |||||
if (sb->sb_flags & SB_TLS_IFNET) { | |||||
force_tso = 1; | |||||
} | |||||
#endif | |||||
if (len > maxseg) { | if (len > maxseg) { | ||||
if (len != 0 && (flags & TH_FIN)) { | if (len != 0 && (flags & TH_FIN)) { | ||||
flags &= ~TH_FIN; | flags &= ~TH_FIN; | ||||
} | } | ||||
if (tso) { | if (tso) { | ||||
uint32_t moff; | uint32_t moff; | ||||
int32_t max_len; | int32_t max_len; | ||||
Show All 17 Lines | if (tso) { | ||||
} else if (len > max_len) { | } else if (len > max_len) { | ||||
len = max_len; | len = max_len; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Prevent the last segment from being fractional | * Prevent the last segment from being fractional | ||||
* unless the send sockbuf can be emptied: | * unless the send sockbuf can be emptied: | ||||
*/ | */ | ||||
if (((sb_offset + len) < sbavail(sb)) && | if ((sb_offset + len) < sbavail(sb)) { | ||||
(hw_tls == 0)) { | |||||
moff = len % (uint32_t)maxseg; | moff = len % (uint32_t)maxseg; | ||||
if (moff != 0) { | if (moff != 0) { | ||||
len -= moff; | len -= moff; | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* In case there are too many small fragments don't | * In case there are too many small fragments don't | ||||
* use TSO: | * use TSO: | ||||
▲ Show 20 Lines • Show All 175 Lines • ▼ Show 20 Lines | #endif | ||||
mb, moff, &len, | mb, moff, &len, | ||||
if_hw_tsomaxsegcount, | if_hw_tsomaxsegcount, | ||||
if_hw_tsomaxsegsize, msb, | if_hw_tsomaxsegsize, msb, | ||||
((rsm == NULL) ? hw_tls : 0) | ((rsm == NULL) ? hw_tls : 0) | ||||
#ifdef NETFLIX_COPY_ARGS | #ifdef NETFLIX_COPY_ARGS | ||||
, &filled_all | , &filled_all | ||||
#endif | #endif | ||||
); | ); | ||||
if (len <= maxseg && !force_tso) { | if (len <= maxseg) { | ||||
/* | /* | ||||
* Must have ran out of mbufs for the copy | * Must have ran out of mbufs for the copy | ||||
* shorten it to no longer need tso. Lets | * shorten it to no longer need tso. Lets | ||||
* not put on sendalot since we are low on | * not put on sendalot since we are low on | ||||
* mbufs. | * mbufs. | ||||
*/ | */ | ||||
tso = 0; | tso = 0; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 309 Lines • ▼ Show 20 Lines | #endif | ||||
} | } | ||||
#endif | #endif | ||||
/* | /* | ||||
* Enable TSO and specify the size of the segments. The TCP pseudo | * Enable TSO and specify the size of the segments. The TCP pseudo | ||||
* header checksum is always provided. XXX: Fixme: This is currently | * header checksum is always provided. XXX: Fixme: This is currently | ||||
* not the case for IPv6. | * not the case for IPv6. | ||||
*/ | */ | ||||
if (tso || force_tso) { | if (tso) { | ||||
KASSERT(force_tso || len > maxseg, | KASSERT(len > maxseg, | ||||
("%s: len:%d <= tso_segsz:%d", __func__, len, maxseg)); | ("%s: len:%d <= tso_segsz:%d", __func__, len, maxseg)); | ||||
m->m_pkthdr.csum_flags |= CSUM_TSO; | m->m_pkthdr.csum_flags |= CSUM_TSO; | ||||
csum_flags |= CSUM_TSO; | csum_flags |= CSUM_TSO; | ||||
m->m_pkthdr.tso_segsz = maxseg; | m->m_pkthdr.tso_segsz = maxseg; | ||||
} | } | ||||
KASSERT(len + hdrlen == m_length(m, NULL), | KASSERT(len + hdrlen == m_length(m, NULL), | ||||
("%s: mbuf chain different than expected: %d + %u != %u", | ("%s: mbuf chain different than expected: %d + %u != %u", | ||||
__func__, len, hdrlen, m_length(m, NULL))); | __func__, len, hdrlen, m_length(m, NULL))); | ||||
▲ Show 20 Lines • Show All 162 Lines • ▼ Show 20 Lines | if (TCPS_HAVEESTABLISHED(tp->t_state) && | ||||
tp->rcv_numsacks > 0) | tp->rcv_numsacks > 0) | ||||
tcp_clean_dsack_blocks(tp); | tcp_clean_dsack_blocks(tp); | ||||
/* We sent an ack clear the bbr_segs_rcvd count */ | /* We sent an ack clear the bbr_segs_rcvd count */ | ||||
bbr->output_error_seen = 0; | bbr->output_error_seen = 0; | ||||
bbr->oerror_cnt = 0; | bbr->oerror_cnt = 0; | ||||
bbr->bbr_segs_rcvd = 0; | bbr->bbr_segs_rcvd = 0; | ||||
if (len == 0) | if (len == 0) | ||||
counter_u64_add(bbr_out_size[TCP_MSS_ACCT_SNDACK], 1); | counter_u64_add(bbr_out_size[TCP_MSS_ACCT_SNDACK], 1); | ||||
else if (hw_tls) { | |||||
if (filled_all || | |||||
(len >= bbr->r_ctl.rc_pace_max_segs)) | |||||
BBR_STAT_INC(bbr_meets_tso_thresh); | |||||
else { | |||||
if (doing_tlp) { | |||||
BBR_STAT_INC(bbr_miss_tlp); | |||||
bbr_log_type_hrdwtso(tp, bbr, len, 1, what_we_can); | |||||
} else if (rsm) { | |||||
BBR_STAT_INC(bbr_miss_retran); | |||||
bbr_log_type_hrdwtso(tp, bbr, len, 2, what_we_can); | |||||
} else if ((ctf_outstanding(tp) + bbr->r_ctl.rc_pace_max_segs) > sbavail(sb)) { | |||||
BBR_STAT_INC(bbr_miss_tso_app); | |||||
bbr_log_type_hrdwtso(tp, bbr, len, 3, what_we_can); | |||||
} else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + | |||||
bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_pace_max_segs) > tp->snd_cwnd) { | |||||
BBR_STAT_INC(bbr_miss_tso_cwnd); | |||||
bbr_log_type_hrdwtso(tp, bbr, len, 4, what_we_can); | |||||
} else if ((ctf_outstanding(tp) + bbr->r_ctl.rc_pace_max_segs) > tp->snd_wnd) { | |||||
BBR_STAT_INC(bbr_miss_tso_rwnd); | |||||
bbr_log_type_hrdwtso(tp, bbr, len, 5, what_we_can); | |||||
} else { | |||||
BBR_STAT_INC(bbr_miss_unknown); | |||||
bbr_log_type_hrdwtso(tp, bbr, len, 6, what_we_can); | |||||
} | |||||
} | |||||
} | |||||
/* Do accounting for new sends */ | /* Do accounting for new sends */ | ||||
if ((len > 0) && (rsm == NULL)) { | if ((len > 0) && (rsm == NULL)) { | ||||
int idx; | int idx; | ||||
if (tp->snd_una == tp->snd_max) { | if (tp->snd_una == tp->snd_max) { | ||||
/* | /* | ||||
* Special case to match google, when | * Special case to match google, when | ||||
* nothing is in flight the delivered | * nothing is in flight the delivered | ||||
* time does get updated to the current | * time does get updated to the current | ||||
▲ Show 20 Lines • Show All 303 Lines • ▼ Show 20 Lines | #endif | ||||
if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) | if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) | ||||
tp->rcv_adv = tp->rcv_nxt + recwin; | tp->rcv_adv = tp->rcv_nxt + recwin; | ||||
tp->last_ack_sent = tp->rcv_nxt; | tp->last_ack_sent = tp->rcv_nxt; | ||||
if ((error == 0) && | if ((error == 0) && | ||||
(bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) && | (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) && | ||||
(doing_tlp == 0) && | (doing_tlp == 0) && | ||||
(tso == 0) && | (tso == 0) && | ||||
(hw_tls == 0) && | |||||
(len > 0) && | (len > 0) && | ||||
((flags & TH_RST) == 0) && | ((flags & TH_RST) == 0) && | ||||
((flags & TH_SYN) == 0) && | ((flags & TH_SYN) == 0) && | ||||
(IN_RECOVERY(tp->t_flags) == 0) && | (IN_RECOVERY(tp->t_flags) == 0) && | ||||
(bbr->rc_in_persist == 0) && | (bbr->rc_in_persist == 0) && | ||||
(tot_len < bbr->r_ctl.rc_pace_max_segs)) { | (tot_len < bbr->r_ctl.rc_pace_max_segs)) { | ||||
/* | /* | ||||
* For non-tso we need to goto again until we have sent out | * For non-tso we need to goto again until we have sent out | ||||
▲ Show 20 Lines • Show All 821 Lines • Show Last 20 Lines |