Page MenuHomeFreeBSD
Differential D24340 Diff 70474 sys/netinet/tcp_stacks/bbr.c

Changeset View

Standalone View

View Options

sys/netinet/tcp_stacks/bbr.c

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 5,595 Lines • ▼ Show 20 Lines
static void static void
bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts) bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts)
{ {
const struct tcp_hwrate_limit_table *nrte; const struct tcp_hwrate_limit_table *nrte;
int error, rate = -1; int error, rate = -1;
if (bbr->r_ctl.crte == NULL) if (bbr->r_ctl.crte == NULL)
return; return;
if ((bbr->rc_inp->inp_route.ro_rt == NULL) || if ((bbr->rc_inp->inp_route.ro_nh == NULL) ||
(bbr->rc_inp->inp_route.ro_rt->rt_ifp == NULL)) { (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) {
/* Lost our routes? */ /* Lost our routes? */
/* Clear the way for a re-attempt */ /* Clear the way for a re-attempt */
bbr->bbr_attempt_hdwr_pace = 0; bbr->bbr_attempt_hdwr_pace = 0;
lost_rate: lost_rate:
bbr->gain_is_limited = 0; bbr->gain_is_limited = 0;
bbr->skip_gain = 0; bbr->skip_gain = 0;
bbr->bbr_hdrw_pacing = 0; bbr->bbr_hdrw_pacing = 0;
counter_u64_add(bbr_flows_whdwr_pacing, -1); counter_u64_add(bbr_flows_whdwr_pacing, -1);
counter_u64_add(bbr_flows_nohdwr_pacing, 1); counter_u64_add(bbr_flows_nohdwr_pacing, 1);
tcp_bbr_tso_size_check(bbr, cts); tcp_bbr_tso_size_check(bbr, cts);
return; return;
} }
rate = bbr_get_hardware_rate(bbr); rate = bbr_get_hardware_rate(bbr);
nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte, nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,
bbr->rc_tp, bbr->rc_tp,
bbr->rc_inp->inp_route.ro_rt->rt_ifp, bbr->rc_inp->inp_route.ro_nh->nh_ifp,
rate, rate,
(RS_PACING_GEQ|RS_PACING_SUB_OK), (RS_PACING_GEQ|RS_PACING_SUB_OK),
&error); &error);
if (nrte == NULL) { if (nrte == NULL) {
goto lost_rate; goto lost_rate;
} }
if (nrte != bbr->r_ctl.crte) { if (nrte != bbr->r_ctl.crte) {
bbr->r_ctl.crte = nrte; bbr->r_ctl.crte = nrte;
▲ Show 20 Lines • Show All 8,340 Lines • ▼ Show 20 Lines if (isipv6) {
TCP_PROBE5(send, NULL, tp, ip6, tp, th); TCP_PROBE5(send, NULL, tp, ip6, tp, th);
/* TODO: IPv6 IP6TOS_ECT bit on */ /* TODO: IPv6 IP6TOS_ECT bit on */
error = ip6_output(m, inp->in6p_outputopts, error = ip6_output(m, inp->in6p_outputopts,
&inp->inp_route6, &inp->inp_route6,
((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0),
NULL, NULL, inp); NULL, NULL, inp);
if (error == EMSGSIZE && inp->inp_route6.ro_rt != NULL) if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)
mtu = inp->inp_route6.ro_rt->rt_mtu; mtu = inp->inp_route6.ro_nh->nh_mtu;
} }
#endif /* INET6 */ #endif /* INET6 */
#if defined(INET) && defined(INET6) #if defined(INET) && defined(INET6)
else else
#endif #endif
#ifdef INET #ifdef INET
{ {
ip->ip_len = htons(m->m_pkthdr.len); ip->ip_len = htons(m->m_pkthdr.len);
Show All 23 Lines#endif /* INET6 */
if (tp->t_state == TCPS_SYN_SENT) if (tp->t_state == TCPS_SYN_SENT)
TCP_PROBE5(connect__request, NULL, tp, ip, tp, th); TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
TCP_PROBE5(send, NULL, tp, ip, tp, th); TCP_PROBE5(send, NULL, tp, ip, tp, th);
error = ip_output(m, inp->inp_options, &inp->inp_route, error = ip_output(m, inp->inp_options, &inp->inp_route,
((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0, ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0,
inp); inp);
if (error == EMSGSIZE && inp->inp_route.ro_rt != NULL) if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)
mtu = inp->inp_route.ro_rt->rt_mtu; mtu = inp->inp_route.ro_nh->nh_mtu;
} }
#endif /* INET */ #endif /* INET */
out: out:
if (lgb) { if (lgb) {
lgb->tlb_errno = error; lgb->tlb_errno = error;
lgb = NULL; lgb = NULL;
} }
▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines
#endif #endif
} }
KMOD_TCPSTAT_INC(tcps_sndtotal); KMOD_TCPSTAT_INC(tcps_sndtotal);
if ((bbr->bbr_hdw_pace_ena) && if ((bbr->bbr_hdw_pace_ena) &&
(bbr->bbr_attempt_hdwr_pace == 0) && (bbr->bbr_attempt_hdwr_pace == 0) &&
(bbr->rc_past_init_win) && (bbr->rc_past_init_win) &&
(bbr->rc_bbr_state != BBR_STATE_STARTUP) && (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
(get_filter_value(&bbr->r_ctl.rc_delrate)) && (get_filter_value(&bbr->r_ctl.rc_delrate)) &&
(inp->inp_route.ro_rt && (inp->inp_route.ro_nh &&
inp->inp_route.ro_rt->rt_ifp)) { inp->inp_route.ro_nh->nh_ifp)) {
/* /*
* We are past the initial window and * We are past the initial window and
* have at least one measurement so we * have at least one measurement so we
* could use hardware pacing if its available. * could use hardware pacing if its available.
* We have an interface and we have not attempted * We have an interface and we have not attempted
* to setup hardware pacing, lets try to now. * to setup hardware pacing, lets try to now.
*/ */
uint64_t rate_wanted; uint64_t rate_wanted;
int err = 0; int err = 0;
rate_wanted = bbr_get_hardware_rate(bbr); rate_wanted = bbr_get_hardware_rate(bbr);
bbr->bbr_attempt_hdwr_pace = 1; bbr->bbr_attempt_hdwr_pace = 1;
bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp, bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,
inp->inp_route.ro_rt->rt_ifp, inp->inp_route.ro_nh->nh_ifp,
rate_wanted, rate_wanted,
(RS_PACING_GEQ|RS_PACING_SUB_OK), (RS_PACING_GEQ|RS_PACING_SUB_OK),
&err); &err);
if (bbr->r_ctl.crte) { if (bbr->r_ctl.crte) {
bbr_type_log_hdwr_pacing(bbr, bbr_type_log_hdwr_pacing(bbr,
bbr->r_ctl.crte->ptbl->rs_ifp, bbr->r_ctl.crte->ptbl->rs_ifp,
rate_wanted, rate_wanted,
bbr->r_ctl.crte->rate, bbr->r_ctl.crte->rate,
Show All 10 Lines if (bbr->r_ctl.crte) {
} else { } else {
/* We are good */ /* We are good */
bbr->gain_is_limited = 0; bbr->gain_is_limited = 0;
bbr->skip_gain = 0; bbr->skip_gain = 0;
} }
tcp_bbr_tso_size_check(bbr, cts); tcp_bbr_tso_size_check(bbr, cts);
} else { } else {
bbr_type_log_hdwr_pacing(bbr, bbr_type_log_hdwr_pacing(bbr,
inp->inp_route.ro_rt->rt_ifp, inp->inp_route.ro_nh->nh_ifp,
rate_wanted, rate_wanted,
0, 0,
__LINE__, cts, err); __LINE__, cts, err);
BBR_STAT_INC(bbr_hdwr_rl_add_fail); BBR_STAT_INC(bbr_hdwr_rl_add_fail);
} }
} }
if (bbr->bbr_hdrw_pacing) { if (bbr->bbr_hdrw_pacing) {
/* /*
* Worry about cases where the route * Worry about cases where the route
* changes or something happened that we * changes or something happened that we
* lost our hardware pacing possibly during * lost our hardware pacing possibly during
* the last ip_output call. * the last ip_output call.
*/ */
if (inp->inp_snd_tag == NULL) { if (inp->inp_snd_tag == NULL) {
/* A change during ip output disabled hw pacing? */ /* A change during ip output disabled hw pacing? */
bbr->bbr_hdrw_pacing = 0; bbr->bbr_hdrw_pacing = 0;
} else if ((inp->inp_route.ro_rt == NULL) || } else if ((inp->inp_route.ro_nh == NULL) ||
(inp->inp_route.ro_rt->rt_ifp != inp->inp_snd_tag->ifp)) { (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) {
/* /*
* We had an interface or route change, * We had an interface or route change,
* detach from the current hdwr pacing * detach from the current hdwr pacing
* and setup to re-attempt next go * and setup to re-attempt next go
* round. * round.
*/ */
bbr->bbr_hdrw_pacing = 0; bbr->bbr_hdrw_pacing = 0;
bbr->bbr_attempt_hdwr_pace = 0; bbr->bbr_attempt_hdwr_pace = 0;
▲ Show 20 Lines • Show All 832 Lines • Show Last 20 Lines