Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/tcp_stacks/bbr.c
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
| Show First 20 Lines • Show All 436 Lines • ▼ Show 20 Lines | |||||
| static inline uint64_t bbr_get_bw(struct tcp_bbr *bbr); | static inline uint64_t bbr_get_bw(struct tcp_bbr *bbr); | ||||
| /* | /* | ||||
| * Static defintions we need for forward declarations. | * Static defintions we need for forward declarations. | ||||
| */ | */ | ||||
| static uint32_t | static uint32_t | ||||
| bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, | bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, | ||||
| uint32_t useconds_time, uint64_t bw); | uint32_t useconds_time, uint64_t bw); | ||||
| static uint32_t | static uint32_t | ||||
| bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain); | bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain); | ||||
| static void | static void | ||||
| bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win); | bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win); | ||||
| static void | static void | ||||
| bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses); | bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses); | ||||
| static void | static void | ||||
| bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line, | bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line, | ||||
| int dolog); | int dolog); | ||||
| static uint32_t | static uint32_t | ||||
| bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain); | bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain); | ||||
| static void | static void | ||||
| bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, | bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, | ||||
| int32_t pkt_epoch, uint32_t losses); | int32_t pkt_epoch, uint32_t losses); | ||||
| static uint32_t | static uint32_t | ||||
| bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm); | bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, | ||||
| static uint32_t bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp); | struct bbr_sendmap *rsm); | ||||
| static uint32_t | static uint32_t | ||||
| bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp); | |||||
| static uint32_t | |||||
| bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, | bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, | ||||
| struct bbr_sendmap *rsm, uint32_t srtt, | struct bbr_sendmap *rsm, uint32_t srtt, uint32_t cts); | ||||
| uint32_t cts); | |||||
| static void | static void | ||||
| bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, | bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, | ||||
| int32_t line); | int32_t line); | ||||
| static void | static void | ||||
| bbr_set_state_target(struct tcp_bbr *bbr, int line); | bbr_set_state_target(struct tcp_bbr *bbr, int line); | ||||
| static void | static void | ||||
| bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line); | bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line); | ||||
| static void | static void | ||||
| bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line); | bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, | ||||
| int event, int line); | |||||
| static void | static void | ||||
| tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts); | tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts); | ||||
| static void | static void | ||||
| bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts); | bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts); | ||||
| static void | static void | ||||
| bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, uint32_t rtt, | bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, | ||||
| uint32_t line, uint8_t is_start, uint16_t set); | uint32_t rtt, uint32_t line, uint8_t is_start, | ||||
| uint16_t set); | |||||
| static struct bbr_sendmap * | static struct bbr_sendmap * | ||||
| bbr_find_lowest_rsm(struct tcp_bbr *bbr); | bbr_find_lowest_rsm(struct tcp_bbr *bbr); | ||||
| static __inline uint32_t | static __inline uint32_t | ||||
| bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type); | bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type); | ||||
| static void | static void | ||||
| bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which); | bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, | ||||
| uint8_t which); | |||||
| static void | static void | ||||
| bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt, | bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, | ||||
| uint32_t time_since_sent, uint32_t srtt, | |||||
| uint32_t thresh, uint32_t to); | uint32_t thresh, uint32_t to); | ||||
| static void | static void | ||||
| bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag); | bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag); | ||||
| static void | static void | ||||
| bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, | bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, | ||||
| uint32_t del_by, uint32_t cts, uint32_t sloton, uint32_t prev_delay); | uint32_t del_by, uint32_t cts, uint32_t sloton, | ||||
| uint32_t prev_delay); | |||||
| static void | static void | ||||
| bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, | bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, | ||||
| uint32_t cts, int32_t line); | int32_t line); | ||||
| static void | static void | ||||
| bbr_stop_all_timers(struct tcpcb *tp); | bbr_stop_all_timers(struct tcpcb *tp); | ||||
| static void | static void | ||||
| bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts); | bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts); | ||||
| static void | static void | ||||
| bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts); | bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts); | ||||
| static void | static void | ||||
| bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts); | bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts); | ||||
| static void | static void | ||||
| bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len, | bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len, | ||||
| uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod); | uint32_t cts, uint32_t usecs, uint64_t bw, | ||||
| uint32_t override, int mod); | |||||
| static int | static int | ||||
| bbr_ctloutput(struct inpcb *inp, struct sockopt *sopt); | bbr_ctloutput(struct inpcb *inp, struct sockopt *sopt); | ||||
| static inline uint8_t | static inline uint8_t | ||||
| bbr_state_val(struct tcp_bbr *bbr) | bbr_state_val(struct tcp_bbr *bbr) | ||||
| { | { | ||||
| return(bbr->rc_bbr_substate); | return(bbr->rc_bbr_substate); | ||||
| } | } | ||||
| static inline uint32_t | static inline uint32_t | ||||
| get_min_cwnd(struct tcp_bbr *bbr) | get_min_cwnd(struct tcp_bbr *bbr) | ||||
| { | { | ||||
| int mss; | int mss; | ||||
| mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); | mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), | ||||
| bbr->r_ctl.rc_pace_max_segs); | |||||
| if (bbr_get_rtt(bbr, BBR_RTT_PROP) < BBR_HIGH_SPEED) | if (bbr_get_rtt(bbr, BBR_RTT_PROP) < BBR_HIGH_SPEED) | ||||
| return (bbr_cwnd_min_val_hs * mss); | return (bbr_cwnd_min_val_hs * mss); | ||||
| else | else | ||||
| return (bbr_cwnd_min_val * mss); | return (bbr_cwnd_min_val * mss); | ||||
| } | } | ||||
| static uint32_t | static uint32_t | ||||
| bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr) | bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr) | ||||
| ▲ Show 20 Lines • Show All 7,198 Lines • ▼ Show 20 Lines | if (SEQ_GT(th->th_ack, tp->snd_max)) { | ||||
| bbr->r_wanted_output = 1; | bbr->r_wanted_output = 1; | ||||
| return (1); | return (1); | ||||
| } | } | ||||
| if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) { | if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) { | ||||
| /* Process the ack */ | /* Process the ack */ | ||||
| if (bbr->rc_in_persist) | if (bbr->rc_in_persist) | ||||
| tp->t_rxtshift = 0; | tp->t_rxtshift = 0; | ||||
| if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd)) | if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd)) | ||||
| bbr_strike_dupack(bbr); | bbr_strike_dupack(bbr); | ||||
| sack_changed = bbr_log_ack(tp, to, th, &prev_acked); | sack_changed = bbr_log_ack(tp, to, th, &prev_acked); | ||||
| } | } | ||||
| bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost)); | bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost)); | ||||
| if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { | if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { | ||||
| /* | /* | ||||
| * Old ack, behind the last one rcv'd or a duplicate ack | * Old ack, behind the last one rcv'd or a duplicate ack | ||||
| * with SACK info. | * with SACK info. | ||||
| */ | */ | ||||
| ▲ Show 20 Lines • Show All 7,150 Lines • Show Last 20 Lines | |||||