Page MenuHomeFreeBSD
Differential D48046

pci: Make generic_pcie_read_ivar non static
ClosedPublic
Actions

Authored by andrew on Dec 12 2024, 9:24 AM.
Tags
None
Referenced Files
F113669625: D48046.diff
Tue, Apr 1, 6:34 PM
Unknown Object (File)
Wed, Mar 19, 6:54 AM
Unknown Object (File)
Wed, Mar 19, 6:54 AM
Unknown Object (File)
Wed, Mar 19, 3:49 AM
Unknown Object (File)
Feb 27 2025, 5:56 PM
Unknown Object (File)
Feb 26 2025, 8:23 AM
Unknown Object (File)
Feb 21 2025, 9:25 PM
Unknown Object (File)
Feb 21 2025, 7:31 AM
View All 18 Files
Subscribers
imp

Details

Reviewers
jhb
jkim
cperciva
imp
Commits
rG506e41a1517e: pci: Make generic_pcie_read_ivar non static
rG7cafe75c8c52: pci: Make generic_pcie_read_ivar non static
Summary

Allow this to be called from attachments to allow more ivars to be
implemented.

Sponsored by: Arm Ltd

Diff Detail

Repository
rG FreeBSD src repository
Lint
Lint Not Applicable
Unit
Tests Not Applicable

Revision Contents
Changeset List

PathSize
sys/
dev/
pci/
1 line
4 lines

Diff 147886

View Options

sys/dev/pci/pci_host_generic.h

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 2,416 Lines • ▼ Show 20 Lines if (tp->t_logstate != TCP_LOG_STATE_OFF) {
log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.delivered = rack->r_ctl.rack_rs.rs_us_rtrcnt; log.u_bbr.delivered = rack->r_ctl.rack_rs.rs_us_rtrcnt;
log.u_bbr.pkts_out = rack->r_ctl.rack_rs.rs_flags; log.u_bbr.pkts_out = rack->r_ctl.rack_rs.rs_flags;
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
if (rsm) { if (rsm) {
log.u_bbr.pkt_epoch = rsm->r_start; log.u_bbr.pkt_epoch = rsm->r_start;
log.u_bbr.lost = rsm->r_end; log.u_bbr.lost = rsm->r_end;
log.u_bbr.cwnd_gain = rsm->r_rtr_cnt; log.u_bbr.cwnd_gain = rsm->r_rtr_cnt;
log.u_bbr.pacing_gain = rsm->r_flags; /* We loose any upper of the 24 bits */
log.u_bbr.pacing_gain = (uint16_t)rsm->r_flags;
} else { } else {
/* Its a SYN */ /* Its a SYN */
log.u_bbr.pkt_epoch = rack->rc_tp->iss; log.u_bbr.pkt_epoch = rack->rc_tp->iss;
log.u_bbr.lost = 0; log.u_bbr.lost = 0;
log.u_bbr.cwnd_gain = 0; log.u_bbr.cwnd_gain = 0;
log.u_bbr.pacing_gain = 0; log.u_bbr.pacing_gain = 0;
} }
/* Write out general bits of interest rrs here */ /* Write out general bits of interest rrs here */
▲ Show 20 Lines • Show All 4,233 Lines • ▼ Show 20 Lines RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) {
} else { } else {
TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, trsm, rsm, r_tnext); TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, trsm, rsm, r_tnext);
} }
rsm->r_in_tmap = 1; rsm->r_in_tmap = 1;
trsm = rsm; trsm = rsm;
if (rsm->r_flags & RACK_ACKED) if (rsm->r_flags & RACK_ACKED)
rsm->r_flags |= RACK_WAS_ACKED; rsm->r_flags |= RACK_WAS_ACKED;
rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS); rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS);
rsm->r_flags |= RACK_MUST_RXT;
} }
/* Clear the count (we just un-acked them) */ /* Clear the count (we just un-acked them) */
rack->r_ctl.rc_last_timeout_snduna = tp->snd_una; rack->r_ctl.rc_last_timeout_snduna = tp->snd_una;
rack->r_ctl.rc_sacked = 0; rack->r_ctl.rc_sacked = 0;
rack->r_ctl.rc_sacklast = NULL; rack->r_ctl.rc_sacklast = NULL;
rack->r_ctl.rc_agg_delayed = 0; rack->r_ctl.rc_agg_delayed = 0;
rack->r_early = 0; rack->r_early = 0;
rack->r_ctl.rc_agg_early = 0; rack->r_ctl.rc_agg_early = 0;
▲ Show 20 Lines • Show All 571 Lines • ▼ Show 20 Linesrack_stop_all_timers(struct tcpcb *tp)
tcp_timer_suspend(tp, TT_DELACK); tcp_timer_suspend(tp, TT_DELACK);
} }
static void static void
rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack, rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag) struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag)
{ {
int32_t idx; int32_t idx;
uint16_t stripped_flags;
rsm->r_rtr_cnt++; rsm->r_rtr_cnt++;
rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
rsm->r_dupack = 0; rsm->r_dupack = 0;
if (rsm->r_rtr_cnt > RACK_NUM_OF_RETRANS) { if (rsm->r_rtr_cnt > RACK_NUM_OF_RETRANS) {
rsm->r_rtr_cnt = RACK_NUM_OF_RETRANS; rsm->r_rtr_cnt = RACK_NUM_OF_RETRANS;
rsm->r_flags |= RACK_OVERMAX; rsm->r_flags |= RACK_OVERMAX;
} }
if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & RACK_TLP) == 0)) { if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & RACK_TLP) == 0)) {
rack->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start); rack->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start); rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
} }
idx = rsm->r_rtr_cnt - 1; idx = rsm->r_rtr_cnt - 1;
rsm->r_tim_lastsent[idx] = ts; rsm->r_tim_lastsent[idx] = ts;
/* /*
* Here we don't add in the len of send, since its already * Here we don't add in the len of send, since its already
* in snduna <->snd_max. * in snduna <->snd_max.
*/ */
rsm->r_fas = ctf_flight_size(rack->rc_tp, rsm->r_fas = ctf_flight_size(rack->rc_tp,
rack->r_ctl.rc_sacked); rack->r_ctl.rc_sacked);
stripped_flags = rsm->r_flags & ~(RACK_SENT_SP|RACK_SENT_FP);
if (rsm->r_flags & RACK_ACKED) { if (rsm->r_flags & RACK_ACKED) {
/* Problably MTU discovery messing with us */ /* Problably MTU discovery messing with us */
rsm->r_flags &= ~RACK_ACKED; rsm->r_flags &= ~RACK_ACKED;
rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
} }
if (rsm->r_in_tmap) { if (rsm->r_in_tmap) {
TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext); TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
rsm->r_in_tmap = 0; rsm->r_in_tmap = 0;
▲ Show 20 Lines • Show All 8,807 Lines • ▼ Show 20 Lines if ((slot == 0) ||
* are using old-style rack or * are using old-style rack or
* we are overriden to use the old 1ms pacing. * we are overriden to use the old 1ms pacing.
*/ */
slot = rack->r_ctl.rc_min_to; slot = rack->r_ctl.rc_min_to;
} }
rack_start_hpts_timer(rack, tp, cts, slot, len, 0); rack_start_hpts_timer(rack, tp, cts, slot, len, 0);
if (rack->r_must_retran) { if (rack->r_must_retran) {
rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start); rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start);
if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) { if ((SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) ||
((rsm->r_flags & RACK_MUST_RXT) == 0)) {
/* /*
* We have retransmitted all we need. * We have retransmitted all we need. If
* RACK_MUST_RXT is not set then we need to
* not retransmit this guy.
*/ */
rack->r_must_retran = 0; rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0; rack->r_ctl.rc_out_at_rto = 0;
if ((rsm->r_flags & RACK_MUST_RXT) == 0) {
/* Not one we should rxt */
goto failed;
} else {
/* Clear the flag */
rsm->r_flags &= ~RACK_MUST_RXT;
} }
} else {
/* Remove the flag */
rsm->r_flags &= ~RACK_MUST_RXT;
} }
}
#ifdef TCP_ACCOUNTING #ifdef TCP_ACCOUNTING
crtsc = get_cyclecount(); crtsc = get_cyclecount();
if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru; tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru;
} }
counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru); counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru);
if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val); tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
▲ Show 20 Lines • Show All 868 Lines • ▼ Show 20 Lines KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
rsm->r_start, tp->snd_una, tp, rack, rsm)); rsm->r_start, tp->snd_una, tp, rack, rsm));
sb_offset = rsm->r_start - tp->snd_una; sb_offset = rsm->r_start - tp->snd_una;
cwin = min(tp->snd_wnd, tlen); cwin = min(tp->snd_wnd, tlen);
len = cwin; len = cwin;
} }
if (rack->r_must_retran && if (rack->r_must_retran &&
(rsm == NULL)) { (rsm == NULL)) {
/* /*
* Non-Sack and we had a RTO or MTU change, we * Non-Sack and we had a RTO or Sack/non-Sack and a
* need to retransmit until we reach * MTU change, we need to retransmit until we reach
* the former snd_max (rack->r_ctl.rc_snd_max_at_rto). * the former snd_max (rack->r_ctl.rc_snd_max_at_rto).
*/ */
if (SEQ_GT(tp->snd_max, tp->snd_una)) { if (SEQ_GT(tp->snd_max, tp->snd_una)) {
int sendwin, flight; int sendwin, flight;
sendwin = min(tp->snd_wnd, tp->snd_cwnd); sendwin = min(tp->snd_wnd, tp->snd_cwnd);
flight = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto); flight = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto);
if (flight >= sendwin) { if (flight >= sendwin) {
so = inp->inp_socket; so = inp->inp_socket;
sb = &so->so_snd; sb = &so->so_snd;
goto just_return_nolock; goto just_return_nolock;
} }
rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
if (rsm == NULL) { if (rsm == NULL) {
/* TSNH */ /* TSNH */
rack->r_must_retran = 0; rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0; rack->r_ctl.rc_out_at_rto = 0;
rack->r_must_retran = 0; rack->r_must_retran = 0;
so = inp->inp_socket; so = inp->inp_socket;
sb = &so->so_snd; sb = &so->so_snd;
goto just_return_nolock; goto just_return_nolock;
} }
if ((rsm->r_flags & RACK_MUST_RXT) == 0) {
/* It does not have the flag, we are done */
rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0;
} else {
sack_rxmit = 1; sack_rxmit = 1;
len = rsm->r_end - rsm->r_start; len = rsm->r_end - rsm->r_start;
sendalot = 0; sendalot = 0;
sb_offset = rsm->r_start - tp->snd_una; sb_offset = rsm->r_start - tp->snd_una;
if (len >= segsiz) if (len >= segsiz)
len = segsiz; len = segsiz;
/*
* Delay removing the flag RACK_MUST_RXT so
* that the fastpath for retransmit will
* work with this rsm.
*/
}
} else { } else {
/* We must be done if there is nothing outstanding */ /* We must be done if there is nothing outstanding */
rack->r_must_retran = 0; rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0; rack->r_ctl.rc_out_at_rto = 0;
} }
} }
/* /*
* Enforce a connection sendmap count limit if set * Enforce a connection sendmap count limit if set
Show All 30 Lines#endif
if (rsm && rack->r_fsb_inited && rack_use_rsm_rfo && if (rsm && rack->r_fsb_inited && rack_use_rsm_rfo &&
((rsm->r_flags & RACK_HAS_FIN) == 0)) { ((rsm->r_flags & RACK_HAS_FIN) == 0)) {
int ret; int ret;
ret = rack_fast_rsm_output(tp, rack, rsm, ts_val, cts, ms_cts, &tv, len, doing_tlp); ret = rack_fast_rsm_output(tp, rack, rsm, ts_val, cts, ms_cts, &tv, len, doing_tlp);
if (ret == 0) if (ret == 0)
return (0); return (0);
} }
if (rsm && (rsm->r_flags & RACK_MUST_RXT)) {
/*
* Clear the flag in prep for the send
* note that if we can't get an mbuf
* and fail, we won't retransmit this
* rsm but that should be ok (its rare).
*/
rsm->r_flags &= ~RACK_MUST_RXT;
}
so = inp->inp_socket; so = inp->inp_socket;
sb = &so->so_snd; sb = &so->so_snd;
if (do_a_prefetch == 0) { if (do_a_prefetch == 0) {
kern_prefetch(sb, &do_a_prefetch); kern_prefetch(sb, &do_a_prefetch);
do_a_prefetch = 1; do_a_prefetch = 1;
} }
#ifdef NETFLIX_SHARED_CWND #ifdef NETFLIX_SHARED_CWND
if ((tp->t_flags2 & TF2_TCP_SCWND_ALLOWED) && if ((tp->t_flags2 & TF2_TCP_SCWND_ALLOWED) &&
▲ Show 20 Lines • Show All 2,217 Lines • ▼ Show 20 Lines
static void static void
rack_mtu_change(struct tcpcb *tp) rack_mtu_change(struct tcpcb *tp)
{ {
/* /*
* The MSS may have changed * The MSS may have changed
*/ */
struct tcp_rack *rack; struct tcp_rack *rack;
struct rack_sendmap *rsm;
rack = (struct tcp_rack *)tp->t_fb_ptr; rack = (struct tcp_rack *)tp->t_fb_ptr;
if (rack->r_ctl.rc_pace_min_segs != ctf_fixed_maxseg(tp)) { if (rack->r_ctl.rc_pace_min_segs != ctf_fixed_maxseg(tp)) {
/* /*
* The MTU has changed we need to resend everything * The MTU has changed we need to resend everything
* since all we have sent is lost. We first fix * since all we have sent is lost. We first fix
* up the mtu though. * up the mtu though.
*/ */
rack_set_pace_segments(tp, rack, __LINE__, NULL); rack_set_pace_segments(tp, rack, __LINE__, NULL);
/* We treat this like a full retransmit timeout without the cwnd adjustment */ /* We treat this like a full retransmit timeout without the cwnd adjustment */
rack_remxt_tmr(tp); rack_remxt_tmr(tp);
rack->r_fast_output = 0; rack->r_fast_output = 0;
rack->r_ctl.rc_out_at_rto = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto = ctf_flight_size(tp,
rack->r_ctl.rc_sacked); rack->r_ctl.rc_sacked);
rack->r_ctl.rc_snd_max_at_rto = tp->snd_max; rack->r_ctl.rc_snd_max_at_rto = tp->snd_max;
rack->r_must_retran = 1; rack->r_must_retran = 1;
/* Mark all inflight to needing to be rxt'd */
TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
rsm->r_flags |= RACK_MUST_RXT;
}
} }
sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una); sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
/* We don't use snd_nxt to retransmit */ /* We don't use snd_nxt to retransmit */
tp->snd_nxt = tp->snd_max; tp->snd_nxt = tp->snd_max;
} }
static int static int
rack_set_profile(struct tcp_rack *rack, int prof) rack_set_profile(struct tcp_rack *rack, int prof)
▲ Show 20 Lines • Show All 1,468 Lines • Show Last 20 Lines
View Options

sys/dev/pci/pci_host_generic.c

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 2,416 Lines • ▼ Show 20 Lines if (tp->t_logstate != TCP_LOG_STATE_OFF) {
log.u_bbr.timeStamp = tcp_get_usecs(&tv); log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.delivered = rack->r_ctl.rack_rs.rs_us_rtrcnt; log.u_bbr.delivered = rack->r_ctl.rack_rs.rs_us_rtrcnt;
log.u_bbr.pkts_out = rack->r_ctl.rack_rs.rs_flags; log.u_bbr.pkts_out = rack->r_ctl.rack_rs.rs_flags;
log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
if (rsm) { if (rsm) {
log.u_bbr.pkt_epoch = rsm->r_start; log.u_bbr.pkt_epoch = rsm->r_start;
log.u_bbr.lost = rsm->r_end; log.u_bbr.lost = rsm->r_end;
log.u_bbr.cwnd_gain = rsm->r_rtr_cnt; log.u_bbr.cwnd_gain = rsm->r_rtr_cnt;
log.u_bbr.pacing_gain = rsm->r_flags; /* We loose any upper of the 24 bits */
log.u_bbr.pacing_gain = (uint16_t)rsm->r_flags;
} else { } else {
/* Its a SYN */ /* Its a SYN */
log.u_bbr.pkt_epoch = rack->rc_tp->iss; log.u_bbr.pkt_epoch = rack->rc_tp->iss;
log.u_bbr.lost = 0; log.u_bbr.lost = 0;
log.u_bbr.cwnd_gain = 0; log.u_bbr.cwnd_gain = 0;
log.u_bbr.pacing_gain = 0; log.u_bbr.pacing_gain = 0;
} }
/* Write out general bits of interest rrs here */ /* Write out general bits of interest rrs here */
▲ Show 20 Lines • Show All 4,233 Lines • ▼ Show 20 Lines RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) {
} else { } else {
TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, trsm, rsm, r_tnext); TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, trsm, rsm, r_tnext);
} }
rsm->r_in_tmap = 1; rsm->r_in_tmap = 1;
trsm = rsm; trsm = rsm;
if (rsm->r_flags & RACK_ACKED) if (rsm->r_flags & RACK_ACKED)
rsm->r_flags |= RACK_WAS_ACKED; rsm->r_flags |= RACK_WAS_ACKED;
rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS); rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS);
rsm->r_flags |= RACK_MUST_RXT;
} }
/* Clear the count (we just un-acked them) */ /* Clear the count (we just un-acked them) */
rack->r_ctl.rc_last_timeout_snduna = tp->snd_una; rack->r_ctl.rc_last_timeout_snduna = tp->snd_una;
rack->r_ctl.rc_sacked = 0; rack->r_ctl.rc_sacked = 0;
rack->r_ctl.rc_sacklast = NULL; rack->r_ctl.rc_sacklast = NULL;
rack->r_ctl.rc_agg_delayed = 0; rack->r_ctl.rc_agg_delayed = 0;
rack->r_early = 0; rack->r_early = 0;
rack->r_ctl.rc_agg_early = 0; rack->r_ctl.rc_agg_early = 0;
▲ Show 20 Lines • Show All 571 Lines • ▼ Show 20 Linesrack_stop_all_timers(struct tcpcb *tp)
tcp_timer_suspend(tp, TT_DELACK); tcp_timer_suspend(tp, TT_DELACK);
} }
static void static void
rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack, rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag) struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag)
{ {
int32_t idx; int32_t idx;
uint16_t stripped_flags;
rsm->r_rtr_cnt++; rsm->r_rtr_cnt++;
rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
rsm->r_dupack = 0; rsm->r_dupack = 0;
if (rsm->r_rtr_cnt > RACK_NUM_OF_RETRANS) { if (rsm->r_rtr_cnt > RACK_NUM_OF_RETRANS) {
rsm->r_rtr_cnt = RACK_NUM_OF_RETRANS; rsm->r_rtr_cnt = RACK_NUM_OF_RETRANS;
rsm->r_flags |= RACK_OVERMAX; rsm->r_flags |= RACK_OVERMAX;
} }
if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & RACK_TLP) == 0)) { if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & RACK_TLP) == 0)) {
rack->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start); rack->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start); rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
} }
idx = rsm->r_rtr_cnt - 1; idx = rsm->r_rtr_cnt - 1;
rsm->r_tim_lastsent[idx] = ts; rsm->r_tim_lastsent[idx] = ts;
/* /*
* Here we don't add in the len of send, since its already * Here we don't add in the len of send, since its already
* in snduna <->snd_max. * in snduna <->snd_max.
*/ */
rsm->r_fas = ctf_flight_size(rack->rc_tp, rsm->r_fas = ctf_flight_size(rack->rc_tp,
rack->r_ctl.rc_sacked); rack->r_ctl.rc_sacked);
stripped_flags = rsm->r_flags & ~(RACK_SENT_SP|RACK_SENT_FP);
if (rsm->r_flags & RACK_ACKED) { if (rsm->r_flags & RACK_ACKED) {
/* Problably MTU discovery messing with us */ /* Problably MTU discovery messing with us */
rsm->r_flags &= ~RACK_ACKED; rsm->r_flags &= ~RACK_ACKED;
rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
} }
if (rsm->r_in_tmap) { if (rsm->r_in_tmap) {
TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext); TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
rsm->r_in_tmap = 0; rsm->r_in_tmap = 0;
▲ Show 20 Lines • Show All 8,807 Lines • ▼ Show 20 Lines if ((slot == 0) ||
* are using old-style rack or * are using old-style rack or
* we are overriden to use the old 1ms pacing. * we are overriden to use the old 1ms pacing.
*/ */
slot = rack->r_ctl.rc_min_to; slot = rack->r_ctl.rc_min_to;
} }
rack_start_hpts_timer(rack, tp, cts, slot, len, 0); rack_start_hpts_timer(rack, tp, cts, slot, len, 0);
if (rack->r_must_retran) { if (rack->r_must_retran) {
rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start); rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start);
if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) { if ((SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) ||
((rsm->r_flags & RACK_MUST_RXT) == 0)) {
/* /*
* We have retransmitted all we need. * We have retransmitted all we need. If
* RACK_MUST_RXT is not set then we need to
* not retransmit this guy.
*/ */
rack->r_must_retran = 0; rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0; rack->r_ctl.rc_out_at_rto = 0;
if ((rsm->r_flags & RACK_MUST_RXT) == 0) {
/* Not one we should rxt */
goto failed;
} else {
/* Clear the flag */
rsm->r_flags &= ~RACK_MUST_RXT;
} }
} else {
/* Remove the flag */
rsm->r_flags &= ~RACK_MUST_RXT;
} }
}
#ifdef TCP_ACCOUNTING #ifdef TCP_ACCOUNTING
crtsc = get_cyclecount(); crtsc = get_cyclecount();
if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru; tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru;
} }
counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru); counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru);
if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val); tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
▲ Show 20 Lines • Show All 868 Lines • ▼ Show 20 Lines KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
rsm->r_start, tp->snd_una, tp, rack, rsm)); rsm->r_start, tp->snd_una, tp, rack, rsm));
sb_offset = rsm->r_start - tp->snd_una; sb_offset = rsm->r_start - tp->snd_una;
cwin = min(tp->snd_wnd, tlen); cwin = min(tp->snd_wnd, tlen);
len = cwin; len = cwin;
} }
if (rack->r_must_retran && if (rack->r_must_retran &&
(rsm == NULL)) { (rsm == NULL)) {
/* /*
* Non-Sack and we had a RTO or MTU change, we * Non-Sack and we had a RTO or Sack/non-Sack and a
* need to retransmit until we reach * MTU change, we need to retransmit until we reach
* the former snd_max (rack->r_ctl.rc_snd_max_at_rto). * the former snd_max (rack->r_ctl.rc_snd_max_at_rto).
*/ */
if (SEQ_GT(tp->snd_max, tp->snd_una)) { if (SEQ_GT(tp->snd_max, tp->snd_una)) {
int sendwin, flight; int sendwin, flight;
sendwin = min(tp->snd_wnd, tp->snd_cwnd); sendwin = min(tp->snd_wnd, tp->snd_cwnd);
flight = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto); flight = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto);
if (flight >= sendwin) { if (flight >= sendwin) {
so = inp->inp_socket; so = inp->inp_socket;
sb = &so->so_snd; sb = &so->so_snd;
goto just_return_nolock; goto just_return_nolock;
} }
rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
if (rsm == NULL) { if (rsm == NULL) {
/* TSNH */ /* TSNH */
rack->r_must_retran = 0; rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0; rack->r_ctl.rc_out_at_rto = 0;
rack->r_must_retran = 0; rack->r_must_retran = 0;
so = inp->inp_socket; so = inp->inp_socket;
sb = &so->so_snd; sb = &so->so_snd;
goto just_return_nolock; goto just_return_nolock;
} }
if ((rsm->r_flags & RACK_MUST_RXT) == 0) {
/* It does not have the flag, we are done */
rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0;
} else {
sack_rxmit = 1; sack_rxmit = 1;
len = rsm->r_end - rsm->r_start; len = rsm->r_end - rsm->r_start;
sendalot = 0; sendalot = 0;
sb_offset = rsm->r_start - tp->snd_una; sb_offset = rsm->r_start - tp->snd_una;
if (len >= segsiz) if (len >= segsiz)
len = segsiz; len = segsiz;
/*
* Delay removing the flag RACK_MUST_RXT so
* that the fastpath for retransmit will
* work with this rsm.
*/
}
} else { } else {
/* We must be done if there is nothing outstanding */ /* We must be done if there is nothing outstanding */
rack->r_must_retran = 0; rack->r_must_retran = 0;
rack->r_ctl.rc_out_at_rto = 0; rack->r_ctl.rc_out_at_rto = 0;
} }
} }
/* /*
* Enforce a connection sendmap count limit if set * Enforce a connection sendmap count limit if set
Show All 30 Lines#endif
if (rsm && rack->r_fsb_inited && rack_use_rsm_rfo && if (rsm && rack->r_fsb_inited && rack_use_rsm_rfo &&
((rsm->r_flags & RACK_HAS_FIN) == 0)) { ((rsm->r_flags & RACK_HAS_FIN) == 0)) {
int ret; int ret;
ret = rack_fast_rsm_output(tp, rack, rsm, ts_val, cts, ms_cts, &tv, len, doing_tlp); ret = rack_fast_rsm_output(tp, rack, rsm, ts_val, cts, ms_cts, &tv, len, doing_tlp);
if (ret == 0) if (ret == 0)
return (0); return (0);
} }
if (rsm && (rsm->r_flags & RACK_MUST_RXT)) {
/*
* Clear the flag in prep for the send
* note that if we can't get an mbuf
* and fail, we won't retransmit this
* rsm but that should be ok (its rare).
*/
rsm->r_flags &= ~RACK_MUST_RXT;
}
so = inp->inp_socket; so = inp->inp_socket;
sb = &so->so_snd; sb = &so->so_snd;
if (do_a_prefetch == 0) { if (do_a_prefetch == 0) {
kern_prefetch(sb, &do_a_prefetch); kern_prefetch(sb, &do_a_prefetch);
do_a_prefetch = 1; do_a_prefetch = 1;
} }
#ifdef NETFLIX_SHARED_CWND #ifdef NETFLIX_SHARED_CWND
if ((tp->t_flags2 & TF2_TCP_SCWND_ALLOWED) && if ((tp->t_flags2 & TF2_TCP_SCWND_ALLOWED) &&
▲ Show 20 Lines • Show All 2,217 Lines • ▼ Show 20 Lines
static void static void
rack_mtu_change(struct tcpcb *tp) rack_mtu_change(struct tcpcb *tp)
{ {
/* /*
* The MSS may have changed * The MSS may have changed
*/ */
struct tcp_rack *rack; struct tcp_rack *rack;
struct rack_sendmap *rsm;
rack = (struct tcp_rack *)tp->t_fb_ptr; rack = (struct tcp_rack *)tp->t_fb_ptr;
if (rack->r_ctl.rc_pace_min_segs != ctf_fixed_maxseg(tp)) { if (rack->r_ctl.rc_pace_min_segs != ctf_fixed_maxseg(tp)) {
/* /*
* The MTU has changed we need to resend everything * The MTU has changed we need to resend everything
* since all we have sent is lost. We first fix * since all we have sent is lost. We first fix
* up the mtu though. * up the mtu though.
*/ */
rack_set_pace_segments(tp, rack, __LINE__, NULL); rack_set_pace_segments(tp, rack, __LINE__, NULL);
/* We treat this like a full retransmit timeout without the cwnd adjustment */ /* We treat this like a full retransmit timeout without the cwnd adjustment */
rack_remxt_tmr(tp); rack_remxt_tmr(tp);
rack->r_fast_output = 0; rack->r_fast_output = 0;
rack->r_ctl.rc_out_at_rto = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto = ctf_flight_size(tp,
rack->r_ctl.rc_sacked); rack->r_ctl.rc_sacked);
rack->r_ctl.rc_snd_max_at_rto = tp->snd_max; rack->r_ctl.rc_snd_max_at_rto = tp->snd_max;
rack->r_must_retran = 1; rack->r_must_retran = 1;
/* Mark all inflight to needing to be rxt'd */
TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
rsm->r_flags |= RACK_MUST_RXT;
}
} }
sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una); sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
/* We don't use snd_nxt to retransmit */ /* We don't use snd_nxt to retransmit */
tp->snd_nxt = tp->snd_max; tp->snd_nxt = tp->snd_max;
} }
static int static int
rack_set_profile(struct tcp_rack *rack, int prof) rack_set_profile(struct tcp_rack *rack, int prof)
▲ Show 20 Lines • Show All 1,468 Lines • Show Last 20 Lines