Page MenuHomeFreeBSD
Files F163437427

D40391.id122756.diff
No OneTemporary
Actions

D40391.id122756.diff
View Options

Index: sys/netinet/tcp.h
===================================================================
--- sys/netinet/tcp.h
+++ sys/netinet/tcp.h
@@ -499,6 +499,8 @@
#define TCP_HYBRID_PACING_ENABLE 0x0010 /* We are enabling hybrid pacing else disable */
#define TCP_HYBRID_PACING_S_MSS 0x0020 /* Clent wants us to set the mss overriding gp est in CU */
#define TCP_HYBRID_PACING_SETMSS 0x1000 /* Internal flag that tellsus we set the mss on this entry */
+#define TCP_HYBRID_PACING_WASSET 0x2000 /* We init to this to know if a hybrid command was issued */
+
struct tcp_hybrid_req {
struct tcp_snd_req req;
Index: sys/netinet/tcp_stacks/rack.c
===================================================================
--- sys/netinet/tcp_stacks/rack.c
+++ sys/netinet/tcp_stacks/rack.c
@@ -320,15 +320,28 @@
/* Part of pacing */
static int32_t rack_max_per_above = 30; /* When we go to increment stop if above 100+this% */
-/* Timely information */
-/* Combine these two gives the range of 'no change' to bw */
-/* ie the up/down provide the upper and lower bound */
+/* Timely information:
+ *
+ * Here we have various control parameters on how
+ * timely may change the multiplier. rack_gain_p5_ub
+ * is associated with timely but not directly influencing
+ * the rate decision like the other variables. It controls
+ * the way fill-cw interacts with timely and caps how much
+ * timely can boost the fill-cw b/w.
+ *
+ * The other values are various boost/shrink numbers as well
+ * as potential caps when adjustments are made to the timely
+ * gain (returned by rack_get_output_gain(). Remember too that
+ * the gain returned can be overriden by other factors such as
+ * probeRTT as well as fixed-rate-pacing.
+ */
+static int32_t rack_gain_p5_ub = 250;
static int32_t rack_gp_per_bw_mul_up = 2; /* 2% */
static int32_t rack_gp_per_bw_mul_down = 4; /* 4% */
static int32_t rack_gp_rtt_maxmul = 3; /* 3 x maxmin */
static int32_t rack_gp_rtt_minmul = 1; /* minrtt + (minrtt/mindiv) is lower rtt */
static int32_t rack_gp_rtt_mindiv = 4; /* minrtt + (minrtt * minmul/mindiv) is lower rtt */
-static int32_t rack_gp_decrease_per = 20; /* 20% decrease in multiplier */
+static int32_t rack_gp_decrease_per = 80; /* Beta value of timely decrease (.8) = 80 */
static int32_t rack_gp_increase_per = 2; /* 2% increase in multiplier */
static int32_t rack_per_lower_bound = 50; /* Don't allow to drop below this multiplier */
static int32_t rack_per_upper_bound_ss = 0; /* Don't allow SS to grow above this */
@@ -713,7 +726,7 @@
rack_log_gpset(struct tcp_rack *rack, uint32_t seq_end, uint32_t ack_end_t,
uint32_t send_end_t, int line, uint8_t mode, struct rack_sendmap *rsm)
{
- if (tcp_bblogging_on(rack->rc_tp)) {
+ if (tcp_bblogging_on(rack->rc_tp) && (rack_verbose_logging != 0)) {
union tcp_log_stackspecific log;
struct timeval tv;
@@ -1175,8 +1188,8 @@
SYSCTL_ADD_S32(&rack_sysctl_ctx,
SYSCTL_CHILDREN(rack_timely),
OID_AUTO, "decrease", CTLFLAG_RW,
- &rack_gp_decrease_per, 20,
- "Rack timely decrease percentage of our GP multiplication factor");
+ &rack_gp_decrease_per, 80,
+ "Rack timely Beta value 80 = .8 (scaled by 100)");
SYSCTL_ADD_S32(&rack_sysctl_ctx,
SYSCTL_CHILDREN(rack_timely),
OID_AUTO, "increase", CTLFLAG_RW,
@@ -1187,6 +1200,12 @@
OID_AUTO, "lowerbound", CTLFLAG_RW,
&rack_per_lower_bound, 50,
"Rack timely lowest percentage we allow GP multiplier to fall to");
+ SYSCTL_ADD_S32(&rack_sysctl_ctx,
+ SYSCTL_CHILDREN(rack_timely),
+ OID_AUTO, "p5_upper", CTLFLAG_RW,
+ &rack_gain_p5_ub, 250,
+ "Profile 5 upper bound to timely gain");
+
SYSCTL_ADD_S32(&rack_sysctl_ctx,
SYSCTL_CHILDREN(rack_timely),
OID_AUTO, "upperboundss", CTLFLAG_RW,
@@ -1967,7 +1986,7 @@
static void
rack_log_hybrid_bw(struct tcp_rack *rack, uint32_t seq, uint64_t cbw, uint64_t tim,
uint64_t data, uint8_t mod, uint16_t aux,
- struct tcp_sendfile_track *cur)
+ struct tcp_sendfile_track *cur, int line)
{
#ifdef TCP_REQUEST_TRK
int do_log = 0;
@@ -1991,7 +2010,7 @@
* All other less noisy logs here except the measure which
* also needs to come out on the point and the log.
*/
- do_log = tcp_bblogging_on(rack->rc_tp);
+ do_log = tcp_bblogging_on(rack->rc_tp);
} else {
do_log = tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING);
}
@@ -2004,6 +2023,7 @@
/* Convert our ms to a microsecond */
memset(&log, 0, sizeof(log));
+ log.u_bbr.cwnd_gain = line;
log.u_bbr.timeStamp = tcp_get_usecs(&tv);
log.u_bbr.rttProp = tim;
log.u_bbr.bw_inuse = cbw;
@@ -2049,8 +2069,10 @@
/* localtime = <delivered | applimited>*/
log.u_bbr.applimited = (uint32_t)(cur->localtime & 0x00000000ffffffff);
log.u_bbr.delivered = (uint32_t)((cur->localtime >> 32) & 0x00000000ffffffff);
+#ifdef TCP_REQUEST_TRK
off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
log.u_bbr.bbr_substate = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
+#endif
log.u_bbr.flex4 = (uint32_t)(rack->rc_tp->t_sndbytes - cur->sent_at_fs);
log.u_bbr.flex5 = (uint32_t)(rack->rc_tp->t_snd_rxt_bytes - cur->rxt_at_fs);
log.u_bbr.flex7 = (uint16_t)cur->hybrid_flags;
@@ -2083,6 +2105,60 @@
#endif
}
+#ifdef TCP_REQUEST_TRK
+static void
+rack_log_hybrid_sends(struct tcp_rack *rack, struct tcp_sendfile_track *cur, int line)
+{
+ if (tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING)) {
+ union tcp_log_stackspecific log;
+ struct timeval tv;
+ uint64_t off;
+
+ /* Convert our ms to a microsecond */
+ memset(&log, 0, sizeof(log));
+
+ log.u_bbr.timeStamp = tcp_get_usecs(&tv);
+ log.u_bbr.cur_del_rate = rack->rc_tp->t_sndbytes;
+ log.u_bbr.delRate = cur->sent_at_fs;
+ log.u_bbr.rttProp = rack->rc_tp->t_snd_rxt_bytes;
+ log.u_bbr.bw_inuse = cur->rxt_at_fs;
+ log.u_bbr.cwnd_gain = line;
+ off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
+ log.u_bbr.bbr_substate = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
+ /* start = < flex1 | flex2 > */
+ log.u_bbr.flex2 = (uint32_t)(cur->start & 0x00000000ffffffff);
+ log.u_bbr.flex1 = (uint32_t)((cur->start >> 32) & 0x00000000ffffffff);
+ /* end = < flex3 | flex4 > */
+ log.u_bbr.flex4 = (uint32_t)(cur->end & 0x00000000ffffffff);
+ log.u_bbr.flex3 = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff);
+
+ /* localtime = <delivered | applimited>*/
+ log.u_bbr.applimited = (uint32_t)(cur->localtime & 0x00000000ffffffff);
+ log.u_bbr.delivered = (uint32_t)((cur->localtime >> 32) & 0x00000000ffffffff);
+ /* client timestamp = <lt_epoch | epoch>*/
+ log.u_bbr.epoch = (uint32_t)(cur->timestamp & 0x00000000ffffffff);
+ log.u_bbr.lt_epoch = (uint32_t)((cur->timestamp >> 32) & 0x00000000ffffffff);
+ /* now set all the flags in */
+ log.u_bbr.pkts_out = cur->hybrid_flags;
+ log.u_bbr.flex6 = cur->flags;
+ /*
+ * Last send time = <flex5 | pkt_epoch> note we do not distinguish cases
+ * where a false retransmit occurred so first_send <-> lastsend may
+ * include longer time then it actually took if we have a false rxt.
+ */
+ log.u_bbr.pkt_epoch = (uint32_t)(rack->r_ctl.last_tmit_time_acked & 0x00000000ffffffff);
+ log.u_bbr.flex5 = (uint32_t)((rack->r_ctl.last_tmit_time_acked >> 32) & 0x00000000ffffffff);
+
+ log.u_bbr.flex8 = HYBRID_LOG_SENT_LOST;
+ tcp_log_event(rack->rc_tp, NULL,
+ &rack->rc_inp->inp_socket->so_rcv,
+ &rack->rc_inp->inp_socket->so_snd,
+ TCP_HYBRID_PACING_LOG, 0,
+ 0, &log, false, NULL, __func__, __LINE__, &tv);
+ }
+}
+#endif
+
static inline uint64_t
rack_compensate_for_linerate(struct tcp_rack *rack, uint64_t bw)
{
@@ -2128,13 +2204,13 @@
*/
struct tcp_sendfile_track *ent;
- ent = rack->r_ctl.rc_last_sft;
+ ent = rack->r_ctl.rc_last_sft;
microuptime(&tv);
timenow = tcp_tv_to_lusectick(&tv);
if (timenow >= ent->deadline) {
/* No time left we do DGP only */
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- 0, 0, 0, HYBRID_LOG_OUTOFTIME, 0, ent);
+ 0, 0, 0, HYBRID_LOG_OUTOFTIME, 0, ent, __LINE__);
rack->r_ctl.bw_rate_cap = 0;
return;
}
@@ -2143,7 +2219,7 @@
if (timeleft < HPTS_MSEC_IN_SEC) {
/* If there is less than a ms left just use DGPs rate */
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- 0, timeleft, 0, HYBRID_LOG_OUTOFTIME, 0, ent);
+ 0, timeleft, 0, HYBRID_LOG_OUTOFTIME, 0, ent, __LINE__);
rack->r_ctl.bw_rate_cap = 0;
return;
}
@@ -2159,7 +2235,7 @@
else {
/* TSNH, we should catch it at the send */
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- 0, timeleft, 0, HYBRID_LOG_CAPERROR, 0, ent);
+ 0, timeleft, 0, HYBRID_LOG_CAPERROR, 0, ent, __LINE__);
rack->r_ctl.bw_rate_cap = 0;
return;
}
@@ -2178,7 +2254,7 @@
else {
/* TSNH, we should catch it at the send */
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- 0, timeleft, lengone, HYBRID_LOG_CAPERROR, 0, ent);
+ 0, timeleft, lengone, HYBRID_LOG_CAPERROR, 0, ent, __LINE__);
rack->r_ctl.bw_rate_cap = 0;
return;
}
@@ -2186,7 +2262,7 @@
if (lenleft == 0) {
/* We have it all sent */
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- 0, timeleft, lenleft, HYBRID_LOG_ALLSENT, 0, ent);
+ 0, timeleft, lenleft, HYBRID_LOG_ALLSENT, 0, ent, __LINE__);
if (rack->r_ctl.bw_rate_cap)
goto normal_ratecap;
else
@@ -2210,10 +2286,10 @@
rack_log_type_pacing_sizes(rack->rc_tp, rack, rack->r_ctl.client_suggested_maxseg, orig_max, __LINE__, 5);
}
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- calcbw, timeleft, lenleft, HYBRID_LOG_CAP_CALC, 0, ent);
+ calcbw, timeleft, lenleft, HYBRID_LOG_CAP_CALC, 0, ent, __LINE__);
if ((calcbw > 0) && (*bw > calcbw)) {
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- *bw, ent->deadline, lenleft, HYBRID_LOG_RATE_CAP, 0, ent);
+ *bw, ent->deadline, lenleft, HYBRID_LOG_RATE_CAP, 0, ent, __LINE__);
*capped = 1;
*bw = calcbw;
}
@@ -2241,7 +2317,7 @@
*bw = rack->r_ctl.bw_rate_cap;
rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
*bw, 0, 0,
- HYBRID_LOG_RATE_CAP, 1, NULL);
+ HYBRID_LOG_RATE_CAP, 1, NULL, __LINE__);
}
}
@@ -2916,7 +2992,7 @@
static void
rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_t cts, struct timeval *tv, int line)
{
- if (tcp_bblogging_on(rack->rc_tp)) {
+ if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
union tcp_log_stackspecific log;
memset(&log.u_bbr, 0, sizeof(log.u_bbr));
@@ -3116,7 +3192,7 @@
static void
rack_log_to_processing(struct tcp_rack *rack, uint32_t cts, int32_t ret, int32_t timers)
{
- if (tcp_bblogging_on(rack->rc_tp)) {
+ if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
union tcp_log_stackspecific log;
struct timeval tv;
@@ -3819,15 +3895,30 @@
static uint32_t
rack_get_decrease(struct tcp_rack *rack, uint32_t curper, int32_t rtt_diff)
{
- /*
+ /*-
* norm_grad = rtt_diff / minrtt;
* new_per = curper * (1 - B * norm_grad)
*
- * B = rack_gp_decrease_per (default 10%)
+ * B = rack_gp_decrease_per (default 80%)
* rtt_dif = input var current rtt-diff
* curper = input var current percentage
* minrtt = from rack filter
*
+ * In order to do the floating point calculations above we
+ * do an integer conversion. The code looks confusing so let me
+ * translate it into something that use more variables and
+ * is clearer for us humans :)
+ *
+ * uint64_t norm_grad, inverse, reduce_by, final_result;
+ * uint32_t perf;
+ *
+ * norm_grad = (((uint64_t)rtt_diff * 1000000) /
+ * (uint64_t)get_filter_small(&rack->r_ctl.rc_gp_min_rtt));
+ * inverse = ((uint64_t)rack_gp_decrease * (uint64_t)1000000) * norm_grad;
+ * inverse /= 1000000;
+ * reduce_by = (1000000 - inverse);
+ * final_result = (cur_per * reduce_by) / 1000000;
+ * perf = (uint32_t)final_result;
*/
uint64_t perf;
@@ -3852,7 +3943,7 @@
* result = curper * (1 - (B * ( 1 - ------ ))
* gp_srtt
*
- * B = rack_gp_decrease_per (default 10%)
+ * B = rack_gp_decrease_per (default .8 i.e. 80)
* highrttthresh = filter_min * rack_gp_rtt_maxmul
*/
uint64_t perf;
@@ -3864,6 +3955,20 @@
((uint64_t)rack_gp_decrease_per * ((uint64_t)1000000 -
((uint64_t)highrttthresh * (uint64_t)1000000) /
(uint64_t)rtt)) / 100)) /(uint64_t)1000000);
+ if (tcp_bblogging_on(rack->rc_tp)) {
+ uint64_t log1;
+
+ log1 = rtt;
+ log1 <<= 32;
+ log1 |= highrttthresh;
+ rack_log_timely(rack,
+ rack_gp_decrease_per,
+ (uint64_t)curper,
+ log1,
+ perf,
+ __LINE__,
+ 15);
+ }
return (perf);
}
@@ -3911,7 +4016,7 @@
/* Sent in SS */
if (timely_says == 2) {
new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ss, rtt);
- alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
+ alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ss, rtt_diff);
if (alt < new_per)
val = alt;
else
@@ -3944,7 +4049,7 @@
/* Sent in CA */
if (timely_says == 2) {
new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ca, rtt);
- alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
+ alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ca, rtt_diff);
if (alt < new_per)
val = alt;
else
@@ -5040,7 +5145,7 @@
(rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) {
/* We have enough measurements now */
rack->gp_ready = 1;
- if ((rack->rc_always_pace && (rack->use_fixed_rate == 0)) ||
+ if (rack->dgp_on ||
rack->rack_hibeta)
rack_set_cc_pacing(rack);
if (rack->defer_options)
@@ -6860,21 +6965,20 @@
* even a SACK should not disturb us (with
* the exception of r_rr_config 3).
*/
- if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK) {
+ if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK) ||
+ (IN_RECOVERY(tp->t_flags))) {
if (rack->r_rr_config != 3)
tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
else if (rack->rc_pace_dnd) {
- if (IN_RECOVERY(tp->t_flags)) {
- /*
- * When DND is on, we only let a sack
- * interrupt us if we are not in recovery.
- *
- * If DND is off, then we never hit here
- * and let all sacks wake us up.
- *
- */
- tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
- }
+ /*
+ * When DND is on, we only let a sack
+ * interrupt us if we are not in recovery.
+ *
+ * If DND is off, then we never hit here
+ * and let all sacks wake us up.
+ *
+ */
+ tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
}
}
/* For sack attackers we want to ignore sack */
@@ -10357,7 +10461,7 @@
rack->r_wanted_output = 1;
if (SEQ_GT(th_ack, tp->snd_una))
- rack->r_ctl.last_cumack_advance = acktime;
+ rack->r_ctl.last_cumack_advance = acktime;
/* Tend any TLP that has been marked for 1/2 the seq space (its old) */
if ((rack->rc_last_tlp_acked_set == 1)&&
@@ -10484,6 +10588,7 @@
}
}
/* Now do we consume the whole thing? */
+ rack->r_ctl.last_tmit_time_acked = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
if (SEQ_GEQ(th_ack, rsm->r_end)) {
/* Its all consumed. */
uint32_t left;
@@ -10619,16 +10724,43 @@
/* The trim will move th_ack into r_start for us */
tqhash_trim(rack->r_ctl.tqh, th_ack);
/* Now do we need to move the mbuf fwd too? */
- if (rsm->m) {
- while (rsm->soff >= rsm->m->m_len) {
- rsm->soff -= rsm->m->m_len;
- rsm->m = rsm->m->m_next;
- KASSERT((rsm->m != NULL),
- (" nrsm:%p hit at soff:%u null m",
- rsm, rsm->soff));
+ {
+ struct mbuf *m;
+ uint32_t soff;
+
+ m = rsm->m;
+ soff = rsm->soff;
+ if (m) {
+ while (soff >= m->m_len) {
+ soff -= m->m_len;
+ KASSERT((m->m_next != NULL),
+ (" rsm:%p off:%u soff:%u m:%p",
+ rsm, rsm->soff, soff, m));
+ m = m->m_next;
+ if (m == NULL) {
+ /*
+ * This is a fall-back that prevents a panic. In reality
+ * we should be able to walk the mbuf's and find our place.
+ * At this point snd_una has not been updated with the sbcut() yet
+ * but tqhash_trim did update rsm->r_start so the offset calcuation
+ * should work fine. This is undesirable since we will take cache
+ * hits to access the socket buffer. And even more puzzling is that
+ * it happens occasionally. It should not :(
+ */
+ m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd,
+ (rsm->r_start - tp->snd_una),
+ &soff);
+ break;
+ }
+ }
+ /*
+ * Now save in our updated values.
+ */
+ rsm->m = m;
+ rsm->soff = soff;
+ rsm->orig_m_len = rsm->m->m_len;
+ rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
}
- rsm->orig_m_len = rsm->m->m_len;
- rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
}
if (rack->app_limited_needs_set &&
SEQ_GEQ(th_ack, tp->gput_seq))
@@ -11516,7 +11648,7 @@
(rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) {
/* We have enough measurements now */
rack->gp_ready = 1;
- if ((rack->rc_always_pace && (rack->use_fixed_rate == 0)) ||
+ if (rack->dgp_on ||
rack->rack_hibeta)
rack_set_cc_pacing(rack);
if (rack->defer_options)
@@ -11557,7 +11689,7 @@
int do_log;
do_log = tcp_bblogging_on(rack->rc_tp);
- if (do_log == 0) {
+ if (do_log == 0) {
if ((do_log = tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING) )== 0)
return;
/* We only allow the three below with point logging on */
@@ -11565,7 +11697,7 @@
(mod != HYBRID_LOG_RULES_SET) &&
(mod != HYBRID_LOG_REQ_COMP))
return;
-
+
}
if (do_log) {
union tcp_log_stackspecific log;
@@ -11593,8 +11725,10 @@
log.u_bbr.epoch = (uint32_t)(cur->deadline & 0x00000000ffffffff);
log.u_bbr.lt_epoch = (uint32_t)((cur->deadline >> 32) & 0x00000000ffffffff) ;
log.u_bbr.bbr_state = 1;
+#ifdef TCP_REQUEST_TRK
off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
log.u_bbr.use_lt_bw = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
+#endif
} else {
log.u_bbr.flex2 = err;
}
@@ -11633,7 +11767,8 @@
rc_cur = tcp_req_find_req_for_seq(rack->rc_tp, seq);
if (rc_cur == NULL) {
/* If not in the beginning what about the end piece */
- rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);
+ if (rack->rc_hybrid_mode)
+ rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);
rc_cur = tcp_req_find_req_for_seq(rack->rc_tp, (seq + len - 1));
} else {
err = 12345;
@@ -11644,12 +11779,17 @@
rack->r_ctl.client_suggested_maxseg = 0;
rack->rc_catch_up = 0;
rack->r_ctl.bw_rate_cap = 0;
- rack_log_hybrid(rack, (seq + len - 1), NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);
+ if (rack->rc_hybrid_mode)
+ rack_log_hybrid(rack, (seq + len - 1), NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);
if (rack->r_ctl.rc_last_sft) {
rack->r_ctl.rc_last_sft = NULL;
}
return;
}
+ if ((rc_cur->hybrid_flags & TCP_HYBRID_PACING_WASSET) == 0) {
+ /* This entry was never setup for hybrid pacing on/off etc */
+ return;
+ }
/*
* Ok if we have a new entry *or* have never
* set up an entry we need to proceed. If
@@ -11661,7 +11801,8 @@
if ((rack->r_ctl.rc_last_sft != NULL) &&
(rack->r_ctl.rc_last_sft == rc_cur)) {
/* Its already in place */
- rack_log_hybrid(rack, seq, rc_cur, HYBRID_LOG_ISSAME, __LINE__, 0);
+ if (rack->rc_hybrid_mode)
+ rack_log_hybrid(rack, seq, rc_cur, HYBRID_LOG_ISSAME, __LINE__, 0);
return;
}
if (rack->rc_hybrid_mode == 0) {
@@ -11757,7 +11898,8 @@
* way it will complete when all of it is acked.
*/
ent->end_seq = (seq + len);
- rack_log_hybrid_bw(rack, seq, len, 0, 0, HYBRID_LOG_EXTEND, 0, ent);
+ if (rack->rc_hybrid_mode)
+ rack_log_hybrid_bw(rack, seq, len, 0, 0, HYBRID_LOG_EXTEND, 0, ent, __LINE__);
}
/* Now validate we have set the send time of this one */
if ((ent->flags & TCP_TRK_TRACK_FLG_FSND) == 0) {
@@ -11941,6 +12083,7 @@
/* Ok this ack frees it */
rack_log_hybrid(rack, th_ack,
ent, HYBRID_LOG_REQ_COMP, __LINE__, 0);
+ rack_log_hybrid_sends(rack, ent, __LINE__);
/* calculate the time based on the ack arrival */
data = ent->end - ent->start;
laa = tcp_tv_to_lusectick(&rack->r_ctl.act_rcv_time);
@@ -11962,7 +12105,7 @@
cbw /= tim;
else
cbw = 0;
- rack_log_hybrid_bw(rack, th_ack, cbw, tim, data, HYBRID_LOG_BW_MEASURE, 0, ent);
+ rack_log_hybrid_bw(rack, th_ack, cbw, tim, data, HYBRID_LOG_BW_MEASURE, 0, ent, __LINE__);
/*
* Check to see if we are freeing what we are pointing to send wise
* if so be sure to NULL the pointer so we know we are no longer
@@ -14254,7 +14397,7 @@
if (fill_override)
rate_wanted = *fill_override;
else
- rate_wanted = rack_get_output_bw(rack, bw_est, NULL, NULL);
+ rate_wanted = rack_get_gp_est(rack);
if (rate_wanted) {
/* We have something */
rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack,
@@ -14885,8 +15028,6 @@
*/
rack->rc_new_rnd_needed = 1;
rack->r_ctl.rc_split_limit = V_tcp_map_split_limit;
- rack->r_ctl.rc_saved_beta.beta = V_newreno_beta_ecn;
- rack->r_ctl.rc_saved_beta.beta_ecn = V_newreno_beta_ecn;
/* We want abe like behavior as well */
rack->r_ctl.rc_saved_beta.newreno_flags |= CC_NEWRENO_BETA_ECN_ENABLED;
rack->r_ctl.rc_reorder_fade = rack_reorder_fade;
@@ -14924,8 +15065,18 @@
rack->rc_user_set_max_segs = rack_hptsi_segments;
rack->rc_force_max_seg = 0;
TAILQ_INIT(&rack->r_ctl.opt_list);
- if (rack_hibeta_setting)
+ rack->r_ctl.rc_saved_beta.beta = V_newreno_beta_ecn;
+ rack->r_ctl.rc_saved_beta.beta_ecn = V_newreno_beta_ecn;
+ if (rack_hibeta_setting) {
rack->rack_hibeta = 1;
+ if ((rack_hibeta_setting >= 50) &&
+ (rack_hibeta_setting <= 100)) {
+ rack->r_ctl.rc_saved_beta.beta = rack_hibeta_setting;
+ rack->r_ctl.saved_hibeta = rack_hibeta_setting;
+ }
+ } else {
+ rack->r_ctl.saved_hibeta = 50;
+ }
rack->r_ctl.rc_reorder_shift = rack_reorder_thresh;
rack->r_ctl.rc_pkt_delay = rack_pkt_delay;
rack->r_ctl.rc_tlp_cwnd_reduce = rack_lower_cwnd_at_tlp;
@@ -14941,7 +15092,7 @@
rack->rc_gp_no_rec_chg = 1;
if (rack_pace_every_seg && tcp_can_enable_pacing()) {
rack->rc_always_pace = 1;
- if ((rack->gp_ready) && (rack->rc_always_pace && (rack->use_fixed_rate == 0)))
+ if (rack->rack_hibeta)
rack_set_cc_pacing(rack);
} else
rack->rc_always_pace = 0;
@@ -17204,6 +17355,19 @@
union tcp_log_stackspecific log;
struct timeval tv;
+ if (rack_verbose_logging == 0) {
+ /*
+ * We are not verbose screen out all but
+ * ones we always want.
+ */
+ if ((method != 2) &&
+ (method != 3) &&
+ (method != 7) &&
+ (method != 14) &&
+ (method != 20)) {
+ return;
+ }
+ }
memset(&log, 0, sizeof(log));
log.u_bbr.flex1 = slot;
log.u_bbr.flex2 = len;
@@ -17307,6 +17471,60 @@
return (new_tso);
}
+static uint64_t
+rack_arrive_at_discounted_rate(struct tcp_rack *rack, uint64_t window_input, uint32_t *rate_set, uint32_t *gain_b)
+{
+ uint64_t reduced_win;
+ uint32_t gain;
+
+ if (window_input < rc_init_window(rack)) {
+ /*
+ * The cwnd is collapsed to
+ * nearly zero, maybe because of a time-out?
+ * Lets drop back to the lt-bw.
+ */
+ reduced_win = rack_get_lt_bw(rack);
+ /* Set the flag so the caller knows its a rate and not a reduced window */
+ *rate_set = 1;
+ gain = 100;
+ } else if (IN_RECOVERY(rack->rc_tp->t_flags)) {
+ /*
+ * If we are in recover our cwnd needs to be less for
+ * our pacing consideration.
+ */
+ if (rack->rack_hibeta == 0) {
+ reduced_win = window_input / 2;
+ gain = 50;
+ } else {
+ reduced_win = window_input * rack->r_ctl.saved_hibeta;
+ reduced_win /= 100;
+ gain = rack->r_ctl.saved_hibeta;
+ }
+ } else {
+ /*
+ * Apply Timely factor to increase/decrease the
+ * amount we are pacing at.
+ */
+ gain = rack_get_output_gain(rack, NULL);
+ if (gain > rack_gain_p5_ub) {
+ gain = rack_gain_p5_ub;
+ }
+ reduced_win = window_input * gain;
+ reduced_win /= 100;
+ }
+ if (gain_b != NULL)
+ *gain_b = gain;
+ /*
+ * What is being returned here is a trimmed down
+ * window values in all cases where rate_set is left
+ * at 0. In one case we actually return the rate (lt_bw).
+ * the "reduced_win" is returned as a slimmed down cwnd that
+ * is then calculated by the caller into a rate when rate_set
+ * is 0.
+ */
+ return (reduced_win);
+}
+
static int32_t
pace_to_fill_cwnd(struct tcp_rack *rack, int32_t slot, uint32_t len, uint32_t segsiz, int *capped, uint64_t *rate_wanted, uint8_t non_paced)
{
@@ -17331,12 +17549,21 @@
* and the the smallest send window.
*/
fill_bw = min(rack->rc_tp->snd_cwnd, rack->r_ctl.cwnd_to_use);
+ if (rack->rc_fillcw_apply_discount) {
+ uint32_t rate_set = 0;
+
+ fill_bw = rack_arrive_at_discounted_rate(rack, fill_bw, &rate_set, NULL);
+ if (rate_set) {
+ goto at_lt_bw;
+ }
+ }
/* Take the rwnd if its smaller */
if (fill_bw > rack->rc_tp->snd_wnd)
fill_bw = rack->rc_tp->snd_wnd;
/* Now lets make it into a b/w */
fill_bw *= (uint64_t)HPTS_USEC_IN_SEC;
fill_bw /= (uint64_t)rack->r_ctl.rc_last_us_rtt;
+at_lt_bw:
if (rack->r_fill_less_agg) {
/*
* We want the average of the rate_wanted
@@ -17404,8 +17631,9 @@
}
}
if (rack->r_ctl.bw_rate_cap && (fill_bw > rack->r_ctl.bw_rate_cap)) {
- rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
- fill_bw, 0, 0, HYBRID_LOG_RATE_CAP, 2, NULL);
+ if (rack->rc_hybrid_mode)
+ rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
+ fill_bw, 0, 0, HYBRID_LOG_RATE_CAP, 2, NULL, __LINE__);
fill_bw = rack->r_ctl.bw_rate_cap;
}
/*
@@ -17513,9 +17741,27 @@
(rack->r_ctl.gp_bw == 0)) {
/* no way to yet do an estimate */
bw_est = rate_wanted = 0;
- } else {
+ } else if (rack->dgp_on) {
bw_est = rack_get_bw(rack);
rate_wanted = rack_get_output_bw(rack, bw_est, rsm, &capped);
+ } else {
+ uint32_t gain, rate_set = 0;
+
+ rate_wanted = min(rack->rc_tp->snd_cwnd, rack->r_ctl.cwnd_to_use);
+ rate_wanted = rack_arrive_at_discounted_rate(rack, rate_wanted, &rate_set, &gain);
+ if (rate_set == 0) {
+ if (rate_wanted > rack->rc_tp->snd_wnd)
+ rate_wanted = rack->rc_tp->snd_wnd;
+ /* Now lets make it into a b/w */
+ rate_wanted *= (uint64_t)HPTS_USEC_IN_SEC;
+ rate_wanted /= (uint64_t)rack->r_ctl.rc_last_us_rtt;
+ }
+ bw_est = rate_wanted;
+ rack_log_pacing_delay_calc(rack, rack->rc_tp->snd_cwnd,
+ rack->r_ctl.cwnd_to_use,
+ rate_wanted, bw_est,
+ rack->r_ctl.rc_last_us_rtt,
+ 88, __LINE__, NULL, gain);
}
if ((bw_est == 0) || (rate_wanted == 0) ||
((rack->gp_ready == 0) && (rack->use_fixed_rate == 0))) {
@@ -17534,16 +17780,16 @@
* means we may be off if we are larger than 1500 bytes
* or smaller. But this just makes us more conservative.
*/
-
+
oh = (tp->t_maxseg - segsiz) + sizeof(struct tcphdr);
if (rack->r_is_v6) {
#ifdef INET6
oh += sizeof(struct ip6_hdr);
-#endif
+#endif
} else {
#ifdef INET
oh += sizeof(struct ip);
-#endif
+#endif
}
/* We add a fixed 14 for the ethernet header */
oh += 14;
@@ -17602,6 +17848,7 @@
prev_fill = rack->r_via_fill_cw;
if ((rack->rc_pace_to_cwnd) &&
(capped == 0) &&
+ (rack->dgp_on == 1) &&
(rack->use_fixed_rate == 0) &&
(rack->in_probe_rtt == 0) &&
(IN_FASTRECOVERY(rack->rc_tp->t_flags) == 0)) {
@@ -17652,8 +17899,8 @@
if (rack->r_ctl.crte) {
rack->rack_hdrw_pacing = 1;
rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size_w_divisor(tp, rate_wanted, segsiz,
- pace_one, rack->r_ctl.crte,
- NULL, rack->r_ctl.pace_len_divisor);
+ pace_one, rack->r_ctl.crte,
+ NULL, rack->r_ctl.pace_len_divisor);
rack_log_hdwr_pacing(rack,
rate_wanted, rack->r_ctl.crte->rate, __LINE__,
err, 0);
@@ -17695,8 +17942,8 @@
* do allow hardware pacing to be restarted.
*/
rack_log_hdwr_pacing(rack,
- bw_est, rack->r_ctl.crte->rate, __LINE__,
- 0, 5);
+ bw_est, rack->r_ctl.crte->rate, __LINE__,
+ 0, 5);
tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
rack->r_ctl.crte = NULL;
rack->rack_attempt_hdwr_pace = 0;
@@ -17705,11 +17952,11 @@
goto done_w_hdwr;
}
nrte = tcp_chg_pacing_rate(rack->r_ctl.crte,
- rack->rc_tp,
- rack->rc_inp->inp_route.ro_nh->nh_ifp,
- rate_wanted,
- RS_PACING_GEQ,
- &err, &rack->r_ctl.crte_prev_rate);
+ rack->rc_tp,
+ rack->rc_inp->inp_route.ro_nh->nh_ifp,
+ rate_wanted,
+ RS_PACING_GEQ,
+ &err, &rack->r_ctl.crte_prev_rate);
if (nrte == NULL) {
/*
* Lost the rate, lets drop hardware pacing
@@ -17725,8 +17972,8 @@
} else if (nrte != rack->r_ctl.crte) {
rack->r_ctl.crte = nrte;
rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size_w_divisor(tp, rate_wanted,
- segsiz, pace_one, rack->r_ctl.crte,
- NULL, rack->r_ctl.pace_len_divisor);
+ segsiz, pace_one, rack->r_ctl.crte,
+ NULL, rack->r_ctl.pace_len_divisor);
rack_log_hdwr_pacing(rack,
rate_wanted, rack->r_ctl.crte->rate, __LINE__,
err, 2);
@@ -17747,7 +17994,7 @@
98, __LINE__, NULL, 0);
slot = minslot;
}
-done_w_hdwr:
+ done_w_hdwr:
if (rack_limit_time_with_srtt &&
(rack->use_fixed_rate == 0) &&
(rack->rack_hdrw_pacing == 0)) {
@@ -18070,7 +18317,7 @@
unsigned ipoptlen, int32_t orig_len, int32_t len, int error,
int rsm_is_null, int optlen, int line, uint16_t mode)
{
- if (tcp_bblogging_on(rack->rc_tp)) {
+ if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
union tcp_log_stackspecific log;
struct timeval tv;
@@ -18869,6 +19116,10 @@
rack->r_ctl.last_sent_tlp_seq = rsm->r_start;
rack->r_ctl.last_sent_tlp_len = rsm->r_end - rsm->r_start;
}
+ if (rack->r_ctl.rc_prr_sndcnt >= len)
+ rack->r_ctl.rc_prr_sndcnt -= len;
+ else
+ rack->r_ctl.rc_prr_sndcnt = 0;
}
tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
rack->forced_ack = 0; /* If we send something zap the FA flag */
@@ -19049,6 +19300,7 @@
m = NULL;
goto failed;
}
+ rack->r_ctl.cwnd_to_use = tp->snd_cwnd;
startseq = tp->snd_max;
segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
inp = rack->rc_inp;
@@ -22406,6 +22658,7 @@
if (tcp_can_enable_pacing() == 0)
return (EBUSY);
}
+ rack->rc_fillcw_apply_discount = 0;
rack->dgp_on = 1;
rack->rc_always_pace = 1;
rack->use_fixed_rate = 0;
@@ -22494,6 +22747,26 @@
err = rack_set_dgp(rack);
if (err)
return (err);
+ } else if (prof == 5) {
+ err = rack_set_dgp(rack);
+ if (err)
+ return (err);
+ /*
+ * By turning DGP off we change the rate
+ * picked to be only the one the cwnd and rtt
+ * get us.
+ */
+ rack->dgp_on = 0;
+ } else if (prof == 6) {
+ err = rack_set_dgp(rack);
+ if (err)
+ return (err);
+ /*
+ * Profile 6 tweaks DGP so that it will apply to
+ * fill-cw the same settings that profile5 does
+ * to replace DGP. It gets then the max(dgp-rate, fillcw(discounted).
+ */
+ rack->rc_fillcw_apply_discount = 1;
} else if (prof == 0) {
/* This changes things back to the default settings */
rack->dgp_on = 0;
@@ -22510,7 +22783,7 @@
}
if (rack_pace_every_seg && tcp_can_enable_pacing()) {
rack->rc_always_pace = 1;
- if ((rack->gp_ready) && (rack->use_fixed_rate == 0))
+ if (rack->rack_hibeta)
rack_set_cc_pacing(rack);
} else
rack->rc_always_pace = 0;
@@ -22662,7 +22935,7 @@
}
}
/* Now set in our flags */
- sft->hybrid_flags = hybrid->hybrid_flags;
+ sft->hybrid_flags = hybrid->hybrid_flags | TCP_HYBRID_PACING_WASSET;
if (hybrid->hybrid_flags & TCP_HYBRID_PACING_CSPR)
sft->cspr = hybrid->cspr;
else
@@ -22731,10 +23004,25 @@
break;
case TCP_RACK_HI_BETA:
RACK_OPTS_INC(tcp_rack_hi_beta);
- if (optval)
+ if (optval > 0) {
rack->rack_hibeta = 1;
- else
+ if ((optval >= 50) &&
+ (optval <= 100)) {
+ /*
+ * User wants to set a custom beta.
+ */
+ rack->r_ctl.saved_hibeta = optval;
+ if (rack->rc_pacing_cc_set)
+ rack_undo_cc_pacing(rack);
+ rack->r_ctl.rc_saved_beta.beta = optval;
+ }
+ if (rack->rc_pacing_cc_set == 0)
+ rack_set_cc_pacing(rack);
+ } else {
rack->rack_hibeta = 0;
+ if (rack->rc_pacing_cc_set)
+ rack_undo_cc_pacing(rack);
+ }
break;
case TCP_RACK_PACING_BETA:
RACK_OPTS_INC(tcp_rack_beta);
@@ -23007,7 +23295,7 @@
break;
} else if (tcp_can_enable_pacing()) {
rack->rc_always_pace = 1;
- if ((rack->gp_ready) && (rack->use_fixed_rate == 0))
+ if (rack->rack_hibeta)
rack_set_cc_pacing(rack);
}
else {
@@ -23103,7 +23391,10 @@
case TCP_RACK_PACE_MAX_SEG:
/* Max segments size in a pace in bytes */
RACK_OPTS_INC(tcp_rack_max_seg);
- rack->rc_user_set_max_segs = optval;
+ if (optval <= MAX_USER_SET_SEG)
+ rack->rc_user_set_max_segs = optval;
+ else
+ rack->rc_user_set_max_segs = MAX_USER_SET_SEG;
rack_set_pace_segments(tp, rack, __LINE__, NULL);
break;
case TCP_RACK_PACE_RATE_REC:
@@ -23115,7 +23406,7 @@
if (rack->r_ctl.rc_fixed_pacing_rate_ss == 0)
rack->r_ctl.rc_fixed_pacing_rate_ss = optval;
rack->use_fixed_rate = 1;
- if (rack->rc_always_pace && rack->gp_ready && rack->rack_hibeta)
+ if (rack->rack_hibeta)
rack_set_cc_pacing(rack);
rack_log_pacing_delay_calc(rack,
rack->r_ctl.rc_fixed_pacing_rate_ss,
@@ -23133,7 +23424,7 @@
if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0)
rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
rack->use_fixed_rate = 1;
- if (rack->rc_always_pace && rack->gp_ready && rack->rack_hibeta)
+ if (rack->rack_hibeta)
rack_set_cc_pacing(rack);
rack_log_pacing_delay_calc(rack,
rack->r_ctl.rc_fixed_pacing_rate_ss,
@@ -23151,7 +23442,7 @@
if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0)
rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
rack->use_fixed_rate = 1;
- if (rack->rc_always_pace && rack->gp_ready && rack->rack_hibeta)
+ if (rack->rack_hibeta)
rack_set_cc_pacing(rack);
rack_log_pacing_delay_calc(rack,
rack->r_ctl.rc_fixed_pacing_rate_ss,
Index: sys/netinet/tcp_stacks/tcp_rack.h
===================================================================
--- sys/netinet/tcp_stacks/tcp_rack.h
+++ sys/netinet/tcp_stacks/tcp_rack.h
@@ -458,6 +458,7 @@
uint64_t last_sndbytes;
uint64_t last_snd_rxt_bytes;
uint64_t rxt_threshold;
+ uint64_t last_tmit_time_acked; /* Holds the last cumack point's last send time */
uint32_t last_rnd_rxt_clamped;
uint32_t num_of_clamps_applied;
uint32_t clamp_options;
@@ -526,6 +527,7 @@
uint8_t rc_no_push_at_mrtt; /* No push when we exceed max rtt */
uint8_t num_measurements; /* Number of measurements (up to 0xff, we freeze at 0xff) */
uint8_t req_measurements; /* How many measurements are required? */
+ uint8_t saved_hibeta;
uint8_t rc_tlp_cwnd_reduce; /* Socket option value Lock(a) */
uint8_t rc_prr_sendalot;/* Socket option value Lock(a) */
uint8_t rc_rate_sample_method;
@@ -577,6 +579,7 @@
#define HYBRID_LOG_OUTOFTIME 12 /* We are past the deadline DGP */
#define HYBRID_LOG_CAPERROR 13 /* Hit one of the TSNH cases */
#define HYBRID_LOG_EXTEND 14 /* We extended the end */
+#define HYBRID_LOG_SENT_LOST 15 /* A closing sent/lost report */
#define RACK_TIMELY_CNT_BOOST 5 /* At 5th increase boost */
#define RACK_MINRTT_FILTER_TIM 10 /* Seconds */
@@ -589,6 +592,8 @@
* +Slam cwnd
*/
+#define MAX_USER_SET_SEG 0x3f /* The max we can set is 63 which is probably too many */
+
#ifdef _KERNEL
struct tcp_rack {
@@ -659,7 +664,8 @@
r_via_fill_cw : 1,
r_fill_less_agg : 1;
- uint8_t rc_user_set_max_segs; /* Socket option value Lock(a) */
+ uint8_t rc_user_set_max_segs : 7, /* Socket option value Lock(a) */
+ rc_fillcw_apply_discount;
uint8_t rc_labc; /* Appropriate Byte Counting Value */
uint16_t forced_ack : 1,
rc_gp_incr : 1,

File Metadata

Mime Type
text/plain
Expires
Fri, Jul 24, 4:55 AM (1 h, 36 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
35431423
Default Alt Text
D40391.id122756.diff (35 KB)

Event Timeline