Index: sys/netinet/tcp_output.c =================================================================== --- sys/netinet/tcp_output.c +++ sys/netinet/tcp_output.c @@ -1560,7 +1560,7 @@ out: if (error == 0) - tcp_account_for_send(tp, len, (tp->snd_nxt != tp->snd_max), 0); + tcp_account_for_send(tp, len, (tp->snd_nxt != tp->snd_max), 0, hw_tls); /* * In transmit state, time the transmission and arrange for * the retransmit. In persist state, just set snd_max. Index: sys/netinet/tcp_stacks/bbr.c =================================================================== --- sys/netinet/tcp_stacks/bbr.c +++ sys/netinet/tcp_stacks/bbr.c @@ -13750,7 +13750,7 @@ * retransmit. In persist state, just set snd_max. */ if (error == 0) { - tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp); + tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls); if (TCPS_HAVEESTABLISHED(tp->t_state) && (tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0) Index: sys/netinet/tcp_stacks/rack.c =================================================================== --- sys/netinet/tcp_stacks/rack.c +++ sys/netinet/tcp_stacks/rack.c @@ -32,6 +32,7 @@ #include "opt_ipsec.h" #include "opt_tcpdebug.h" #include "opt_ratelimit.h" +#include "opt_kern_tls.h" #include #include #include @@ -483,7 +484,7 @@ static void rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len, uint32_t seq_out, uint8_t th_flags, int32_t err, uint64_t ts, - struct rack_sendmap *hintrsm, uint16_t add_flags, struct mbuf *s_mb, uint32_t s_moff); + struct rack_sendmap *hintrsm, uint16_t add_flags, struct mbuf *s_mb, uint32_t s_moff, int hw_tls); static void rack_log_sack_passed(struct tcpcb *tp, struct tcp_rack *rack, @@ -6061,7 +6062,8 @@ /* Push bit must go to the right edge as well */ if (rsm->r_flags & RACK_HAD_PUSH) rsm->r_flags &= ~RACK_HAD_PUSH; - + /* Clone over the state of the hw_tls flag */ + nrsm->r_hw_tls = rsm->r_hw_tls; /* * Now we need to find nrsm's new location in the mbuf chain * we basically calculate a new offset, which is soff + @@ -7191,7 +7193,7 @@ static void rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len, uint32_t seq_out, uint8_t th_flags, int32_t err, uint64_t cts, - struct rack_sendmap *hintrsm, uint16_t add_flag, struct mbuf *s_mb, uint32_t s_moff) + struct rack_sendmap *hintrsm, uint16_t add_flag, struct mbuf *s_mb, uint32_t s_moff, int hw_tls) { struct tcp_rack *rack; struct rack_sendmap *rsm, *nrsm, *insret, fe; @@ -7295,6 +7297,8 @@ } else { rsm->r_flags = add_flag; } + if (hw_tls) + rsm->r_hw_tls = 1; rsm->r_tim_lastsent[0] = cts; rsm->r_rtr_cnt = 1; rsm->r_rtr_bytes = 0; @@ -14875,7 +14879,7 @@ static struct mbuf * rack_fo_base_copym(struct mbuf *the_m, uint32_t the_off, int32_t *plen, struct rack_fast_send_blk *fsb, - int32_t seglimit, int32_t segsize) + int32_t seglimit, int32_t segsize, int hw_tls) { #ifdef KERN_TLS struct ktls_session *tls, *ntls; @@ -15059,7 +15063,7 @@ *s_mb = rack->r_ctl.fsb.m; n = rack_fo_base_copym(m, soff, plen, &rack->r_ctl.fsb, - seglimit, segsize); + seglimit, segsize, rack->r_ctl.fsb.hw_tls); return (n); } @@ -15243,7 +15247,7 @@ /* Fix up the orig_m_len and possibly the mbuf offset */ rack_adjust_orig_mlen(rsm); } - m->m_next = rack_fo_base_copym(rsm->m, rsm->soff, &len, NULL, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize); + m->m_next = rack_fo_base_copym(rsm->m, rsm->soff, &len, NULL, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, rsm->r_hw_tls); if (len <= segsiz) { /* * Must have ran out of mbufs for the copy @@ -15405,13 +15409,13 @@ goto failed; } rack_log_output(tp, &to, len, rsm->r_start, flags, error, rack_to_usec_ts(tv), - rsm, RACK_SENT_FP, rsm->m, rsm->soff); + rsm, RACK_SENT_FP, rsm->m, rsm->soff, rsm->r_hw_tls); if (doing_tlp && (rack->fast_rsm_hack == 0)) { rack->rc_tlp_in_progress = 1; rack->r_ctl.rc_tlp_cnt_out++; } if (error == 0) - tcp_account_for_send(tp, len, 1, doing_tlp); + tcp_account_for_send(tp, len, 1, doing_tlp, rsm->r_hw_tls); tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); rack->forced_ack = 0; /* If we send something zap the FA flag */ if (IN_FASTRECOVERY(tp->t_flags) && rsm) @@ -15688,7 +15692,8 @@ goto failed; /* s_mb and s_soff are saved for rack_log_output */ - m->m_next = rack_fo_m_copym(rack, &len, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, &s_mb, &s_soff); + m->m_next = rack_fo_m_copym(rack, &len, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, + &s_mb, &s_soff); if (len <= segsiz) { /* * Must have ran out of mbufs for the copy @@ -15883,7 +15888,7 @@ goto failed; } rack_log_output(tp, &to, len, tp->snd_max, flags, error, rack_to_usec_ts(tv), - NULL, add_flag, s_mb, s_soff); + NULL, add_flag, s_mb, s_soff, rack->r_ctl.fsb.hw_tls); m = NULL; if (tp->snd_una == tp->snd_max) { rack->r_ctl.rc_tlp_rxt_last_time = cts; @@ -15891,7 +15896,7 @@ tp->t_acktime = ticks; } if (error == 0) - tcp_account_for_send(tp, len, 0, 0); + tcp_account_for_send(tp, len, 0, 0, rack->r_ctl.fsb.hw_tls); rack->forced_ack = 0; /* If we send something zap the FA flag */ tot_len += len; @@ -17030,6 +17035,10 @@ rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len; rack->r_ctl.fsb.tcp_flags = flags; rack->r_ctl.fsb.left_to_send = orig_len - len; + if (hw_tls) + rack->r_ctl.fsb.hw_tls = 1; + else + rack->r_ctl.fsb.hw_tls = 0; KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))), ("rack:%p left_to_send:%u sbavail:%u out:%u", rack, rack->r_ctl.fsb.left_to_send, sbavail(sb), @@ -18118,7 +18127,7 @@ * retransmit. In persist state, just set snd_max. */ if (error == 0) { - tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp); + tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls); rack->forced_ack = 0; /* If we send something zap the FA flag */ if (rsm && (doing_tlp == 0)) { /* Set we retransmitted */ @@ -18166,7 +18175,7 @@ } rack_log_output(tp, &to, len, rack_seq, (uint8_t) flags, error, rack_to_usec_ts(&tv), - rsm, add_flag, s_mb, s_moff); + rsm, add_flag, s_mb, s_moff, hw_tls); if ((error == 0) && @@ -18489,6 +18498,10 @@ rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len; rack->r_ctl.fsb.tcp_flags = flags; rack->r_ctl.fsb.left_to_send = orig_len - len; + if (hw_tls) + rack->r_ctl.fsb.hw_tls = 1; + else + rack->r_ctl.fsb.hw_tls = 0; KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))), ("rack:%p left_to_send:%u sbavail:%u out:%u", rack, rack->r_ctl.fsb.left_to_send, sbavail(sb), @@ -18535,6 +18548,10 @@ rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len; rack->r_ctl.fsb.tcp_flags = flags; rack->r_ctl.fsb.left_to_send = orig_len - len; + if (hw_tls) + rack->r_ctl.fsb.hw_tls = 1; + else + rack->r_ctl.fsb.hw_tls = 0; KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))), ("rack:%p left_to_send:%u sbavail:%u out:%u", rack, rack->r_ctl.fsb.left_to_send, sbavail(sb), @@ -19458,6 +19475,29 @@ } } +static void +rack_hw_tls_change(struct tcpcb *tp, int chg) +{ + /* + * HW tls state has changed.. fix all + * rsm's in flight. + */ + struct tcp_rack *rack; + struct rack_sendmap *rsm; + + rack = (struct tcp_rack *)tp->t_fb_ptr; + RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) { + if (chg) + rsm->r_hw_tls = 1; + else + rsm->r_hw_tls = 0; + } + if (chg) + rack->r_ctl.fsb.hw_tls = 1; + else + rack->r_ctl.fsb.hw_tls = 0; +} + static int rack_pru_options(struct tcpcb *tp, int flags) { @@ -19483,7 +19523,7 @@ .tfb_tcp_handoff_ok = rack_handoff_ok, .tfb_tcp_mtu_chg = rack_mtu_change, .tfb_pru_options = rack_pru_options, - + .tfb_hwtls_change = rack_hw_tls_change, }; /* Index: sys/netinet/tcp_stacks/tcp_rack.h =================================================================== --- sys/netinet/tcp_stacks/tcp_rack.h +++ sys/netinet/tcp_stacks/tcp_rack.h @@ -68,7 +68,8 @@ uint8_t r_just_ret : 1, /* After sending, the next pkt was just returned, i.e. limited */ r_one_out_nr : 1, /* Special case 1 outstanding and not in recovery */ r_no_rtt_allowed : 1, /* No rtt measurement allowed */ - r_avail : 5; + r_hw_tls : 1, + r_avail : 4; uint64_t r_tim_lastsent[RACK_NUM_OF_RETRANS]; uint64_t r_ack_arrival; /* This is the time of ack-arrival (if SACK'd) */ RB_ENTRY(rack_sendmap) r_next; /* RB Tree next */ @@ -330,7 +331,8 @@ struct mbuf *m; uint32_t o_m_len; uint32_t rfo_apply_push : 1, - unused : 31; + hw_tls : 1, + unused : 30; }; struct rack_control { Index: sys/netinet/tcp_var.h =================================================================== --- sys/netinet/tcp_var.h +++ sys/netinet/tcp_var.h @@ -367,6 +367,7 @@ int (*tfb_tcp_handoff_ok)(struct tcpcb *); void (*tfb_tcp_mtu_chg)(struct tcpcb *); int (*tfb_pru_options)(struct tcpcb *, int); + void (*tfb_hwtls_change)(struct tcpcb *, int); volatile uint32_t tfb_refcnt; uint32_t tfb_flags; uint8_t tfb_id; @@ -1137,7 +1138,8 @@ } static inline void -tcp_account_for_send(struct tcpcb *tp, uint32_t len, uint8_t is_rxt, uint8_t is_tlp) +tcp_account_for_send(struct tcpcb *tp, uint32_t len, uint8_t is_rxt, + uint8_t is_tlp, int hw_tls __unused) { if (is_tlp) { tp->t_sndtlppack++; @@ -1148,9 +1150,7 @@ tp->t_snd_rxt_bytes += len; else tp->t_sndbytes += len; - } - #endif /* _KERNEL */ #endif /* _NETINET_TCP_VAR_H_ */