diff --git a/sys/netinet/tcp_hpts.c b/sys/netinet/tcp_hpts.c index d673ccbe4a73..e88a3f24dfcc 100644 --- a/sys/netinet/tcp_hpts.c +++ b/sys/netinet/tcp_hpts.c @@ -1,2013 +1,2007 @@ /*- * Copyright (c) 2016-2018 Netflix, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include "opt_inet.h" #include "opt_inet6.h" #include "opt_rss.h" /** * Some notes about usage. * * The tcp_hpts system is designed to provide a high precision timer * system for tcp. Its main purpose is to provide a mechanism for * pacing packets out onto the wire. It can be used in two ways * by a given TCP stack (and those two methods can be used simultaneously). * * First, and probably the main thing its used by Rack and BBR, it can * be used to call tcp_output() of a transport stack at some time in the future. * The normal way this is done is that tcp_output() of the stack schedules * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The * slot is the time from now that the stack wants to be called but it * must be converted to tcp_hpts's notion of slot. This is done with * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical * call from the tcp_output() routine might look like: * * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550)); * * The above would schedule tcp_ouput() to be called in 550 useconds. * Note that if using this mechanism the stack will want to add near * its top a check to prevent unwanted calls (from user land or the * arrival of incoming ack's). So it would add something like: * * if (tcp_in_hpts(inp)) * return; * * to prevent output processing until the time alotted has gone by. * Of course this is a bare bones example and the stack will probably * have more consideration then just the above. * * In order to run input queued segments from the HPTS context the * tcp stack must define an input function for * tfb_do_queued_segments(). This function understands * how to dequeue a array of packets that were input and * knows how to call the correct processing routine. * * Locking in this is important as well so most likely the * stack will need to define the tfb_do_segment_nounlock() * splitting tfb_do_segment() into two parts. The main processing * part that does not unlock the INP and returns a value of 1 or 0. * It returns 0 if all is well and the lock was not released. It * returns 1 if we had to destroy the TCB (a reset received etc). * The remains of tfb_do_segment() then become just a simple call * to the tfb_do_segment_nounlock() function and check the return * code and possibly unlock. * * The stack must also set the flag on the INP that it supports this * feature i.e. INP_SUPPORTS_MBUFQ. The LRO code recoginizes * this flag as well and will queue packets when it is set. * There are other flags as well INP_MBUF_QUEUE_READY and * INP_DONT_SACK_QUEUE. The first flag tells the LRO code * that we are in the pacer for output so there is no * need to wake up the hpts system to get immediate * input. The second tells the LRO code that its okay * if a SACK arrives you can still defer input and let * the current hpts timer run (this is usually set when * a rack timer is up so we know SACK's are happening * on the connection already and don't want to wakeup yet). * * There is a common functions within the rack_bbr_common code * version i.e. ctf_do_queued_segments(). This function * knows how to take the input queue of packets from tp->t_inqueue * and process them digging out all the arguments, calling any bpf tap and * calling into tfb_do_segment_nounlock(). The common * function (ctf_do_queued_segments()) requires that * you have defined the tfb_do_segment_nounlock() as * described above. */ #include #include #include #include #include #include #include #include #include /* for proc0 declaration */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef RSS #include #include #endif #define TCPSTATES /* for logging */ #include #include #include #include #include /* required for icmp_var.h */ #include /* for ICMP_BANDLIM */ #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef tcp_offload #include #endif /* * The hpts uses a 102400 wheel. The wheel * defines the time in 10 usec increments (102400 x 10). * This gives a range of 10usec - 1024ms to place * an entry within. If the user requests more than * 1.024 second, a remaineder is attached and the hpts * when seeing the remainder will re-insert the * inpcb forward in time from where it is until * the remainder is zero. */ #define NUM_OF_HPTSI_SLOTS 102400 /* Each hpts has its own p_mtx which is used for locking */ #define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED) #define HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx) #define HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx) struct tcp_hpts_entry { /* Cache line 0x00 */ struct mtx p_mtx; /* Mutex for hpts */ struct timeval p_mysleep; /* Our min sleep time */ uint64_t syscall_cnt; uint64_t sleeping; /* What the actual sleep was (if sleeping) */ uint16_t p_hpts_active; /* Flag that says hpts is awake */ uint8_t p_wheel_complete; /* have we completed the wheel arc walk? */ uint32_t p_curtick; /* Tick in 10 us the hpts is going to */ uint32_t p_runningslot; /* Current tick we are at if we are running */ uint32_t p_prev_slot; /* Previous slot we were on */ uint32_t p_cur_slot; /* Current slot in wheel hpts is draining */ uint32_t p_nxt_slot; /* The next slot outside the current range of * slots that the hpts is running on. */ int32_t p_on_queue_cnt; /* Count on queue in this hpts */ uint32_t p_lasttick; /* Last tick before the current one */ uint8_t p_direct_wake :1, /* boolean */ p_on_min_sleep:1, /* boolean */ p_hpts_wake_scheduled:1, /* boolean */ p_avail:5; uint8_t p_fill[3]; /* Fill to 32 bits */ /* Cache line 0x40 */ struct hptsh { TAILQ_HEAD(, tcpcb) head; uint32_t count; uint32_t gencnt; } *p_hptss; /* Hptsi wheel */ uint32_t p_hpts_sleep_time; /* Current sleep interval having a max * of 255ms */ uint32_t overidden_sleep; /* what was overrided by min-sleep for logging */ uint32_t saved_lasttick; /* for logging */ uint32_t saved_curtick; /* for logging */ uint32_t saved_curslot; /* for logging */ uint32_t saved_prev_slot; /* for logging */ uint32_t p_delayed_by; /* How much were we delayed by */ /* Cache line 0x80 */ struct sysctl_ctx_list hpts_ctx; struct sysctl_oid *hpts_root; struct intr_event *ie; void *ie_cookie; uint16_t p_num; /* The hpts number one per cpu */ uint16_t p_cpu; /* The hpts CPU */ /* There is extra space in here */ /* Cache line 0x100 */ struct callout co __aligned(CACHE_LINE_SIZE); } __aligned(CACHE_LINE_SIZE); static struct tcp_hptsi { struct cpu_group **grps; struct tcp_hpts_entry **rp_ent; /* Array of hptss */ uint32_t *cts_last_ran; uint32_t grp_cnt; uint32_t rp_num_hptss; /* Number of hpts threads */ } tcp_pace; MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts"); #ifdef RSS static int tcp_bind_threads = 1; #else static int tcp_bind_threads = 2; #endif static int tcp_use_irq_cpu = 0; static uint32_t *cts_last_ran; static int hpts_does_tp_logging = 0; static int32_t tcp_hptsi(struct tcp_hpts_entry *hpts, int from_callout); static void tcp_hpts_thread(void *ctx); static void tcp_init_hptsi(void *st); int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP; static int conn_cnt_thresh = DEFAULT_CONNECTION_THESHOLD; static int32_t dynamic_min_sleep = DYNAMIC_MIN_SLEEP; static int32_t dynamic_max_sleep = DYNAMIC_MAX_SLEEP; SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "TCP Hpts controls"); SYSCTL_NODE(_net_inet_tcp_hpts, OID_AUTO, stats, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "TCP Hpts statistics"); #define timersub(tvp, uvp, vvp) \ do { \ (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ if ((vvp)->tv_usec < 0) { \ (vvp)->tv_sec--; \ (vvp)->tv_usec += 1000000; \ } \ } while (0) static int32_t tcp_hpts_precision = 120; static struct hpts_domain_info { int count; int cpu[MAXCPU]; } hpts_domains[MAXMEMDOM]; counter_u64_t hpts_hopelessly_behind; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, hopeless, CTLFLAG_RD, &hpts_hopelessly_behind, "Number of times hpts could not catch up and was behind hopelessly"); counter_u64_t hpts_loops; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, loops, CTLFLAG_RD, &hpts_loops, "Number of times hpts had to loop to catch up"); counter_u64_t back_tosleep; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, no_tcbsfound, CTLFLAG_RD, &back_tosleep, "Number of times hpts found no tcbs"); counter_u64_t combined_wheel_wrap; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, comb_wheel_wrap, CTLFLAG_RD, &combined_wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap"); counter_u64_t wheel_wrap; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, wheel_wrap, CTLFLAG_RD, &wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap"); counter_u64_t hpts_direct_call; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, direct_call, CTLFLAG_RD, &hpts_direct_call, "Number of times hpts was called by syscall/trap or other entry"); counter_u64_t hpts_wake_timeout; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, timeout_wakeup, CTLFLAG_RD, &hpts_wake_timeout, "Number of times hpts threads woke up via the callout expiring"); counter_u64_t hpts_direct_awakening; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, direct_awakening, CTLFLAG_RD, &hpts_direct_awakening, "Number of times hpts threads woke up via the callout expiring"); counter_u64_t hpts_back_tosleep; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, back_tosleep, CTLFLAG_RD, &hpts_back_tosleep, "Number of times hpts threads woke up via the callout expiring and went back to sleep no work"); counter_u64_t cpu_uses_flowid; counter_u64_t cpu_uses_random; SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_flowid, CTLFLAG_RD, &cpu_uses_flowid, "Number of times when setting cpuid we used the flowid field"); SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_random, CTLFLAG_RD, &cpu_uses_random, "Number of times when setting cpuid we used the a random value"); TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads); TUNABLE_INT("net.inet.tcp.use_irq", &tcp_use_irq_cpu); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, bind_hptss, CTLFLAG_RD, &tcp_bind_threads, 2, "Thread Binding tunable"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, use_irq, CTLFLAG_RD, &tcp_use_irq_cpu, 0, "Use of irq CPU tunable"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW, &tcp_hpts_precision, 120, "Value for PRE() precision of callout"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, cnt_thresh, CTLFLAG_RW, &conn_cnt_thresh, 0, "How many connections (below) make us use the callout based mechanism"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW, &hpts_does_tp_logging, 0, "Do we add to any tp that has logging on pacer logs"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, dyn_minsleep, CTLFLAG_RW, &dynamic_min_sleep, 250, "What is the dynamic minsleep value?"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, dyn_maxsleep, CTLFLAG_RW, &dynamic_max_sleep, 5000, "What is the dynamic maxsleep value?"); static int32_t max_pacer_loops = 10; SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, loopmax, CTLFLAG_RW, &max_pacer_loops, 10, "What is the maximum number of times the pacer will loop trying to catch up"); #define HPTS_MAX_SLEEP_ALLOWED (NUM_OF_HPTSI_SLOTS/2) static uint32_t hpts_sleep_max = HPTS_MAX_SLEEP_ALLOWED; static int sysctl_net_inet_tcp_hpts_max_sleep(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = hpts_sleep_max; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr) { if ((new < (dynamic_min_sleep/HPTS_TICKS_PER_SLOT)) || (new > HPTS_MAX_SLEEP_ALLOWED)) error = EINVAL; else hpts_sleep_max = new; } return (error); } static int sysctl_net_inet_tcp_hpts_min_sleep(SYSCTL_HANDLER_ARGS) { int error; uint32_t new; new = tcp_min_hptsi_time; error = sysctl_handle_int(oidp, &new, 0, req); if (error == 0 && req->newptr) { if (new < LOWEST_SLEEP_ALLOWED) error = EINVAL; else tcp_min_hptsi_time = new; } return (error); } SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLTYPE_UINT | CTLFLAG_RW, &hpts_sleep_max, 0, &sysctl_net_inet_tcp_hpts_max_sleep, "IU", "Maximum time hpts will sleep in slots"); SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLTYPE_UINT | CTLFLAG_RW, &tcp_min_hptsi_time, 0, &sysctl_net_inet_tcp_hpts_min_sleep, "IU", "The minimum time the hpts must sleep before processing more slots"); static int ticks_indicate_more_sleep = TICKS_INDICATE_MORE_SLEEP; static int ticks_indicate_less_sleep = TICKS_INDICATE_LESS_SLEEP; static int tcp_hpts_no_wake_over_thresh = 1; SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, more_sleep, CTLFLAG_RW, &ticks_indicate_more_sleep, 0, "If we only process this many or less on a timeout, we need longer sleep on the next callout"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, less_sleep, CTLFLAG_RW, &ticks_indicate_less_sleep, 0, "If we process this many or more on a timeout, we need less sleep on the next callout"); SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, nowake_over_thresh, CTLFLAG_RW, &tcp_hpts_no_wake_over_thresh, 0, "When we are over the threshold on the pacer do we prohibit wakeups?"); static uint16_t hpts_random_cpu(void) { uint16_t cpuid; uint32_t ran; ran = arc4random(); cpuid = (((ran & 0xffff) % mp_ncpus) % tcp_pace.rp_num_hptss); return (cpuid); } static void tcp_hpts_log(struct tcp_hpts_entry *hpts, struct tcpcb *tp, struct timeval *tv, int slots_to_run, int idx, int from_callout) { union tcp_log_stackspecific log; /* * Unused logs are * 64 bit - delRate, rttProp, bw_inuse * 16 bit - cwnd_gain * 8 bit - bbr_state, bbr_substate, inhpts; */ memset(&log.u_bbr, 0, sizeof(log.u_bbr)); log.u_bbr.flex1 = hpts->p_nxt_slot; log.u_bbr.flex2 = hpts->p_cur_slot; log.u_bbr.flex3 = hpts->p_prev_slot; log.u_bbr.flex4 = idx; log.u_bbr.flex5 = hpts->p_curtick; log.u_bbr.flex6 = hpts->p_on_queue_cnt; log.u_bbr.flex7 = hpts->p_cpu; log.u_bbr.flex8 = (uint8_t)from_callout; log.u_bbr.inflight = slots_to_run; log.u_bbr.applimited = hpts->overidden_sleep; log.u_bbr.delivered = hpts->saved_curtick; log.u_bbr.timeStamp = tcp_tv_to_usectick(tv); log.u_bbr.epoch = hpts->saved_curslot; log.u_bbr.lt_epoch = hpts->saved_prev_slot; log.u_bbr.pkts_out = hpts->p_delayed_by; log.u_bbr.lost = hpts->p_hpts_sleep_time; log.u_bbr.pacing_gain = hpts->p_cpu; log.u_bbr.pkt_epoch = hpts->p_runningslot; log.u_bbr.use_lt_bw = 1; TCP_LOG_EVENTP(tp, NULL, &tptosocket(tp)->so_rcv, &tptosocket(tp)->so_snd, BBR_LOG_HPTSDIAG, 0, 0, &log, false, tv); } static void tcp_wakehpts(struct tcp_hpts_entry *hpts) { HPTS_MTX_ASSERT(hpts); if (tcp_hpts_no_wake_over_thresh && (hpts->p_on_queue_cnt >= conn_cnt_thresh)) { hpts->p_direct_wake = 0; return; } if (hpts->p_hpts_wake_scheduled == 0) { hpts->p_hpts_wake_scheduled = 1; swi_sched(hpts->ie_cookie, 0); } } static void hpts_timeout_swi(void *arg) { struct tcp_hpts_entry *hpts; hpts = (struct tcp_hpts_entry *)arg; swi_sched(hpts->ie_cookie, 0); } static void tcp_hpts_insert_internal(struct tcpcb *tp, struct tcp_hpts_entry *hpts) { struct inpcb *inp = tptoinpcb(tp); struct hptsh *hptsh; INP_WLOCK_ASSERT(inp); HPTS_MTX_ASSERT(hpts); MPASS(hpts->p_cpu == tp->t_hpts_cpu); MPASS(!(inp->inp_flags & INP_DROPPED)); hptsh = &hpts->p_hptss[tp->t_hpts_slot]; if (tp->t_in_hpts == IHPTS_NONE) { tp->t_in_hpts = IHPTS_ONQUEUE; in_pcbref(inp); } else if (tp->t_in_hpts == IHPTS_MOVING) { tp->t_in_hpts = IHPTS_ONQUEUE; } else MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); tp->t_hpts_gencnt = hptsh->gencnt; TAILQ_INSERT_TAIL(&hptsh->head, tp, t_hpts); hptsh->count++; hpts->p_on_queue_cnt++; } static struct tcp_hpts_entry * tcp_hpts_lock(struct tcpcb *tp) { struct tcp_hpts_entry *hpts; INP_LOCK_ASSERT(tptoinpcb(tp)); hpts = tcp_pace.rp_ent[tp->t_hpts_cpu]; HPTS_LOCK(hpts); return (hpts); } static void tcp_hpts_release(struct tcpcb *tp) { bool released __diagused; tp->t_in_hpts = IHPTS_NONE; released = in_pcbrele_wlocked(tptoinpcb(tp)); MPASS(released == false); } /* * Initialize newborn tcpcb to get ready for use with HPTS. */ void tcp_hpts_init(struct tcpcb *tp) { tp->t_hpts_cpu = hpts_random_cpu(); tp->t_lro_cpu = HPTS_CPU_NONE; MPASS(!(tp->t_flags2 & TF2_HPTS_CPU_SET)); } /* * Called normally with the INP_LOCKED but it * does not matter, the hpts lock is the key * but the lock order allows us to hold the * INP lock and then get the hpts lock. */ void tcp_hpts_remove(struct tcpcb *tp) { struct tcp_hpts_entry *hpts; struct hptsh *hptsh; INP_WLOCK_ASSERT(tptoinpcb(tp)); hpts = tcp_hpts_lock(tp); if (tp->t_in_hpts == IHPTS_ONQUEUE) { hptsh = &hpts->p_hptss[tp->t_hpts_slot]; tp->t_hpts_request = 0; if (__predict_true(tp->t_hpts_gencnt == hptsh->gencnt)) { TAILQ_REMOVE(&hptsh->head, tp, t_hpts); MPASS(hptsh->count > 0); hptsh->count--; MPASS(hpts->p_on_queue_cnt > 0); hpts->p_on_queue_cnt--; tcp_hpts_release(tp); } else { /* * tcp_hptsi() now owns the TAILQ head of this inp. * Can't TAILQ_REMOVE, just mark it. */ #ifdef INVARIANTS struct tcpcb *tmp; TAILQ_FOREACH(tmp, &hptsh->head, t_hpts) MPASS(tmp != tp); #endif tp->t_in_hpts = IHPTS_MOVING; tp->t_hpts_slot = -1; } } else if (tp->t_in_hpts == IHPTS_MOVING) { /* * Handle a special race condition: * tcp_hptsi() moves inpcb to detached tailq * tcp_hpts_remove() marks as IHPTS_MOVING, slot = -1 * tcp_hpts_insert() sets slot to a meaningful value * tcp_hpts_remove() again (we are here!), then in_pcbdrop() * tcp_hptsi() finds pcb with meaningful slot and INP_DROPPED */ tp->t_hpts_slot = -1; } HPTS_UNLOCK(hpts); } static inline int hpts_slot(uint32_t wheel_slot, uint32_t plus) { /* * Given a slot on the wheel, what slot * is that plus ticks out? */ KASSERT(wheel_slot < NUM_OF_HPTSI_SLOTS, ("Invalid tick %u not on wheel", wheel_slot)); return ((wheel_slot + plus) % NUM_OF_HPTSI_SLOTS); } static inline int tick_to_wheel(uint32_t cts_in_wticks) { /* * Given a timestamp in ticks (so by * default to get it to a real time one * would multiply by 10.. i.e the number * of ticks in a slot) map it to our limited * space wheel. */ return (cts_in_wticks % NUM_OF_HPTSI_SLOTS); } static inline int hpts_slots_diff(int prev_slot, int slot_now) { /* * Given two slots that are someplace * on our wheel. How far are they apart? */ if (slot_now > prev_slot) return (slot_now - prev_slot); else if (slot_now == prev_slot) /* * Special case, same means we can go all of our * wheel less one slot. */ return (NUM_OF_HPTSI_SLOTS - 1); else return ((NUM_OF_HPTSI_SLOTS - prev_slot) + slot_now); } /* * Given a slot on the wheel that is the current time * mapped to the wheel (wheel_slot), what is the maximum * distance forward that can be obtained without * wrapping past either prev_slot or running_slot * depending on the htps state? Also if passed * a uint32_t *, fill it with the slot location. * * Note if you do not give this function the current * time (that you think it is) mapped to the wheel slot * then the results will not be what you expect and * could lead to invalid inserts. */ static inline int32_t max_slots_available(struct tcp_hpts_entry *hpts, uint32_t wheel_slot, uint32_t *target_slot) { uint32_t dis_to_travel, end_slot, pacer_to_now, avail_on_wheel; if ((hpts->p_hpts_active == 1) && (hpts->p_wheel_complete == 0)) { end_slot = hpts->p_runningslot; /* Back up one tick */ if (end_slot == 0) end_slot = NUM_OF_HPTSI_SLOTS - 1; else end_slot--; if (target_slot) *target_slot = end_slot; } else { /* * For the case where we are * not active, or we have * completed the pass over * the wheel, we can use the * prev tick and subtract one from it. This puts us * as far out as possible on the wheel. */ end_slot = hpts->p_prev_slot; if (end_slot == 0) end_slot = NUM_OF_HPTSI_SLOTS - 1; else end_slot--; if (target_slot) *target_slot = end_slot; /* * Now we have close to the full wheel left minus the * time it has been since the pacer went to sleep. Note * that wheel_tick, passed in, should be the current time * from the perspective of the caller, mapped to the wheel. */ if (hpts->p_prev_slot != wheel_slot) dis_to_travel = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); else dis_to_travel = 1; /* * dis_to_travel in this case is the space from when the * pacer stopped (p_prev_slot) and where our wheel_slot * is now. To know how many slots we can put it in we * subtract from the wheel size. We would not want * to place something after p_prev_slot or it will * get ran too soon. */ return (NUM_OF_HPTSI_SLOTS - dis_to_travel); } /* * So how many slots are open between p_runningslot -> p_cur_slot * that is what is currently un-available for insertion. Special * case when we are at the last slot, this gets 1, so that * the answer to how many slots are available is all but 1. */ if (hpts->p_runningslot == hpts->p_cur_slot) dis_to_travel = 1; else dis_to_travel = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); /* * How long has the pacer been running? */ if (hpts->p_cur_slot != wheel_slot) { /* The pacer is a bit late */ pacer_to_now = hpts_slots_diff(hpts->p_cur_slot, wheel_slot); } else { /* The pacer is right on time, now == pacers start time */ pacer_to_now = 0; } /* * To get the number left we can insert into we simply * subtract the distance the pacer has to run from how * many slots there are. */ avail_on_wheel = NUM_OF_HPTSI_SLOTS - dis_to_travel; /* * Now how many of those we will eat due to the pacer's * time (p_cur_slot) of start being behind the * real time (wheel_slot)? */ if (avail_on_wheel <= pacer_to_now) { /* * Wheel wrap, we can't fit on the wheel, that * is unusual the system must be way overloaded! * Insert into the assured slot, and return special * "0". */ counter_u64_add(combined_wheel_wrap, 1); *target_slot = hpts->p_nxt_slot; return (0); } else { /* * We know how many slots are open * on the wheel (the reverse of what * is left to run. Take away the time * the pacer started to now (wheel_slot) * and that tells you how many slots are * open that can be inserted into that won't * be touched by the pacer until later. */ return (avail_on_wheel - pacer_to_now); } } #ifdef INVARIANTS static void check_if_slot_would_be_wrong(struct tcp_hpts_entry *hpts, struct tcpcb *tp, uint32_t hptsslot, int line) { /* * Sanity checks for the pacer with invariants * on insert. */ KASSERT(hptsslot < NUM_OF_HPTSI_SLOTS, ("hpts:%p tp:%p slot:%d > max", hpts, tp, hptsslot)); if ((hpts->p_hpts_active) && (hpts->p_wheel_complete == 0)) { /* * If the pacer is processing a arc * of the wheel, we need to make * sure we are not inserting within * that arc. */ int distance, yet_to_run; distance = hpts_slots_diff(hpts->p_runningslot, hptsslot); if (hpts->p_runningslot != hpts->p_cur_slot) yet_to_run = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); else yet_to_run = 0; /* processing last slot */ KASSERT(yet_to_run <= distance, ("hpts:%p tp:%p slot:%d " "distance:%d yet_to_run:%d rs:%d cs:%d", hpts, tp, hptsslot, distance, yet_to_run, hpts->p_runningslot, hpts->p_cur_slot)); } } #endif uint32_t tcp_hpts_insert_diag(struct tcpcb *tp, uint32_t slot, int32_t line, struct hpts_diag *diag) { struct tcp_hpts_entry *hpts; struct timeval tv; uint32_t slot_on, wheel_cts, last_slot, need_new_to = 0; int32_t wheel_slot, maxslots; bool need_wakeup = false; INP_WLOCK_ASSERT(tptoinpcb(tp)); MPASS(!(tptoinpcb(tp)->inp_flags & INP_DROPPED)); MPASS(!tcp_in_hpts(tp)); /* * We now return the next-slot the hpts will be on, beyond its * current run (if up) or where it was when it stopped if it is * sleeping. */ hpts = tcp_hpts_lock(tp); microuptime(&tv); if (diag) { memset(diag, 0, sizeof(struct hpts_diag)); diag->p_hpts_active = hpts->p_hpts_active; diag->p_prev_slot = hpts->p_prev_slot; diag->p_runningslot = hpts->p_runningslot; diag->p_nxt_slot = hpts->p_nxt_slot; diag->p_cur_slot = hpts->p_cur_slot; diag->p_curtick = hpts->p_curtick; diag->p_lasttick = hpts->p_lasttick; diag->slot_req = slot; diag->p_on_min_sleep = hpts->p_on_min_sleep; diag->hpts_sleep_time = hpts->p_hpts_sleep_time; } if (slot == 0) { /* Ok we need to set it on the hpts in the current slot */ tp->t_hpts_request = 0; if ((hpts->p_hpts_active == 0) || (hpts->p_wheel_complete)) { /* * A sleeping hpts we want in next slot to run * note that in this state p_prev_slot == p_cur_slot */ tp->t_hpts_slot = hpts_slot(hpts->p_prev_slot, 1); if ((hpts->p_on_min_sleep == 0) && (hpts->p_hpts_active == 0)) need_wakeup = true; } else tp->t_hpts_slot = hpts->p_runningslot; if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) tcp_hpts_insert_internal(tp, hpts); if (need_wakeup) { /* * Activate the hpts if it is sleeping and its * timeout is not 1. */ hpts->p_direct_wake = 1; tcp_wakehpts(hpts); } slot_on = hpts->p_nxt_slot; HPTS_UNLOCK(hpts); return (slot_on); } /* Get the current time relative to the wheel */ wheel_cts = tcp_tv_to_hptstick(&tv); /* Map it onto the wheel */ wheel_slot = tick_to_wheel(wheel_cts); /* Now what's the max we can place it at? */ maxslots = max_slots_available(hpts, wheel_slot, &last_slot); if (diag) { diag->wheel_slot = wheel_slot; diag->maxslots = maxslots; diag->wheel_cts = wheel_cts; } if (maxslots == 0) { /* The pacer is in a wheel wrap behind, yikes! */ if (slot > 1) { /* * Reduce by 1 to prevent a forever loop in * case something else is wrong. Note this * probably does not hurt because the pacer * if its true is so far behind we will be * > 1second late calling anyway. */ slot--; } tp->t_hpts_slot = last_slot; tp->t_hpts_request = slot; } else if (maxslots >= slot) { /* It all fits on the wheel */ tp->t_hpts_request = 0; tp->t_hpts_slot = hpts_slot(wheel_slot, slot); } else { /* It does not fit */ tp->t_hpts_request = slot - maxslots; tp->t_hpts_slot = last_slot; } if (diag) { diag->slot_remaining = tp->t_hpts_request; diag->inp_hptsslot = tp->t_hpts_slot; } #ifdef INVARIANTS check_if_slot_would_be_wrong(hpts, tp, tp->t_hpts_slot, line); #endif if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) tcp_hpts_insert_internal(tp, hpts); if ((hpts->p_hpts_active == 0) && (tp->t_hpts_request == 0) && (hpts->p_on_min_sleep == 0)) { /* * The hpts is sleeping and NOT on a minimum * sleep time, we need to figure out where * it will wake up at and if we need to reschedule * its time-out. */ uint32_t have_slept, yet_to_sleep; /* Now do we need to restart the hpts's timer? */ have_slept = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); if (have_slept < hpts->p_hpts_sleep_time) yet_to_sleep = hpts->p_hpts_sleep_time - have_slept; else { /* We are over-due */ yet_to_sleep = 0; need_wakeup = 1; } if (diag) { diag->have_slept = have_slept; diag->yet_to_sleep = yet_to_sleep; } if (yet_to_sleep && (yet_to_sleep > slot)) { /* * We need to reschedule the hpts's time-out. */ hpts->p_hpts_sleep_time = slot; need_new_to = slot * HPTS_TICKS_PER_SLOT; } } /* * Now how far is the hpts sleeping to? if active is 1, its * up and ticking we do nothing, otherwise we may need to * reschedule its callout if need_new_to is set from above. */ if (need_wakeup) { hpts->p_direct_wake = 1; tcp_wakehpts(hpts); if (diag) { diag->need_new_to = 0; diag->co_ret = 0xffff0000; } } else if (need_new_to) { int32_t co_ret; struct timeval tv; sbintime_t sb; tv.tv_sec = 0; tv.tv_usec = 0; while (need_new_to > HPTS_USEC_IN_SEC) { tv.tv_sec++; need_new_to -= HPTS_USEC_IN_SEC; } tv.tv_usec = need_new_to; sb = tvtosbt(tv); co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, hpts_timeout_swi, hpts, hpts->p_cpu, (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); if (diag) { diag->need_new_to = need_new_to; diag->co_ret = co_ret; } } slot_on = hpts->p_nxt_slot; HPTS_UNLOCK(hpts); return (slot_on); } static uint16_t hpts_cpuid(struct tcpcb *tp, int *failed) { struct inpcb *inp = tptoinpcb(tp); u_int cpuid; #ifdef NUMA struct hpts_domain_info *di; #endif *failed = 0; if (tp->t_flags2 & TF2_HPTS_CPU_SET) { return (tp->t_hpts_cpu); } /* * If we are using the irq cpu set by LRO or * the driver then it overrides all other domains. */ if (tcp_use_irq_cpu) { if (tp->t_lro_cpu == HPTS_CPU_NONE) { *failed = 1; return (0); } return (tp->t_lro_cpu); } /* If one is set the other must be the same */ #ifdef RSS cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); if (cpuid == NETISR_CPUID_NONE) return (hpts_random_cpu()); else return (cpuid); #endif /* * We don't have a flowid -> cpuid mapping, so cheat and just map * unknown cpuids to curcpu. Not the best, but apparently better * than defaulting to swi 0. */ if (inp->inp_flowtype == M_HASHTYPE_NONE) { counter_u64_add(cpu_uses_random, 1); return (hpts_random_cpu()); } /* * Hash to a thread based on the flowid. If we are using numa, * then restrict the hash to the numa domain where the inp lives. */ #ifdef NUMA if ((vm_ndomains == 1) || (inp->inp_numa_domain == M_NODOM)) { #endif cpuid = inp->inp_flowid % mp_ncpus; #ifdef NUMA } else { /* Hash into the cpu's that use that domain */ di = &hpts_domains[inp->inp_numa_domain]; cpuid = di->cpu[inp->inp_flowid % di->count]; } #endif counter_u64_add(cpu_uses_flowid, 1); return (cpuid); } static void tcp_hpts_set_max_sleep(struct tcp_hpts_entry *hpts, int wrap_loop_cnt) { uint32_t t = 0, i; if ((hpts->p_on_queue_cnt) && (wrap_loop_cnt < 2)) { /* * Find next slot that is occupied and use that to * be the sleep time. */ for (i = 0, t = hpts_slot(hpts->p_cur_slot, 1); i < NUM_OF_HPTSI_SLOTS; i++) { if (TAILQ_EMPTY(&hpts->p_hptss[t].head) == 0) { break; } t = (t + 1) % NUM_OF_HPTSI_SLOTS; } KASSERT((i != NUM_OF_HPTSI_SLOTS), ("Hpts:%p cnt:%d but none found", hpts, hpts->p_on_queue_cnt)); hpts->p_hpts_sleep_time = min((i + 1), hpts_sleep_max); } else { /* No one on the wheel sleep for all but 400 slots or sleep max */ hpts->p_hpts_sleep_time = hpts_sleep_max; } } static int32_t tcp_hptsi(struct tcp_hpts_entry *hpts, int from_callout) { struct tcpcb *tp; struct timeval tv; int32_t slots_to_run, i, error; int32_t loop_cnt = 0; int32_t did_prefetch = 0; int32_t prefetch_tp = 0; int32_t wrap_loop_cnt = 0; int32_t slot_pos_of_endpoint = 0; int32_t orig_exit_slot; int8_t completed_measure = 0, seen_endpoint = 0; HPTS_MTX_ASSERT(hpts); NET_EPOCH_ASSERT(); /* record previous info for any logging */ hpts->saved_lasttick = hpts->p_lasttick; hpts->saved_curtick = hpts->p_curtick; hpts->saved_curslot = hpts->p_cur_slot; hpts->saved_prev_slot = hpts->p_prev_slot; hpts->p_lasttick = hpts->p_curtick; hpts->p_curtick = tcp_gethptstick(&tv); cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv); orig_exit_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); if ((hpts->p_on_queue_cnt == 0) || (hpts->p_lasttick == hpts->p_curtick)) { /* * No time has yet passed, * or nothing to do. */ hpts->p_prev_slot = hpts->p_cur_slot; hpts->p_lasttick = hpts->p_curtick; goto no_run; } again: hpts->p_wheel_complete = 0; HPTS_MTX_ASSERT(hpts); slots_to_run = hpts_slots_diff(hpts->p_prev_slot, hpts->p_cur_slot); if (((hpts->p_curtick - hpts->p_lasttick) > ((NUM_OF_HPTSI_SLOTS-1) * HPTS_TICKS_PER_SLOT)) && (hpts->p_on_queue_cnt != 0)) { /* * Wheel wrap is occuring, basically we * are behind and the distance between * run's has spread so much it has exceeded * the time on the wheel (1.024 seconds). This * is ugly and should NOT be happening. We * need to run the entire wheel. We last processed * p_prev_slot, so that needs to be the last slot * we run. The next slot after that should be our * reserved first slot for new, and then starts * the running position. Now the problem is the * reserved "not to yet" place does not exist * and there may be inp's in there that need * running. We can merge those into the * first slot at the head. */ wrap_loop_cnt++; hpts->p_nxt_slot = hpts_slot(hpts->p_prev_slot, 1); hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 2); /* * Adjust p_cur_slot to be where we are starting from * hopefully we will catch up (fat chance if something * is broken this bad :( ) */ hpts->p_cur_slot = hpts->p_prev_slot; /* * The next slot has guys to run too, and that would * be where we would normally start, lets move them into * the next slot (p_prev_slot + 2) so that we will * run them, the extra 10usecs of late (by being * put behind) does not really matter in this situation. */ TAILQ_FOREACH(tp, &hpts->p_hptss[hpts->p_nxt_slot].head, t_hpts) { MPASS(tp->t_hpts_slot == hpts->p_nxt_slot); MPASS(tp->t_hpts_gencnt == hpts->p_hptss[hpts->p_nxt_slot].gencnt); MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); /* * Update gencnt and nextslot accordingly to match * the new location. This is safe since it takes both * the INP lock and the pacer mutex to change the * t_hptsslot and t_hpts_gencnt. */ tp->t_hpts_gencnt = hpts->p_hptss[hpts->p_runningslot].gencnt; tp->t_hpts_slot = hpts->p_runningslot; } TAILQ_CONCAT(&hpts->p_hptss[hpts->p_runningslot].head, &hpts->p_hptss[hpts->p_nxt_slot].head, t_hpts); hpts->p_hptss[hpts->p_runningslot].count += hpts->p_hptss[hpts->p_nxt_slot].count; hpts->p_hptss[hpts->p_nxt_slot].count = 0; hpts->p_hptss[hpts->p_nxt_slot].gencnt++; slots_to_run = NUM_OF_HPTSI_SLOTS - 1; counter_u64_add(wheel_wrap, 1); } else { /* * Nxt slot is always one after p_runningslot though * its not used usually unless we are doing wheel wrap. */ hpts->p_nxt_slot = hpts->p_prev_slot; hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 1); } if (hpts->p_on_queue_cnt == 0) { goto no_one; } for (i = 0; i < slots_to_run; i++) { struct tcpcb *tp, *ntp; TAILQ_HEAD(, tcpcb) head = TAILQ_HEAD_INITIALIZER(head); struct hptsh *hptsh; uint32_t runningslot; /* * Calculate our delay, if there are no extra ticks there * was not any (i.e. if slots_to_run == 1, no delay). */ hpts->p_delayed_by = (slots_to_run - (i + 1)) * HPTS_TICKS_PER_SLOT; runningslot = hpts->p_runningslot; hptsh = &hpts->p_hptss[runningslot]; TAILQ_SWAP(&head, &hptsh->head, tcpcb, t_hpts); hpts->p_on_queue_cnt -= hptsh->count; hptsh->count = 0; hptsh->gencnt++; HPTS_UNLOCK(hpts); TAILQ_FOREACH_SAFE(tp, &head, t_hpts, ntp) { struct inpcb *inp = tptoinpcb(tp); bool set_cpu; if (ntp != NULL) { /* * If we have a next tcpcb, see if we can * prefetch it. Note this may seem * "risky" since we have no locks (other * than the previous inp) and there no * assurance that ntp was not pulled while * we were processing tp and freed. If this * occurred it could mean that either: * * a) Its NULL (which is fine we won't go * here) b) Its valid (which is cool we * will prefetch it) c) The inp got * freed back to the slab which was * reallocated. Then the piece of memory was * re-used and something else (not an * address) is in inp_ppcb. If that occurs * we don't crash, but take a TLB shootdown * performance hit (same as if it was NULL * and we tried to pre-fetch it). * * Considering that the likelyhood of is * quite rare we will take a risk on doing * this. If performance drops after testing * we can always take this out. NB: the * kern_prefetch on amd64 actually has * protection against a bad address now via * the DMAP_() tests. This will prevent the * TLB hit, and instead if occurs just * cause us to load cache with a useless * address (to us). * * XXXGL: this comment and the prefetch action * could be outdated after tp == inp change. */ kern_prefetch(ntp, &prefetch_tp); prefetch_tp = 1; } /* For debugging */ if (seen_endpoint == 0) { seen_endpoint = 1; orig_exit_slot = slot_pos_of_endpoint = runningslot; } else if (completed_measure == 0) { /* Record the new position */ orig_exit_slot = runningslot; } INP_WLOCK(inp); if ((tp->t_flags2 & TF2_HPTS_CPU_SET) == 0) { set_cpu = true; } else { set_cpu = false; } if (__predict_false(tp->t_in_hpts == IHPTS_MOVING)) { if (tp->t_hpts_slot == -1) { tp->t_in_hpts = IHPTS_NONE; if (in_pcbrele_wlocked(inp) == false) INP_WUNLOCK(inp); } else { HPTS_LOCK(hpts); tcp_hpts_insert_internal(tp, hpts); HPTS_UNLOCK(hpts); INP_WUNLOCK(inp); } continue; } MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); MPASS(!(inp->inp_flags & INP_DROPPED)); KASSERT(runningslot == tp->t_hpts_slot, ("Hpts:%p inp:%p slot mis-aligned %u vs %u", hpts, inp, runningslot, tp->t_hpts_slot)); if (tp->t_hpts_request) { /* * This guy is deferred out further in time * then our wheel had available on it. * Push him back on the wheel or run it * depending. */ uint32_t maxslots, last_slot, remaining_slots; remaining_slots = slots_to_run - (i + 1); if (tp->t_hpts_request > remaining_slots) { HPTS_LOCK(hpts); /* * How far out can we go? */ maxslots = max_slots_available(hpts, hpts->p_cur_slot, &last_slot); if (maxslots >= tp->t_hpts_request) { /* We can place it finally to * be processed. */ tp->t_hpts_slot = hpts_slot( hpts->p_runningslot, tp->t_hpts_request); tp->t_hpts_request = 0; } else { /* Work off some more time */ tp->t_hpts_slot = last_slot; tp->t_hpts_request -= maxslots; } tcp_hpts_insert_internal(tp, hpts); HPTS_UNLOCK(hpts); INP_WUNLOCK(inp); continue; } tp->t_hpts_request = 0; /* Fall through we will so do it now */ } tcp_hpts_release(tp); if (set_cpu) { /* * Setup so the next time we will move to * the right CPU. This should be a rare * event. It will sometimes happens when we * are the client side (usually not the * server). Somehow tcp_output() gets called * before the tcp_do_segment() sets the * intial state. This means the r_cpu and * r_hpts_cpu is 0. We get on the hpts, and * then tcp_input() gets called setting up * the r_cpu to the correct value. The hpts * goes off and sees the mis-match. We * simply correct it here and the CPU will * switch to the new hpts nextime the tcb * gets added to the hpts (not this one) * :-) */ tcp_set_hpts(tp); } CURVNET_SET(inp->inp_vnet); /* Lets do any logging that we might want to */ if (hpts_does_tp_logging && tcp_bblogging_on(tp)) { tcp_hpts_log(hpts, tp, &tv, slots_to_run, i, from_callout); } if (tp->t_fb_ptr != NULL) { kern_prefetch(tp->t_fb_ptr, &did_prefetch); did_prefetch = 1; } /* * We set TF2_HPTS_CALLS before any possible output. * The contract with the transport is that if it cares * about hpts calling it should clear the flag. That * way next time it is called it will know it is hpts. * * We also only call tfb_do_queued_segments() * tcp_output(). It is expected that if segments are * queued and come in that the final input mbuf will * cause a call to output if it is needed. */ tp->t_flags2 |= TF2_HPTS_CALLS; if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) && !STAILQ_EMPTY(&tp->t_inqueue)) { error = (*tp->t_fb->tfb_do_queued_segments)(tp, 0); if (error) { /* The input killed the connection */ goto skip_pacing; } } error = tcp_output(tp); if (error < 0) goto skip_pacing; INP_WUNLOCK(inp); skip_pacing: CURVNET_RESTORE(); } if (seen_endpoint) { /* * We now have a accurate distance between * slot_pos_of_endpoint <-> orig_exit_slot * to tell us how late we were, orig_exit_slot * is where we calculated the end of our cycle to * be when we first entered. */ completed_measure = 1; } HPTS_LOCK(hpts); hpts->p_runningslot++; if (hpts->p_runningslot >= NUM_OF_HPTSI_SLOTS) { hpts->p_runningslot = 0; } } no_one: HPTS_MTX_ASSERT(hpts); hpts->p_delayed_by = 0; /* * Check to see if we took an excess amount of time and need to run * more ticks (if we did not hit eno-bufs). */ hpts->p_prev_slot = hpts->p_cur_slot; hpts->p_lasttick = hpts->p_curtick; if ((from_callout == 0) || (loop_cnt > max_pacer_loops)) { /* * Something is serious slow we have * looped through processing the wheel * and by the time we cleared the * needs to run max_pacer_loops time * we still needed to run. That means * the system is hopelessly behind and * can never catch up :( * * We will just lie to this thread * and let it thing p_curtick is * correct. When it next awakens * it will find itself further behind. */ if (from_callout) counter_u64_add(hpts_hopelessly_behind, 1); goto no_run; } hpts->p_curtick = tcp_gethptstick(&tv); hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); if (seen_endpoint == 0) { /* We saw no endpoint but we may be looping */ orig_exit_slot = hpts->p_cur_slot; } if ((wrap_loop_cnt < 2) && (hpts->p_lasttick != hpts->p_curtick)) { counter_u64_add(hpts_loops, 1); loop_cnt++; goto again; } no_run: cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv); /* * Set flag to tell that we are done for * any slot input that happens during * input. */ hpts->p_wheel_complete = 1; /* * Now did we spend too long running input and need to run more ticks? * Note that if wrap_loop_cnt < 2 then we should have the conditions * in the KASSERT's true. But if the wheel is behind i.e. wrap_loop_cnt * is greater than 2, then the condtion most likely are *not* true. * Also if we are called not from the callout, we don't run the wheel * multiple times so the slots may not align either. */ KASSERT(((hpts->p_prev_slot == hpts->p_cur_slot) || (wrap_loop_cnt >= 2) || (from_callout == 0)), ("H:%p p_prev_slot:%u not equal to p_cur_slot:%u", hpts, hpts->p_prev_slot, hpts->p_cur_slot)); KASSERT(((hpts->p_lasttick == hpts->p_curtick) || (wrap_loop_cnt >= 2) || (from_callout == 0)), ("H:%p p_lasttick:%u not equal to p_curtick:%u", hpts, hpts->p_lasttick, hpts->p_curtick)); if (from_callout && (hpts->p_lasttick != hpts->p_curtick)) { hpts->p_curtick = tcp_gethptstick(&tv); counter_u64_add(hpts_loops, 1); hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); goto again; } if (from_callout){ tcp_hpts_set_max_sleep(hpts, wrap_loop_cnt); } if (seen_endpoint) return(hpts_slots_diff(slot_pos_of_endpoint, orig_exit_slot)); else return (0); } void __tcp_set_hpts(struct tcpcb *tp, int32_t line) { struct tcp_hpts_entry *hpts; int failed; INP_WLOCK_ASSERT(tptoinpcb(tp)); hpts = tcp_hpts_lock(tp); if (tp->t_in_hpts == IHPTS_NONE && !(tp->t_flags2 & TF2_HPTS_CPU_SET)) { tp->t_hpts_cpu = hpts_cpuid(tp, &failed); if (failed == 0) tp->t_flags2 |= TF2_HPTS_CPU_SET; } mtx_unlock(&hpts->p_mtx); } +static struct tcp_hpts_entry * +tcp_choose_hpts_to_run(void) +{ + int i, oldest_idx, start, end; + uint32_t cts, time_since_ran, calc; + + cts = tcp_get_usecs(NULL); + time_since_ran = 0; + /* Default is all one group */ + start = 0; + end = tcp_pace.rp_num_hptss; + /* + * If we have more than one L3 group figure out which one + * this CPU is in. + */ + if (tcp_pace.grp_cnt > 1) { + for (i = 0; i < tcp_pace.grp_cnt; i++) { + if (CPU_ISSET(curcpu, &tcp_pace.grps[i]->cg_mask)) { + start = tcp_pace.grps[i]->cg_first; + end = (tcp_pace.grps[i]->cg_last + 1); + break; + } + } + } + oldest_idx = -1; + for (i = start; i < end; i++) { + if (TSTMP_GT(cts, cts_last_ran[i])) + calc = cts - cts_last_ran[i]; + else + calc = 0; + if (calc > time_since_ran) { + oldest_idx = i; + time_since_ran = calc; + } + } + if (oldest_idx >= 0) + return(tcp_pace.rp_ent[oldest_idx]); + else + return(tcp_pace.rp_ent[(curcpu % tcp_pace.rp_num_hptss)]); +} + static void -__tcp_run_hpts(struct tcp_hpts_entry *hpts) +__tcp_run_hpts(void) { + struct epoch_tracker et; + struct tcp_hpts_entry *hpts; int ticks_ran; + hpts = tcp_choose_hpts_to_run(); + if (hpts->p_hpts_active) { /* Already active */ return; } if (mtx_trylock(&hpts->p_mtx) == 0) { /* Someone else got the lock */ return; } + NET_EPOCH_ENTER(et); if (hpts->p_hpts_active) goto out_with_mtx; hpts->syscall_cnt++; counter_u64_add(hpts_direct_call, 1); hpts->p_hpts_active = 1; ticks_ran = tcp_hptsi(hpts, 0); /* We may want to adjust the sleep values here */ if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { if (ticks_ran > ticks_indicate_less_sleep) { struct timeval tv; sbintime_t sb; hpts->p_mysleep.tv_usec /= 2; if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) hpts->p_mysleep.tv_usec = dynamic_min_sleep; /* Reschedule with new to value */ tcp_hpts_set_max_sleep(hpts, 0); tv.tv_sec = 0; tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; /* Validate its in the right ranges */ if (tv.tv_usec < hpts->p_mysleep.tv_usec) { hpts->overidden_sleep = tv.tv_usec; tv.tv_usec = hpts->p_mysleep.tv_usec; } else if (tv.tv_usec > dynamic_max_sleep) { /* Lets not let sleep get above this value */ hpts->overidden_sleep = tv.tv_usec; tv.tv_usec = dynamic_max_sleep; } /* * In this mode the timer is a backstop to * all the userret/lro_flushes so we use * the dynamic value and set the on_min_sleep * flag so we will not be awoken. */ sb = tvtosbt(tv); /* Store off to make visible the actual sleep time */ hpts->sleeping = tv.tv_usec; callout_reset_sbt_on(&hpts->co, sb, 0, hpts_timeout_swi, hpts, hpts->p_cpu, (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); } else if (ticks_ran < ticks_indicate_more_sleep) { /* For the further sleep, don't reschedule hpts */ hpts->p_mysleep.tv_usec *= 2; if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) hpts->p_mysleep.tv_usec = dynamic_max_sleep; } hpts->p_on_min_sleep = 1; } hpts->p_hpts_active = 0; out_with_mtx: HPTS_MTX_ASSERT(hpts); mtx_unlock(&hpts->p_mtx); -} - -static struct tcp_hpts_entry * -tcp_choose_hpts_to_run(void) -{ - int i, oldest_idx, start, end; - uint32_t cts, time_since_ran, calc; - - cts = tcp_get_usecs(NULL); - time_since_ran = 0; - /* Default is all one group */ - start = 0; - end = tcp_pace.rp_num_hptss; - /* - * If we have more than one L3 group figure out which one - * this CPU is in. - */ - if (tcp_pace.grp_cnt > 1) { - for (i = 0; i < tcp_pace.grp_cnt; i++) { - if (CPU_ISSET(curcpu, &tcp_pace.grps[i]->cg_mask)) { - start = tcp_pace.grps[i]->cg_first; - end = (tcp_pace.grps[i]->cg_last + 1); - break; - } - } - } - oldest_idx = -1; - for (i = start; i < end; i++) { - if (TSTMP_GT(cts, cts_last_ran[i])) - calc = cts - cts_last_ran[i]; - else - calc = 0; - if (calc > time_since_ran) { - oldest_idx = i; - time_since_ran = calc; - } - } - if (oldest_idx >= 0) - return(tcp_pace.rp_ent[oldest_idx]); - else - return(tcp_pace.rp_ent[(curcpu % tcp_pace.rp_num_hptss)]); -} - - -void -tcp_run_hpts(void) -{ - struct tcp_hpts_entry *hpts; - struct epoch_tracker et; - - NET_EPOCH_ENTER(et); - hpts = tcp_choose_hpts_to_run(); - __tcp_run_hpts(hpts); NET_EPOCH_EXIT(et); } - static void tcp_hpts_thread(void *ctx) { struct tcp_hpts_entry *hpts; struct epoch_tracker et; struct timeval tv; sbintime_t sb; int ticks_ran; hpts = (struct tcp_hpts_entry *)ctx; mtx_lock(&hpts->p_mtx); if (hpts->p_direct_wake) { /* Signaled by input or output with low occupancy count. */ callout_stop(&hpts->co); counter_u64_add(hpts_direct_awakening, 1); } else { /* Timed out, the normal case. */ counter_u64_add(hpts_wake_timeout, 1); if (callout_pending(&hpts->co) || !callout_active(&hpts->co)) { mtx_unlock(&hpts->p_mtx); return; } } callout_deactivate(&hpts->co); hpts->p_hpts_wake_scheduled = 0; NET_EPOCH_ENTER(et); if (hpts->p_hpts_active) { /* * We are active already. This means that a syscall * trap or LRO is running in behalf of hpts. In that case * we need to double our timeout since there seems to be * enough activity in the system that we don't need to * run as often (if we were not directly woken). */ if (hpts->p_direct_wake == 0) { counter_u64_add(hpts_back_tosleep, 1); if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { hpts->p_mysleep.tv_usec *= 2; if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) hpts->p_mysleep.tv_usec = dynamic_max_sleep; tv.tv_usec = hpts->p_mysleep.tv_usec; hpts->p_on_min_sleep = 1; } else { /* * Here we have low count on the wheel, but * somehow we still collided with one of the * connections. Lets go back to sleep for a * min sleep time, but clear the flag so we * can be awoken by insert. */ hpts->p_on_min_sleep = 0; tv.tv_usec = tcp_min_hptsi_time; } } else { /* * Directly woken most likely to reset the * callout time. */ tv.tv_sec = 0; tv.tv_usec = hpts->p_mysleep.tv_usec; } goto back_to_sleep; } hpts->sleeping = 0; hpts->p_hpts_active = 1; ticks_ran = tcp_hptsi(hpts, 1); tv.tv_sec = 0; tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { if(hpts->p_direct_wake == 0) { /* * Only adjust sleep time if we were * called from the callout i.e. direct_wake == 0. */ if (ticks_ran < ticks_indicate_more_sleep) { hpts->p_mysleep.tv_usec *= 2; if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) hpts->p_mysleep.tv_usec = dynamic_max_sleep; } else if (ticks_ran > ticks_indicate_less_sleep) { hpts->p_mysleep.tv_usec /= 2; if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) hpts->p_mysleep.tv_usec = dynamic_min_sleep; } } if (tv.tv_usec < hpts->p_mysleep.tv_usec) { hpts->overidden_sleep = tv.tv_usec; tv.tv_usec = hpts->p_mysleep.tv_usec; } else if (tv.tv_usec > dynamic_max_sleep) { /* Lets not let sleep get above this value */ hpts->overidden_sleep = tv.tv_usec; tv.tv_usec = dynamic_max_sleep; } /* * In this mode the timer is a backstop to * all the userret/lro_flushes so we use * the dynamic value and set the on_min_sleep * flag so we will not be awoken. */ hpts->p_on_min_sleep = 1; } else if (hpts->p_on_queue_cnt == 0) { /* * No one on the wheel, please wake us up * if you insert on the wheel. */ hpts->p_on_min_sleep = 0; hpts->overidden_sleep = 0; } else { /* * We hit here when we have a low number of * clients on the wheel (our else clause). * We may need to go on min sleep, if we set * the flag we will not be awoken if someone * is inserted ahead of us. Clearing the flag * means we can be awoken. This is "old mode" * where the timer is what runs hpts mainly. */ if (tv.tv_usec < tcp_min_hptsi_time) { /* * Yes on min sleep, which means * we cannot be awoken. */ hpts->overidden_sleep = tv.tv_usec; tv.tv_usec = tcp_min_hptsi_time; hpts->p_on_min_sleep = 1; } else { /* Clear the min sleep flag */ hpts->overidden_sleep = 0; hpts->p_on_min_sleep = 0; } } HPTS_MTX_ASSERT(hpts); hpts->p_hpts_active = 0; back_to_sleep: hpts->p_direct_wake = 0; sb = tvtosbt(tv); /* Store off to make visible the actual sleep time */ hpts->sleeping = tv.tv_usec; callout_reset_sbt_on(&hpts->co, sb, 0, hpts_timeout_swi, hpts, hpts->p_cpu, (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); NET_EPOCH_EXIT(et); mtx_unlock(&hpts->p_mtx); } #undef timersub static int32_t hpts_count_level(struct cpu_group *cg) { int32_t count_l3, i; count_l3 = 0; if (cg->cg_level == CG_SHARE_L3) count_l3++; /* Walk all the children looking for L3 */ for (i = 0; i < cg->cg_children; i++) { count_l3 += hpts_count_level(&cg->cg_child[i]); } return (count_l3); } static void hpts_gather_grps(struct cpu_group **grps, int32_t *at, int32_t max, struct cpu_group *cg) { int32_t idx, i; idx = *at; if (cg->cg_level == CG_SHARE_L3) { grps[idx] = cg; idx++; if (idx == max) { *at = idx; return; } } *at = idx; /* Walk all the children looking for L3 */ for (i = 0; i < cg->cg_children; i++) { hpts_gather_grps(grps, at, max, &cg->cg_child[i]); } } static void tcp_init_hptsi(void *st) { struct cpu_group *cpu_top; int32_t error __diagused; int32_t i, j, bound = 0, created = 0; size_t sz, asz; struct timeval tv; sbintime_t sb; struct tcp_hpts_entry *hpts; struct pcpu *pc; char unit[16]; uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU; int count, domain; #ifdef SMP cpu_top = smp_topo(); #else cpu_top = NULL; #endif tcp_pace.rp_num_hptss = ncpus; hpts_hopelessly_behind = counter_u64_alloc(M_WAITOK); hpts_loops = counter_u64_alloc(M_WAITOK); back_tosleep = counter_u64_alloc(M_WAITOK); combined_wheel_wrap = counter_u64_alloc(M_WAITOK); wheel_wrap = counter_u64_alloc(M_WAITOK); hpts_wake_timeout = counter_u64_alloc(M_WAITOK); hpts_direct_awakening = counter_u64_alloc(M_WAITOK); hpts_back_tosleep = counter_u64_alloc(M_WAITOK); hpts_direct_call = counter_u64_alloc(M_WAITOK); cpu_uses_flowid = counter_u64_alloc(M_WAITOK); cpu_uses_random = counter_u64_alloc(M_WAITOK); sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *)); tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO); sz = (sizeof(uint32_t) * tcp_pace.rp_num_hptss); cts_last_ran = malloc(sz, M_TCPHPTS, M_WAITOK); tcp_pace.grp_cnt = 0; if (cpu_top == NULL) { tcp_pace.grp_cnt = 1; } else { /* Find out how many cache level 3 domains we have */ count = 0; tcp_pace.grp_cnt = hpts_count_level(cpu_top); if (tcp_pace.grp_cnt == 0) { tcp_pace.grp_cnt = 1; } sz = (tcp_pace.grp_cnt * sizeof(struct cpu_group *)); tcp_pace.grps = malloc(sz, M_TCPHPTS, M_WAITOK); /* Now populate the groups */ if (tcp_pace.grp_cnt == 1) { /* * All we need is the top level all cpu's are in * the same cache so when we use grp[0]->cg_mask * with the cg_first <-> cg_last it will include * all cpu's in it. The level here is probably * zero which is ok. */ tcp_pace.grps[0] = cpu_top; } else { /* * Here we must find all the level three cache domains * and setup our pointers to them. */ count = 0; hpts_gather_grps(tcp_pace.grps, &count, tcp_pace.grp_cnt, cpu_top); } } asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS; for (i = 0; i < tcp_pace.rp_num_hptss; i++) { tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry), M_TCPHPTS, M_WAITOK | M_ZERO); tcp_pace.rp_ent[i]->p_hptss = malloc(asz, M_TCPHPTS, M_WAITOK); hpts = tcp_pace.rp_ent[i]; /* * Init all the hpts structures that are not specifically * zero'd by the allocations. Also lets attach them to the * appropriate sysctl block as well. */ mtx_init(&hpts->p_mtx, "tcp_hpts_lck", "hpts", MTX_DEF | MTX_DUPOK); for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) { TAILQ_INIT(&hpts->p_hptss[j].head); hpts->p_hptss[j].count = 0; hpts->p_hptss[j].gencnt = 0; } sysctl_ctx_init(&hpts->hpts_ctx); sprintf(unit, "%d", i); hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx, SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts), OID_AUTO, unit, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, ""); SYSCTL_ADD_INT(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "out_qcnt", CTLFLAG_RD, &hpts->p_on_queue_cnt, 0, "Count TCB's awaiting output processing"); SYSCTL_ADD_U16(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "active", CTLFLAG_RD, &hpts->p_hpts_active, 0, "Is the hpts active"); SYSCTL_ADD_UINT(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "curslot", CTLFLAG_RD, &hpts->p_cur_slot, 0, "What the current running pacers goal"); SYSCTL_ADD_UINT(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "runtick", CTLFLAG_RD, &hpts->p_runningslot, 0, "What the running pacers current slot is"); SYSCTL_ADD_UINT(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "curtick", CTLFLAG_RD, &hpts->p_curtick, 0, "What the running pacers last tick mapped to the wheel was"); SYSCTL_ADD_UINT(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "lastran", CTLFLAG_RD, &cts_last_ran[i], 0, "The last usec tick that this hpts ran"); SYSCTL_ADD_LONG(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "cur_min_sleep", CTLFLAG_RD, &hpts->p_mysleep.tv_usec, "What the running pacers is using for p_mysleep.tv_usec"); SYSCTL_ADD_U64(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "now_sleeping", CTLFLAG_RD, &hpts->sleeping, 0, "What the running pacers is actually sleeping for"); SYSCTL_ADD_U64(&hpts->hpts_ctx, SYSCTL_CHILDREN(hpts->hpts_root), OID_AUTO, "syscall_cnt", CTLFLAG_RD, &hpts->syscall_cnt, 0, "How many times we had syscalls on this hpts"); hpts->p_hpts_sleep_time = hpts_sleep_max; hpts->p_num = i; hpts->p_curtick = tcp_gethptstick(&tv); cts_last_ran[i] = tcp_tv_to_usectick(&tv); hpts->p_prev_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); hpts->p_cpu = 0xffff; hpts->p_nxt_slot = hpts_slot(hpts->p_cur_slot, 1); callout_init(&hpts->co, 1); } /* Don't try to bind to NUMA domains if we don't have any */ if (vm_ndomains == 1 && tcp_bind_threads == 2) tcp_bind_threads = 0; /* * Now lets start ithreads to handle the hptss. */ for (i = 0; i < tcp_pace.rp_num_hptss; i++) { hpts = tcp_pace.rp_ent[i]; hpts->p_cpu = i; error = swi_add(&hpts->ie, "hpts", tcp_hpts_thread, (void *)hpts, SWI_NET, INTR_MPSAFE, &hpts->ie_cookie); KASSERT(error == 0, ("Can't add hpts:%p i:%d err:%d", hpts, i, error)); created++; hpts->p_mysleep.tv_sec = 0; hpts->p_mysleep.tv_usec = tcp_min_hptsi_time; if (tcp_bind_threads == 1) { if (intr_event_bind(hpts->ie, i) == 0) bound++; } else if (tcp_bind_threads == 2) { /* Find the group for this CPU (i) and bind into it */ for (j = 0; j < tcp_pace.grp_cnt; j++) { if (CPU_ISSET(i, &tcp_pace.grps[j]->cg_mask)) { if (intr_event_bind_ithread_cpuset(hpts->ie, &tcp_pace.grps[j]->cg_mask) == 0) { bound++; pc = pcpu_find(i); domain = pc->pc_domain; count = hpts_domains[domain].count; hpts_domains[domain].cpu[count] = i; hpts_domains[domain].count++; break; } } } } tv.tv_sec = 0; tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; hpts->sleeping = tv.tv_usec; sb = tvtosbt(tv); callout_reset_sbt_on(&hpts->co, sb, 0, hpts_timeout_swi, hpts, hpts->p_cpu, (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); } /* * If we somehow have an empty domain, fall back to choosing * among all htps threads. */ for (i = 0; i < vm_ndomains; i++) { if (hpts_domains[i].count == 0) { tcp_bind_threads = 0; break; } } + tcp_hpts_softclock = __tcp_run_hpts; + tcp_lro_hpts_init(); printf("TCP Hpts created %d swi interrupt threads and bound %d to %s\n", created, bound, tcp_bind_threads == 2 ? "NUMA domains" : "cpus"); #ifdef INVARIANTS printf("HPTS is in INVARIANT mode!!\n"); #endif } SYSINIT(tcphptsi, SI_SUB_SOFTINTR, SI_ORDER_ANY, tcp_init_hptsi, NULL); MODULE_VERSION(tcphpts, 1); diff --git a/sys/netinet/tcp_hpts.h b/sys/netinet/tcp_hpts.h index 514ab84227b5..8ca21daf60de 100644 --- a/sys/netinet/tcp_hpts.h +++ b/sys/netinet/tcp_hpts.h @@ -1,222 +1,223 @@ /*- * Copyright (c) 2016-2018 Netflix, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __tcp_hpts_h__ #define __tcp_hpts_h__ /* Number of useconds in a hpts tick */ #define HPTS_TICKS_PER_SLOT 10 #define HPTS_MS_TO_SLOTS(x) ((x * 100) + 1) #define HPTS_USEC_TO_SLOTS(x) ((x+9) /10) #define HPTS_USEC_IN_SEC 1000000 #define HPTS_MSEC_IN_SEC 1000 #define HPTS_USEC_IN_MSEC 1000 struct hpts_diag { uint32_t p_hpts_active; /* bbr->flex7 x */ uint32_t p_nxt_slot; /* bbr->flex1 x */ uint32_t p_cur_slot; /* bbr->flex2 x */ uint32_t p_prev_slot; /* bbr->delivered */ uint32_t p_runningslot; /* bbr->inflight */ uint32_t slot_req; /* bbr->flex3 x */ uint32_t inp_hptsslot; /* bbr->flex4 x */ uint32_t slot_remaining; /* bbr->flex5 x */ uint32_t have_slept; /* bbr->epoch x */ uint32_t hpts_sleep_time; /* bbr->applimited x */ uint32_t yet_to_sleep; /* bbr->lt_epoch x */ uint32_t need_new_to; /* bbr->flex6 x */ uint32_t wheel_slot; /* bbr->bw_inuse x */ uint32_t maxslots; /* bbr->delRate x */ uint32_t wheel_cts; /* bbr->rttProp x */ int32_t co_ret; /* bbr->pkts_out x */ uint32_t p_curtick; /* upper bbr->cur_del_rate */ uint32_t p_lasttick; /* lower bbr->cur_del_rate */ uint8_t p_on_min_sleep; /* bbr->flex8 x */ }; /* Magic flags to tell whats cooking on the pacing wheel */ #define PACE_TMR_DELACK 0x01 /* Delayed ack timer running */ #define PACE_TMR_RACK 0x02 /* RACK timer running */ #define PACE_TMR_TLP 0x04 /* TLP timer running */ #define PACE_TMR_RXT 0x08 /* Retransmit timer running */ #define PACE_TMR_PERSIT 0x10 /* Persists timer running */ #define PACE_TMR_KEEP 0x20 /* Keep alive timer running */ #define PACE_PKT_OUTPUT 0x40 /* Output Packets being paced */ #define PACE_TMR_MASK (PACE_TMR_KEEP|PACE_TMR_PERSIT|PACE_TMR_RXT|PACE_TMR_TLP|PACE_TMR_RACK|PACE_TMR_DELACK) #define DEFAULT_CONNECTION_THESHOLD 100 /* * When using the hpts, a TCP stack must make sure * that once a INP_DROPPED flag is applied to a INP * that it does not expect tcp_output() to ever be * called by the hpts. The hpts will *not* call * any output (or input) functions on a TCB that * is in the DROPPED state. * * This implies final ACK's and RST's that might * be sent when a TCB is still around must be * sent from a routine like tcp_respond(). */ #define LOWEST_SLEEP_ALLOWED 50 #define DEFAULT_MIN_SLEEP 250 /* How many usec's is default for hpts sleep * this determines min granularity of the * hpts. If 1, granularity is 10useconds at * the cost of more CPU (context switching). * Note do not set this to 0. */ #define DYNAMIC_MIN_SLEEP DEFAULT_MIN_SLEEP #define DYNAMIC_MAX_SLEEP 5000 /* 5ms */ /* Thresholds for raising/lowering sleep */ #define TICKS_INDICATE_MORE_SLEEP 100 /* This would be 1ms */ #define TICKS_INDICATE_LESS_SLEEP 1000 /* This would indicate 10ms */ /** * * Dynamic adjustment of sleeping times is done in "new" mode * where we are depending on syscall returns and lro returns * to push hpts forward mainly and the timer is only a backstop. * * When we are in the "new" mode i.e. conn_cnt > conn_cnt_thresh * then we do a dynamic adjustment on the time we sleep. * Our threshold is if the lateness of the first client served (in ticks) is * greater than or equal too ticks_indicate_more_sleep (10ms * or 10000 ticks). If we were that late, the actual sleep time * is adjusted down by 50%. If the ticks_ran is less than * ticks_indicate_more_sleep (100 ticks or 1000usecs). * */ #ifdef _KERNEL void tcp_hpts_init(struct tcpcb *); void tcp_hpts_remove(struct tcpcb *); static inline bool tcp_in_hpts(struct tcpcb *tp) { return (tp->t_in_hpts == IHPTS_ONQUEUE); } /* * To insert a TCB on the hpts you *must* be holding the * INP_WLOCK(). The hpts insert code will then acqurire * the hpts's lock and insert the TCB on the requested * slot possibly waking up the hpts if you are requesting * a time earlier than what the hpts is sleeping to (if * the hpts is sleeping). You may check the inp->inp_in_hpts * flag without the hpts lock. The hpts is the only one * that will clear this flag holding only the hpts lock. This * means that in your tcp_output() routine when you test for * it to be 1 (so you wont call output) it may be transitioning * to 0 (by the hpts). That will be fine since that will just * mean an extra call to tcp_output that most likely will find * the call you executed (when the mis-match occurred) will have * put the TCB back on the hpts and it will return. If your * call did not add it back to the hpts then you will either * over-send or the cwnd will block you from sending more. * * Note you should also be holding the INP_WLOCK() when you * call the remove from the hpts as well. Thoug usually * you are either doing this from a timer, where you need * that INP_WLOCK() or from destroying your TCB where again * you should already have the INP_WLOCK(). */ uint32_t tcp_hpts_insert_diag(struct tcpcb *tp, uint32_t slot, int32_t line, struct hpts_diag *diag); #define tcp_hpts_insert(inp, slot) \ tcp_hpts_insert_diag((inp), (slot), __LINE__, NULL) void __tcp_set_hpts(struct tcpcb *tp, int32_t line); #define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__) void tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason); -void tcp_run_hpts(void); +extern void (*tcp_hpts_softclock)(void); +void tcp_lro_hpts_init(void); extern int32_t tcp_min_hptsi_time; #endif /* _KERNEL */ /* * The following functions should also be available * to userspace as well. */ static __inline uint32_t tcp_tv_to_hptstick(const struct timeval *sv) { return ((sv->tv_sec * 100000) + (sv->tv_usec / HPTS_TICKS_PER_SLOT)); } static __inline uint32_t tcp_tv_to_usectick(const struct timeval *sv) { return ((uint32_t) ((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec)); } static __inline uint32_t tcp_tv_to_mssectick(const struct timeval *sv) { return ((uint32_t) ((sv->tv_sec * HPTS_MSEC_IN_SEC) + (sv->tv_usec/HPTS_USEC_IN_MSEC))); } static __inline uint64_t tcp_tv_to_lusectick(const struct timeval *sv) { return ((uint64_t)((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec)); } #ifdef _KERNEL extern int32_t tcp_min_hptsi_time; static inline int32_t get_hpts_min_sleep_time(void) { return (tcp_min_hptsi_time + HPTS_TICKS_PER_SLOT); } static __inline uint32_t tcp_gethptstick(struct timeval *sv) { struct timeval tv; if (sv == NULL) sv = &tv; microuptime(sv); return (tcp_tv_to_hptstick(sv)); } static __inline uint32_t tcp_get_usecs(struct timeval *tv) { struct timeval tvd; if (tv == NULL) tv = &tvd; microuptime(tv); return (tcp_tv_to_usectick(tv)); } #endif /* _KERNEL */ #endif /* __tcp_hpts_h__ */ diff --git a/sys/netinet/tcp_lro.c b/sys/netinet/tcp_lro.c index 6cf0411b5f65..255e543ae21d 100644 --- a/sys/netinet/tcp_lro.c +++ b/sys/netinet/tcp_lro.c @@ -1,1515 +1,1507 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2007, Myricom Inc. * Copyright (c) 2008, Intel Corporation. * Copyright (c) 2012 The FreeBSD Foundation * Copyright (c) 2016-2021 Mellanox Technologies. * All rights reserved. * * Portions of this software were developed by Bjoern Zeeb * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_inet.h" #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures"); static void tcp_lro_rx_done(struct lro_ctrl *lc); static int tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, bool use_hash); SYSCTL_NODE(_net_inet_tcp, OID_AUTO, lro, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "TCP LRO"); long tcplro_stacks_wanting_mbufq; +int (*tcp_lro_flush_tcphpts)(struct lro_ctrl *lc, struct lro_entry *le); +void (*tcp_hpts_softclock)(void); + counter_u64_t tcp_inp_lro_direct_queue; counter_u64_t tcp_inp_lro_wokeup_queue; counter_u64_t tcp_inp_lro_compressed; counter_u64_t tcp_inp_lro_locks_taken; counter_u64_t tcp_extra_mbuf; counter_u64_t tcp_would_have_but; counter_u64_t tcp_comp_total; counter_u64_t tcp_uncomp_total; counter_u64_t tcp_bad_csums; static unsigned tcp_lro_entries = TCP_LRO_ENTRIES; SYSCTL_UINT(_net_inet_tcp_lro, OID_AUTO, entries, CTLFLAG_RDTUN | CTLFLAG_MPSAFE, &tcp_lro_entries, 0, "default number of LRO entries"); static uint32_t tcp_lro_cpu_set_thresh = TCP_LRO_CPU_DECLARATION_THRESH; SYSCTL_UINT(_net_inet_tcp_lro, OID_AUTO, lro_cpu_threshold, CTLFLAG_RDTUN | CTLFLAG_MPSAFE, &tcp_lro_cpu_set_thresh, 0, "Number of interrupts in a row on the same CPU that will make us declare an 'affinity' cpu?"); static uint32_t tcp_less_accurate_lro_ts = 0; SYSCTL_UINT(_net_inet_tcp_lro, OID_AUTO, lro_less_accurate, CTLFLAG_MPSAFE, &tcp_less_accurate_lro_ts, 0, "Do we trade off efficency by doing less timestamp operations for time accuracy?"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, fullqueue, CTLFLAG_RD, &tcp_inp_lro_direct_queue, "Number of lro's fully queued to transport"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, wokeup, CTLFLAG_RD, &tcp_inp_lro_wokeup_queue, "Number of lro's where we woke up transport via hpts"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, compressed, CTLFLAG_RD, &tcp_inp_lro_compressed, "Number of lro's compressed and sent to transport"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, lockcnt, CTLFLAG_RD, &tcp_inp_lro_locks_taken, "Number of lro's inp_wlocks taken"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, extra_mbuf, CTLFLAG_RD, &tcp_extra_mbuf, "Number of times we had an extra compressed ack dropped into the tp"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, would_have_but, CTLFLAG_RD, &tcp_would_have_but, "Number of times we would have had an extra compressed, but mget failed"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, with_m_ackcmp, CTLFLAG_RD, &tcp_comp_total, "Number of mbufs queued with M_ACKCMP flags set"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, without_m_ackcmp, CTLFLAG_RD, &tcp_uncomp_total, "Number of mbufs queued without M_ACKCMP"); SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, lro_badcsum, CTLFLAG_RD, &tcp_bad_csums, "Number of packets that the common code saw with bad csums"); void tcp_lro_reg_mbufq(void) { atomic_fetchadd_long(&tcplro_stacks_wanting_mbufq, 1); } void tcp_lro_dereg_mbufq(void) { atomic_fetchadd_long(&tcplro_stacks_wanting_mbufq, -1); } static __inline void tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_head *bucket, struct lro_entry *le) { LIST_INSERT_HEAD(&lc->lro_active, le, next); LIST_INSERT_HEAD(bucket, le, hash_next); } static __inline void tcp_lro_active_remove(struct lro_entry *le) { LIST_REMOVE(le, next); /* active list */ LIST_REMOVE(le, hash_next); /* hash bucket */ } int tcp_lro_init(struct lro_ctrl *lc) { return (tcp_lro_init_args(lc, NULL, tcp_lro_entries, 0)); } int tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp, unsigned lro_entries, unsigned lro_mbufs) { struct lro_entry *le; size_t size; unsigned i, elements; lc->lro_bad_csum = 0; lc->lro_queued = 0; lc->lro_flushed = 0; lc->lro_mbuf_count = 0; lc->lro_mbuf_max = lro_mbufs; lc->lro_cnt = lro_entries; lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX; lc->lro_length_lim = TCP_LRO_LENGTH_MAX; lc->ifp = ifp; LIST_INIT(&lc->lro_free); LIST_INIT(&lc->lro_active); /* create hash table to accelerate entry lookup */ if (lro_entries > lro_mbufs) elements = lro_entries; else elements = lro_mbufs; lc->lro_hash = phashinit_flags(elements, M_LRO, &lc->lro_hashsz, HASH_NOWAIT); if (lc->lro_hash == NULL) { memset(lc, 0, sizeof(*lc)); return (ENOMEM); } /* compute size to allocate */ size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) + (lro_entries * sizeof(*le)); lc->lro_mbuf_data = (struct lro_mbuf_sort *) malloc(size, M_LRO, M_NOWAIT | M_ZERO); /* check for out of memory */ if (lc->lro_mbuf_data == NULL) { free(lc->lro_hash, M_LRO); memset(lc, 0, sizeof(*lc)); return (ENOMEM); } /* compute offset for LRO entries */ le = (struct lro_entry *) (lc->lro_mbuf_data + lro_mbufs); /* setup linked list */ for (i = 0; i != lro_entries; i++) LIST_INSERT_HEAD(&lc->lro_free, le + i, next); return (0); } struct vxlan_header { uint32_t vxlh_flags; uint32_t vxlh_vni; }; static inline void * tcp_lro_low_level_parser(void *ptr, struct lro_parser *parser, bool update_data, bool is_vxlan, int mlen) { const struct ether_vlan_header *eh; void *old; uint16_t eth_type; if (update_data) memset(parser, 0, sizeof(*parser)); old = ptr; if (is_vxlan) { const struct vxlan_header *vxh; vxh = ptr; ptr = (uint8_t *)ptr + sizeof(*vxh); if (update_data) { parser->data.vxlan_vni = vxh->vxlh_vni & htonl(0xffffff00); } } eh = ptr; if (__predict_false(eh->evl_encap_proto == htons(ETHERTYPE_VLAN))) { eth_type = eh->evl_proto; if (update_data) { /* strip priority and keep VLAN ID only */ parser->data.vlan_id = eh->evl_tag & htons(EVL_VLID_MASK); } /* advance to next header */ ptr = (uint8_t *)ptr + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; mlen -= (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); } else { eth_type = eh->evl_encap_proto; /* advance to next header */ mlen -= ETHER_HDR_LEN; ptr = (uint8_t *)ptr + ETHER_HDR_LEN; } if (__predict_false(mlen <= 0)) return (NULL); switch (eth_type) { #ifdef INET case htons(ETHERTYPE_IP): parser->ip4 = ptr; if (__predict_false(mlen < sizeof(struct ip))) return (NULL); /* Ensure there are no IPv4 options. */ if ((parser->ip4->ip_hl << 2) != sizeof (*parser->ip4)) break; /* .. and the packet is not fragmented. */ if (parser->ip4->ip_off & htons(IP_MF|IP_OFFMASK)) break; /* .. and the packet has valid src/dst addrs */ if (__predict_false(parser->ip4->ip_src.s_addr == INADDR_ANY || parser->ip4->ip_dst.s_addr == INADDR_ANY)) break; ptr = (uint8_t *)ptr + (parser->ip4->ip_hl << 2); mlen -= sizeof(struct ip); if (update_data) { parser->data.s_addr.v4 = parser->ip4->ip_src; parser->data.d_addr.v4 = parser->ip4->ip_dst; } switch (parser->ip4->ip_p) { case IPPROTO_UDP: if (__predict_false(mlen < sizeof(struct udphdr))) return (NULL); parser->udp = ptr; if (update_data) { parser->data.lro_type = LRO_TYPE_IPV4_UDP; parser->data.s_port = parser->udp->uh_sport; parser->data.d_port = parser->udp->uh_dport; } else { MPASS(parser->data.lro_type == LRO_TYPE_IPV4_UDP); } ptr = ((uint8_t *)ptr + sizeof(*parser->udp)); parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; return (ptr); case IPPROTO_TCP: parser->tcp = ptr; if (__predict_false(mlen < sizeof(struct tcphdr))) return (NULL); if (update_data) { parser->data.lro_type = LRO_TYPE_IPV4_TCP; parser->data.s_port = parser->tcp->th_sport; parser->data.d_port = parser->tcp->th_dport; } else { MPASS(parser->data.lro_type == LRO_TYPE_IPV4_TCP); } if (__predict_false(mlen < (parser->tcp->th_off << 2))) return (NULL); ptr = (uint8_t *)ptr + (parser->tcp->th_off << 2); parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; return (ptr); default: break; } break; #endif #ifdef INET6 case htons(ETHERTYPE_IPV6): parser->ip6 = ptr; if (__predict_false(mlen < sizeof(struct ip6_hdr))) return (NULL); /* Ensure the packet has valid src/dst addrs */ if (__predict_false(IN6_IS_ADDR_UNSPECIFIED(&parser->ip6->ip6_src) || IN6_IS_ADDR_UNSPECIFIED(&parser->ip6->ip6_dst))) return (NULL); ptr = (uint8_t *)ptr + sizeof(*parser->ip6); if (update_data) { parser->data.s_addr.v6 = parser->ip6->ip6_src; parser->data.d_addr.v6 = parser->ip6->ip6_dst; } mlen -= sizeof(struct ip6_hdr); switch (parser->ip6->ip6_nxt) { case IPPROTO_UDP: if (__predict_false(mlen < sizeof(struct udphdr))) return (NULL); parser->udp = ptr; if (update_data) { parser->data.lro_type = LRO_TYPE_IPV6_UDP; parser->data.s_port = parser->udp->uh_sport; parser->data.d_port = parser->udp->uh_dport; } else { MPASS(parser->data.lro_type == LRO_TYPE_IPV6_UDP); } ptr = (uint8_t *)ptr + sizeof(*parser->udp); parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; return (ptr); case IPPROTO_TCP: if (__predict_false(mlen < sizeof(struct tcphdr))) return (NULL); parser->tcp = ptr; if (update_data) { parser->data.lro_type = LRO_TYPE_IPV6_TCP; parser->data.s_port = parser->tcp->th_sport; parser->data.d_port = parser->tcp->th_dport; } else { MPASS(parser->data.lro_type == LRO_TYPE_IPV6_TCP); } if (__predict_false(mlen < (parser->tcp->th_off << 2))) return (NULL); ptr = (uint8_t *)ptr + (parser->tcp->th_off << 2); parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; return (ptr); default: break; } break; #endif default: break; } /* Invalid packet - cannot parse */ return (NULL); } static const int vxlan_csum = CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID; static inline struct lro_parser * tcp_lro_parser(struct mbuf *m, struct lro_parser *po, struct lro_parser *pi, bool update_data) { void *data_ptr; /* Try to parse outer headers first. */ data_ptr = tcp_lro_low_level_parser(m->m_data, po, update_data, false, m->m_len); if (data_ptr == NULL || po->total_hdr_len > m->m_len) return (NULL); if (update_data) { /* Store VLAN ID, if any. */ if (__predict_false(m->m_flags & M_VLANTAG)) { po->data.vlan_id = htons(m->m_pkthdr.ether_vtag) & htons(EVL_VLID_MASK); } /* Store decrypted flag, if any. */ if (__predict_false((m->m_pkthdr.csum_flags & CSUM_TLS_MASK) == CSUM_TLS_DECRYPTED)) po->data.lro_flags |= LRO_FLAG_DECRYPTED; } switch (po->data.lro_type) { case LRO_TYPE_IPV4_UDP: case LRO_TYPE_IPV6_UDP: /* Check for VXLAN headers. */ if ((m->m_pkthdr.csum_flags & vxlan_csum) != vxlan_csum) break; /* Try to parse inner headers. */ data_ptr = tcp_lro_low_level_parser(data_ptr, pi, update_data, true, (m->m_len - ((caddr_t)data_ptr - m->m_data))); if (data_ptr == NULL || (pi->total_hdr_len + po->total_hdr_len) > m->m_len) break; /* Verify supported header types. */ switch (pi->data.lro_type) { case LRO_TYPE_IPV4_TCP: case LRO_TYPE_IPV6_TCP: return (pi); default: break; } break; case LRO_TYPE_IPV4_TCP: case LRO_TYPE_IPV6_TCP: if (update_data) memset(pi, 0, sizeof(*pi)); return (po); default: break; } return (NULL); } static inline int tcp_lro_trim_mbuf_chain(struct mbuf *m, const struct lro_parser *po) { int len; switch (po->data.lro_type) { #ifdef INET case LRO_TYPE_IPV4_TCP: len = ((uint8_t *)po->ip4 - (uint8_t *)m->m_data) + ntohs(po->ip4->ip_len); break; #endif #ifdef INET6 case LRO_TYPE_IPV6_TCP: len = ((uint8_t *)po->ip6 - (uint8_t *)m->m_data) + ntohs(po->ip6->ip6_plen) + sizeof(*po->ip6); break; #endif default: return (TCP_LRO_CANNOT); } /* * If the frame is padded beyond the end of the IP packet, * then trim the extra bytes off: */ if (__predict_true(m->m_pkthdr.len == len)) { return (0); } else if (m->m_pkthdr.len > len) { m_adj(m, len - m->m_pkthdr.len); return (0); } return (TCP_LRO_CANNOT); } static void lro_free_mbuf_chain(struct mbuf *m) { struct mbuf *save; while (m) { save = m->m_nextpkt; m->m_nextpkt = NULL; m_freem(m); m = save; } } void tcp_lro_free(struct lro_ctrl *lc) { struct lro_entry *le; unsigned x; /* reset LRO free list */ LIST_INIT(&lc->lro_free); /* free active mbufs, if any */ while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { tcp_lro_active_remove(le); lro_free_mbuf_chain(le->m_head); } /* free hash table */ free(lc->lro_hash, M_LRO); lc->lro_hash = NULL; lc->lro_hashsz = 0; /* free mbuf array, if any */ for (x = 0; x != lc->lro_mbuf_count; x++) m_freem(lc->lro_mbuf_data[x].mb); lc->lro_mbuf_count = 0; /* free allocated memory, if any */ free(lc->lro_mbuf_data, M_LRO); lc->lro_mbuf_data = NULL; } static uint16_t tcp_lro_rx_csum_tcphdr(const struct tcphdr *th) { const uint16_t *ptr; uint32_t csum; uint16_t len; csum = -th->th_sum; /* exclude checksum field */ len = th->th_off; ptr = (const uint16_t *)th; while (len--) { csum += *ptr; ptr++; csum += *ptr; ptr++; } while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); return (csum); } static uint16_t tcp_lro_rx_csum_data(const struct lro_parser *pa, uint16_t tcp_csum) { uint32_t c; uint16_t cs; c = tcp_csum; switch (pa->data.lro_type) { #ifdef INET6 case LRO_TYPE_IPV6_TCP: /* Compute full pseudo IPv6 header checksum. */ cs = in6_cksum_pseudo(pa->ip6, ntohs(pa->ip6->ip6_plen), pa->ip6->ip6_nxt, 0); break; #endif #ifdef INET case LRO_TYPE_IPV4_TCP: /* Compute full pseudo IPv4 header checsum. */ cs = in_addword(ntohs(pa->ip4->ip_len) - sizeof(*pa->ip4), IPPROTO_TCP); cs = in_pseudo(pa->ip4->ip_src.s_addr, pa->ip4->ip_dst.s_addr, htons(cs)); break; #endif default: cs = 0; /* Keep compiler happy. */ break; } /* Complement checksum. */ cs = ~cs; c += cs; /* Remove TCP header checksum. */ cs = ~tcp_lro_rx_csum_tcphdr(pa->tcp); c += cs; /* Compute checksum remainder. */ while (c > 0xffff) c = (c >> 16) + (c & 0xffff); return (c); } static void tcp_lro_rx_done(struct lro_ctrl *lc) { struct lro_entry *le; while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { tcp_lro_active_remove(le); tcp_lro_flush(lc, le); } } static void tcp_lro_flush_active(struct lro_ctrl *lc) { struct lro_entry *le; /* * Walk through the list of le entries, and * any one that does have packets flush. This * is called because we have an inbound packet * (e.g. SYN) that has to have all others flushed * in front of it. Note we have to do the remove * because tcp_lro_flush() assumes that the entry * is being freed. This is ok it will just get * reallocated again like it was new. */ LIST_FOREACH(le, &lc->lro_active, next) { if (le->m_head != NULL) { tcp_lro_active_remove(le); tcp_lro_flush(lc, le); } } } void tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout) { struct lro_entry *le, *le_tmp; uint64_t now, tov; struct bintime bt; NET_EPOCH_ASSERT(); if (LIST_EMPTY(&lc->lro_active)) return; /* get timeout time and current time in ns */ binuptime(&bt); now = bintime2ns(&bt); tov = ((timeout->tv_sec * 1000000000) + (timeout->tv_usec * 1000)); LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) { if (now >= (bintime2ns(&le->alloc_time) + tov)) { tcp_lro_active_remove(le); tcp_lro_flush(lc, le); } } } #ifdef INET static int tcp_lro_rx_ipv4(struct lro_ctrl *lc, struct mbuf *m, struct ip *ip4) { uint16_t csum; /* Legacy IP has a header checksum that needs to be correct. */ if (m->m_pkthdr.csum_flags & CSUM_IP_CHECKED) { if (__predict_false((m->m_pkthdr.csum_flags & CSUM_IP_VALID) == 0)) { lc->lro_bad_csum++; return (TCP_LRO_CANNOT); } } else { csum = in_cksum_hdr(ip4); if (__predict_false(csum != 0)) { lc->lro_bad_csum++; return (TCP_LRO_CANNOT); } } return (0); } #endif static inline void tcp_lro_assign_and_checksum_16(uint16_t *ptr, uint16_t value, uint16_t *psum) { uint32_t csum; csum = 0xffff - *ptr + value; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); *ptr = value; *psum = csum; } static uint16_t tcp_lro_update_checksum(const struct lro_parser *pa, const struct lro_entry *le, uint16_t payload_len, uint16_t delta_sum) { uint32_t csum; uint16_t tlen; uint16_t temp[5] = {}; switch (pa->data.lro_type) { case LRO_TYPE_IPV4_TCP: /* Compute new IPv4 length. */ tlen = (pa->ip4->ip_hl << 2) + (pa->tcp->th_off << 2) + payload_len; tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]); /* Subtract delta from current IPv4 checksum. */ csum = pa->ip4->ip_sum + 0xffff - temp[0]; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); tcp_lro_assign_and_checksum_16(&pa->ip4->ip_sum, csum, &temp[1]); goto update_tcp_header; case LRO_TYPE_IPV6_TCP: /* Compute new IPv6 length. */ tlen = (pa->tcp->th_off << 2) + payload_len; tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]); goto update_tcp_header; case LRO_TYPE_IPV4_UDP: /* Compute new IPv4 length. */ tlen = (pa->ip4->ip_hl << 2) + sizeof(*pa->udp) + payload_len; tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]); /* Subtract delta from current IPv4 checksum. */ csum = pa->ip4->ip_sum + 0xffff - temp[0]; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); tcp_lro_assign_and_checksum_16(&pa->ip4->ip_sum, csum, &temp[1]); goto update_udp_header; case LRO_TYPE_IPV6_UDP: /* Compute new IPv6 length. */ tlen = sizeof(*pa->udp) + payload_len; tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]); goto update_udp_header; default: return (0); } update_tcp_header: /* Compute current TCP header checksum. */ temp[2] = tcp_lro_rx_csum_tcphdr(pa->tcp); /* Incorporate the latest ACK into the TCP header. */ pa->tcp->th_ack = le->ack_seq; pa->tcp->th_win = le->window; /* Incorporate latest timestamp into the TCP header. */ if (le->timestamp != 0) { uint32_t *ts_ptr; ts_ptr = (uint32_t *)(pa->tcp + 1); ts_ptr[1] = htonl(le->tsval); ts_ptr[2] = le->tsecr; } /* Compute new TCP header checksum. */ temp[3] = tcp_lro_rx_csum_tcphdr(pa->tcp); /* Compute new TCP checksum. */ csum = pa->tcp->th_sum + 0xffff - delta_sum + 0xffff - temp[0] + 0xffff - temp[3] + temp[2]; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); /* Assign new TCP checksum. */ tcp_lro_assign_and_checksum_16(&pa->tcp->th_sum, csum, &temp[4]); /* Compute all modififications affecting next checksum. */ csum = temp[0] + temp[1] + 0xffff - temp[2] + temp[3] + temp[4] + delta_sum; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); /* Return delta checksum to next stage, if any. */ return (csum); update_udp_header: tlen = sizeof(*pa->udp) + payload_len; /* Assign new UDP length and compute checksum delta. */ tcp_lro_assign_and_checksum_16(&pa->udp->uh_ulen, htons(tlen), &temp[2]); /* Check if there is a UDP checksum. */ if (__predict_false(pa->udp->uh_sum != 0)) { /* Compute new UDP checksum. */ csum = pa->udp->uh_sum + 0xffff - delta_sum + 0xffff - temp[0] + 0xffff - temp[2]; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); /* Assign new UDP checksum. */ tcp_lro_assign_and_checksum_16(&pa->udp->uh_sum, csum, &temp[3]); } /* Compute all modififications affecting next checksum. */ csum = temp[0] + temp[1] + temp[2] + temp[3] + delta_sum; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); /* Return delta checksum to next stage, if any. */ return (csum); } static void tcp_flush_out_entry(struct lro_ctrl *lc, struct lro_entry *le) { /* Check if we need to recompute any checksums. */ if (le->needs_merge) { uint16_t csum; switch (le->inner.data.lro_type) { case LRO_TYPE_IPV4_TCP: csum = tcp_lro_update_checksum(&le->inner, le, le->m_head->m_pkthdr.lro_tcp_d_len, le->m_head->m_pkthdr.lro_tcp_d_csum); csum = tcp_lro_update_checksum(&le->outer, NULL, le->m_head->m_pkthdr.lro_tcp_d_len + le->inner.total_hdr_len, csum); le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID; le->m_head->m_pkthdr.csum_data = 0xffff; if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; break; case LRO_TYPE_IPV6_TCP: csum = tcp_lro_update_checksum(&le->inner, le, le->m_head->m_pkthdr.lro_tcp_d_len, le->m_head->m_pkthdr.lro_tcp_d_csum); csum = tcp_lro_update_checksum(&le->outer, NULL, le->m_head->m_pkthdr.lro_tcp_d_len + le->inner.total_hdr_len, csum); le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; le->m_head->m_pkthdr.csum_data = 0xffff; if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; break; case LRO_TYPE_NONE: switch (le->outer.data.lro_type) { case LRO_TYPE_IPV4_TCP: csum = tcp_lro_update_checksum(&le->outer, le, le->m_head->m_pkthdr.lro_tcp_d_len, le->m_head->m_pkthdr.lro_tcp_d_csum); le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID; le->m_head->m_pkthdr.csum_data = 0xffff; if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; break; case LRO_TYPE_IPV6_TCP: csum = tcp_lro_update_checksum(&le->outer, le, le->m_head->m_pkthdr.lro_tcp_d_len, le->m_head->m_pkthdr.lro_tcp_d_csum); le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR; le->m_head->m_pkthdr.csum_data = 0xffff; if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; break; default: break; } break; default: break; } } /* * Break any chain, this is not set to NULL on the singleton * case m_nextpkt points to m_head. Other case set them * m_nextpkt to NULL in push_and_replace. */ le->m_head->m_nextpkt = NULL; lc->lro_queued += le->m_head->m_pkthdr.lro_nsegs; (*lc->ifp->if_input)(lc->ifp, le->m_head); } static void tcp_set_entry_to_mbuf(struct lro_ctrl *lc, struct lro_entry *le, struct mbuf *m, struct tcphdr *th) { uint32_t *ts_ptr; uint16_t tcp_data_len; uint16_t tcp_opt_len; ts_ptr = (uint32_t *)(th + 1); tcp_opt_len = (th->th_off << 2); tcp_opt_len -= sizeof(*th); /* Check if there is a timestamp option. */ if (tcp_opt_len == 0 || __predict_false(tcp_opt_len != TCPOLEN_TSTAMP_APPA || *ts_ptr != TCP_LRO_TS_OPTION)) { /* We failed to find the timestamp option. */ le->timestamp = 0; } else { le->timestamp = 1; le->tsval = ntohl(*(ts_ptr + 1)); le->tsecr = *(ts_ptr + 2); } tcp_data_len = m->m_pkthdr.lro_tcp_d_len; /* Pull out TCP sequence numbers and window size. */ le->next_seq = ntohl(th->th_seq) + tcp_data_len; le->ack_seq = th->th_ack; le->window = th->th_win; le->flags = tcp_get_flags(th); le->needs_merge = 0; /* Setup new data pointers. */ le->m_head = m; le->m_tail = m_last(m); } static void tcp_push_and_replace(struct lro_ctrl *lc, struct lro_entry *le, struct mbuf *m) { struct lro_parser *pa; /* * Push up the stack of the current entry * and replace it with "m". */ struct mbuf *msave; /* Grab off the next and save it */ msave = le->m_head->m_nextpkt; le->m_head->m_nextpkt = NULL; /* Now push out the old entry */ tcp_flush_out_entry(lc, le); /* Re-parse new header, should not fail. */ pa = tcp_lro_parser(m, &le->outer, &le->inner, false); KASSERT(pa != NULL, ("tcp_push_and_replace: LRO parser failed on m=%p\n", m)); /* * Now to replace the data properly in the entry * we have to reset the TCP header and * other fields. */ tcp_set_entry_to_mbuf(lc, le, m, pa->tcp); /* Restore the next list */ m->m_nextpkt = msave; } static void tcp_lro_mbuf_append_pkthdr(struct lro_entry *le, const struct mbuf *p) { struct mbuf *m; uint32_t csum; m = le->m_head; if (m->m_pkthdr.lro_nsegs == 1) { /* Compute relative checksum. */ csum = p->m_pkthdr.lro_tcp_d_csum; } else { /* Merge TCP data checksums. */ csum = (uint32_t)m->m_pkthdr.lro_tcp_d_csum + (uint32_t)p->m_pkthdr.lro_tcp_d_csum; while (csum > 0xffff) csum = (csum >> 16) + (csum & 0xffff); } /* Update various counters. */ m->m_pkthdr.len += p->m_pkthdr.lro_tcp_d_len; m->m_pkthdr.lro_tcp_d_csum = csum; m->m_pkthdr.lro_tcp_d_len += p->m_pkthdr.lro_tcp_d_len; m->m_pkthdr.lro_nsegs += p->m_pkthdr.lro_nsegs; le->needs_merge = 1; } static void tcp_lro_condense(struct lro_ctrl *lc, struct lro_entry *le) { /* * Walk through the mbuf chain we * have on tap and compress/condense * as required. */ uint32_t *ts_ptr; struct mbuf *m; struct tcphdr *th; uint32_t tcp_data_len_total; uint32_t tcp_data_seg_total; uint16_t tcp_data_len; uint16_t tcp_opt_len; /* * First we must check the lead (m_head) * we must make sure that it is *not* * something that should be sent up * right away (sack etc). */ again: m = le->m_head->m_nextpkt; if (m == NULL) { /* Just one left. */ return; } th = tcp_lro_get_th(m); tcp_opt_len = (th->th_off << 2); tcp_opt_len -= sizeof(*th); ts_ptr = (uint32_t *)(th + 1); if (tcp_opt_len != 0 && __predict_false(tcp_opt_len != TCPOLEN_TSTAMP_APPA || *ts_ptr != TCP_LRO_TS_OPTION)) { /* * Its not the timestamp. We can't * use this guy as the head. */ le->m_head->m_nextpkt = m->m_nextpkt; tcp_push_and_replace(lc, le, m); goto again; } if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH)) != 0) { /* * Make sure that previously seen segments/ACKs are delivered * before this segment, e.g. FIN. */ le->m_head->m_nextpkt = m->m_nextpkt; tcp_push_and_replace(lc, le, m); goto again; } while((m = le->m_head->m_nextpkt) != NULL) { /* * condense m into le, first * pull m out of the list. */ le->m_head->m_nextpkt = m->m_nextpkt; m->m_nextpkt = NULL; /* Setup my data */ tcp_data_len = m->m_pkthdr.lro_tcp_d_len; th = tcp_lro_get_th(m); ts_ptr = (uint32_t *)(th + 1); tcp_opt_len = (th->th_off << 2); tcp_opt_len -= sizeof(*th); tcp_data_len_total = le->m_head->m_pkthdr.lro_tcp_d_len + tcp_data_len; tcp_data_seg_total = le->m_head->m_pkthdr.lro_nsegs + m->m_pkthdr.lro_nsegs; if (tcp_data_seg_total >= lc->lro_ackcnt_lim || tcp_data_len_total >= lc->lro_length_lim) { /* Flush now if appending will result in overflow. */ tcp_push_and_replace(lc, le, m); goto again; } if (tcp_opt_len != 0 && __predict_false(tcp_opt_len != TCPOLEN_TSTAMP_APPA || *ts_ptr != TCP_LRO_TS_OPTION)) { /* * Maybe a sack in the new one? We need to * start all over after flushing the * current le. We will go up to the beginning * and flush it (calling the replace again possibly * or just returning). */ tcp_push_and_replace(lc, le, m); goto again; } if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH)) != 0) { tcp_push_and_replace(lc, le, m); goto again; } if (tcp_opt_len != 0) { uint32_t tsval = ntohl(*(ts_ptr + 1)); /* Make sure timestamp values are increasing. */ if (TSTMP_GT(le->tsval, tsval)) { tcp_push_and_replace(lc, le, m); goto again; } le->tsval = tsval; le->tsecr = *(ts_ptr + 2); } /* Try to append the new segment. */ if (__predict_false(ntohl(th->th_seq) != le->next_seq || ((tcp_get_flags(th) & TH_ACK) != (le->flags & TH_ACK)) || (tcp_data_len == 0 && le->ack_seq == th->th_ack && le->window == th->th_win))) { /* Out of order packet, non-ACK + ACK or dup ACK. */ tcp_push_and_replace(lc, le, m); goto again; } if (tcp_data_len != 0 || SEQ_GT(ntohl(th->th_ack), ntohl(le->ack_seq))) { le->next_seq += tcp_data_len; le->ack_seq = th->th_ack; le->window = th->th_win; le->needs_merge = 1; } else if (th->th_ack == le->ack_seq) { if (WIN_GT(th->th_win, le->window)) { le->window = th->th_win; le->needs_merge = 1; } } if (tcp_data_len == 0) { m_freem(m); continue; } /* Merge TCP data checksum and length to head mbuf. */ tcp_lro_mbuf_append_pkthdr(le, m); /* * Adjust the mbuf so that m_data points to the first byte of * the ULP payload. Adjust the mbuf to avoid complications and * append new segment to existing mbuf chain. */ m_adj(m, m->m_pkthdr.len - tcp_data_len); m_demote_pkthdr(m); le->m_tail->m_next = m; le->m_tail = m_last(m); } } void tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le) { - /* Only optimise if there are multiple packets waiting. */ -#ifdef TCPHPTS - int error; -#endif + /* Only optimise if there are multiple packets waiting. */ NET_EPOCH_ASSERT(); -#ifdef TCPHPTS - CURVNET_SET(lc->ifp->if_vnet); - error = tcp_lro_flush_tcphpts(lc, le); - CURVNET_RESTORE(); - if (error != 0) { -#endif + if (tcp_lro_flush_tcphpts == NULL || + tcp_lro_flush_tcphpts(lc, le) != 0) { tcp_lro_condense(lc, le); tcp_flush_out_entry(lc, le); -#ifdef TCPHPTS } -#endif lc->lro_flushed++; bzero(le, sizeof(*le)); LIST_INSERT_HEAD(&lc->lro_free, le, next); } #define tcp_lro_msb_64(x) (1ULL << (flsll(x) - 1)) /* * The tcp_lro_sort() routine is comparable to qsort(), except it has * a worst case complexity limit of O(MIN(N,64)*N), where N is the * number of elements to sort and 64 is the number of sequence bits * available. The algorithm is bit-slicing the 64-bit sequence number, * sorting one bit at a time from the most significant bit until the * least significant one, skipping the constant bits. This is * typically called a radix sort. */ static void tcp_lro_sort(struct lro_mbuf_sort *parray, uint32_t size) { struct lro_mbuf_sort temp; uint64_t ones; uint64_t zeros; uint32_t x; uint32_t y; repeat: /* for small arrays insertion sort is faster */ if (size <= 12) { for (x = 1; x < size; x++) { temp = parray[x]; for (y = x; y > 0 && temp.seq < parray[y - 1].seq; y--) parray[y] = parray[y - 1]; parray[y] = temp; } return; } /* compute sequence bits which are constant */ ones = 0; zeros = 0; for (x = 0; x != size; x++) { ones |= parray[x].seq; zeros |= ~parray[x].seq; } /* compute bits which are not constant into "ones" */ ones &= zeros; if (ones == 0) return; /* pick the most significant bit which is not constant */ ones = tcp_lro_msb_64(ones); /* * Move entries having cleared sequence bits to the beginning * of the array: */ for (x = y = 0; y != size; y++) { /* skip set bits */ if (parray[y].seq & ones) continue; /* swap entries */ temp = parray[x]; parray[x] = parray[y]; parray[y] = temp; x++; } KASSERT(x != 0 && x != size, ("Memory is corrupted\n")); /* sort zeros */ tcp_lro_sort(parray, x); /* sort ones */ parray += x; size -= x; goto repeat; } void tcp_lro_flush_all(struct lro_ctrl *lc) { uint64_t seq; uint64_t nseq; unsigned x; NET_EPOCH_ASSERT(); /* check if no mbufs to flush */ if (lc->lro_mbuf_count == 0) goto done; if (lc->lro_cpu_is_set == 0) { if (lc->lro_last_cpu == curcpu) { lc->lro_cnt_of_same_cpu++; /* Have we reached the threshold to declare a cpu? */ if (lc->lro_cnt_of_same_cpu > tcp_lro_cpu_set_thresh) lc->lro_cpu_is_set = 1; } else { lc->lro_last_cpu = curcpu; lc->lro_cnt_of_same_cpu = 0; } } CURVNET_SET(lc->ifp->if_vnet); /* get current time */ binuptime(&lc->lro_last_queue_time); /* sort all mbufs according to stream */ tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count); /* input data into LRO engine, stream by stream */ seq = 0; for (x = 0; x != lc->lro_mbuf_count; x++) { struct mbuf *mb; /* get mbuf */ mb = lc->lro_mbuf_data[x].mb; /* get sequence number, masking away the packet index */ nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24); /* check for new stream */ if (seq != nseq) { seq = nseq; /* flush active streams */ tcp_lro_rx_done(lc); } /* add packet to LRO engine */ if (tcp_lro_rx_common(lc, mb, 0, false) != 0) { /* Flush anything we have acummulated */ tcp_lro_flush_active(lc); /* input packet to network layer */ (*lc->ifp->if_input)(lc->ifp, mb); lc->lro_queued++; lc->lro_flushed++; } } CURVNET_RESTORE(); done: /* flush active streams */ tcp_lro_rx_done(lc); - -#ifdef TCPHPTS - tcp_run_hpts(); -#endif + if (tcp_hpts_softclock != NULL) + tcp_hpts_softclock(); lc->lro_mbuf_count = 0; } static struct lro_head * tcp_lro_rx_get_bucket(struct lro_ctrl *lc, struct mbuf *m, struct lro_parser *parser) { u_long hash; if (M_HASHTYPE_ISHASH(m)) { hash = m->m_pkthdr.flowid; } else { for (unsigned i = hash = 0; i != LRO_RAW_ADDRESS_MAX; i++) hash += parser->data.raw[i]; } return (&lc->lro_hash[hash % lc->lro_hashsz]); } static int tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, bool use_hash) { struct lro_parser pi; /* inner address data */ struct lro_parser po; /* outer address data */ struct lro_parser *pa; /* current parser for TCP stream */ struct lro_entry *le; struct lro_head *bucket; struct tcphdr *th; int tcp_data_len; int tcp_opt_len; int error; uint16_t tcp_data_sum; #ifdef INET /* Quickly decide if packet cannot be LRO'ed */ if (__predict_false(V_ipforwarding != 0)) return (TCP_LRO_CANNOT); #endif #ifdef INET6 /* Quickly decide if packet cannot be LRO'ed */ if (__predict_false(V_ip6_forwarding != 0)) return (TCP_LRO_CANNOT); #endif if (((m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) != ((CSUM_DATA_VALID | CSUM_PSEUDO_HDR))) || (m->m_pkthdr.csum_data != 0xffff)) { /* * The checksum either did not have hardware offload * or it was a bad checksum. We can't LRO such * a packet. */ counter_u64_add(tcp_bad_csums, 1); return (TCP_LRO_CANNOT); } /* We expect a contiguous header [eh, ip, tcp]. */ pa = tcp_lro_parser(m, &po, &pi, true); if (__predict_false(pa == NULL)) return (TCP_LRO_NOT_SUPPORTED); /* We don't expect any padding. */ error = tcp_lro_trim_mbuf_chain(m, pa); if (__predict_false(error != 0)) return (error); #ifdef INET switch (pa->data.lro_type) { case LRO_TYPE_IPV4_TCP: error = tcp_lro_rx_ipv4(lc, m, pa->ip4); if (__predict_false(error != 0)) return (error); break; default: break; } #endif /* If no hardware or arrival stamp on the packet add timestamp */ if ((m->m_flags & (M_TSTMP_LRO | M_TSTMP)) == 0) { m->m_pkthdr.rcv_tstmp = bintime2ns(&lc->lro_last_queue_time); m->m_flags |= M_TSTMP_LRO; } /* Get pointer to TCP header. */ th = pa->tcp; /* Don't process SYN packets. */ if (__predict_false(tcp_get_flags(th) & TH_SYN)) return (TCP_LRO_CANNOT); /* Get total TCP header length and compute payload length. */ tcp_opt_len = (th->th_off << 2); tcp_data_len = m->m_pkthdr.len - ((uint8_t *)th - (uint8_t *)m->m_data) - tcp_opt_len; tcp_opt_len -= sizeof(*th); /* Don't process invalid TCP headers. */ if (__predict_false(tcp_opt_len < 0 || tcp_data_len < 0)) return (TCP_LRO_CANNOT); /* Compute TCP data only checksum. */ if (tcp_data_len == 0) tcp_data_sum = 0; /* no data, no checksum */ else if (__predict_false(csum != 0)) tcp_data_sum = tcp_lro_rx_csum_data(pa, ~csum); else tcp_data_sum = tcp_lro_rx_csum_data(pa, ~th->th_sum); /* Save TCP info in mbuf. */ m->m_nextpkt = NULL; m->m_pkthdr.rcvif = lc->ifp; m->m_pkthdr.lro_tcp_d_csum = tcp_data_sum; m->m_pkthdr.lro_tcp_d_len = tcp_data_len; m->m_pkthdr.lro_tcp_h_off = ((uint8_t *)th - (uint8_t *)m->m_data); m->m_pkthdr.lro_nsegs = 1; /* Get hash bucket. */ if (!use_hash) { bucket = &lc->lro_hash[0]; } else { bucket = tcp_lro_rx_get_bucket(lc, m, pa); } /* Try to find a matching previous segment. */ LIST_FOREACH(le, bucket, hash_next) { /* Compare addresses and ports. */ if (lro_address_compare(&po.data, &le->outer.data) == false || lro_address_compare(&pi.data, &le->inner.data) == false) continue; /* Check if no data and old ACK. */ if (tcp_data_len == 0 && SEQ_LT(ntohl(th->th_ack), ntohl(le->ack_seq))) { m_freem(m); return (0); } /* Mark "m" in the last spot. */ le->m_last_mbuf->m_nextpkt = m; /* Now set the tail to "m". */ le->m_last_mbuf = m; return (0); } /* Try to find an empty slot. */ if (LIST_EMPTY(&lc->lro_free)) return (TCP_LRO_NO_ENTRIES); /* Start a new segment chain. */ le = LIST_FIRST(&lc->lro_free); LIST_REMOVE(le, next); tcp_lro_active_insert(lc, bucket, le); /* Make sure the headers are set. */ le->inner = pi; le->outer = po; /* Store time this entry was allocated. */ le->alloc_time = lc->lro_last_queue_time; tcp_set_entry_to_mbuf(lc, le, m, th); /* Now set the tail to "m". */ le->m_last_mbuf = m; return (0); } int tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum) { int error; if (((m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) != ((CSUM_DATA_VALID | CSUM_PSEUDO_HDR))) || (m->m_pkthdr.csum_data != 0xffff)) { /* * The checksum either did not have hardware offload * or it was a bad checksum. We can't LRO such * a packet. */ counter_u64_add(tcp_bad_csums, 1); return (TCP_LRO_CANNOT); } /* get current time */ binuptime(&lc->lro_last_queue_time); CURVNET_SET(lc->ifp->if_vnet); error = tcp_lro_rx_common(lc, m, csum, true); if (__predict_false(error != 0)) { /* * Flush anything we have acummulated * ahead of this packet that can't * be LRO'd. This preserves order. */ tcp_lro_flush_active(lc); } CURVNET_RESTORE(); return (error); } void tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb) { NET_EPOCH_ASSERT(); /* sanity checks */ if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL || lc->lro_mbuf_max == 0)) { /* packet drop */ m_freem(mb); return; } /* check if packet is not LRO capable */ if (__predict_false((lc->ifp->if_capenable & IFCAP_LRO) == 0)) { /* input packet to network layer */ (*lc->ifp->if_input) (lc->ifp, mb); return; } /* If no hardware or arrival stamp on the packet add timestamp */ if ((tcplro_stacks_wanting_mbufq > 0) && (tcp_less_accurate_lro_ts == 0) && ((mb->m_flags & M_TSTMP) == 0)) { /* Add in an LRO time since no hardware */ binuptime(&lc->lro_last_queue_time); mb->m_pkthdr.rcv_tstmp = bintime2ns(&lc->lro_last_queue_time); mb->m_flags |= M_TSTMP_LRO; } /* create sequence number */ lc->lro_mbuf_data[lc->lro_mbuf_count].seq = (((uint64_t)M_HASHTYPE_GET(mb)) << 56) | (((uint64_t)mb->m_pkthdr.flowid) << 24) | ((uint64_t)lc->lro_mbuf_count); /* enter mbuf */ lc->lro_mbuf_data[lc->lro_mbuf_count].mb = mb; /* flush if array is full */ if (__predict_false(++lc->lro_mbuf_count == lc->lro_mbuf_max)) tcp_lro_flush_all(lc); } /* end */ diff --git a/sys/netinet/tcp_lro.h b/sys/netinet/tcp_lro.h index d981c940e7eb..b4b5e3f811e4 100644 --- a/sys/netinet/tcp_lro.h +++ b/sys/netinet/tcp_lro.h @@ -1,231 +1,231 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006, Myricom Inc. * Copyright (c) 2008, Intel Corporation. * Copyright (c) 2016-2021 Mellanox Technologies. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _TCP_LRO_H_ #define _TCP_LRO_H_ #include #include #include #include #ifndef TCP_LRO_ENTRIES /* Define default number of LRO entries per RX queue */ #define TCP_LRO_ENTRIES 8 #endif /* * Flags for ACK entry for compression * the bottom 12 bits has the th_x2|th_flags. * LRO itself adds only the TSTMP flags * to indicate if either of the types * of timestamps are filled and the * HAS_TSTMP option to indicate if the * TCP timestamp option is valid. * * The other 1 flag bits are for processing * by a stack. * */ #define TSTMP_LRO 0x1000 #define TSTMP_HDWR 0x2000 #define HAS_TSTMP 0x4000 /* * Default number of interrupts on the same cpu in a row * that will cause us to declare a "affinity cpu". */ #define TCP_LRO_CPU_DECLARATION_THRESH 50 struct inpcb; /* Precompute the LRO_RAW_ADDRESS_MAX value: */ #define LRO_RAW_ADDRESS_MAX \ howmany(12 + 2 * sizeof(struct in6_addr), sizeof(u_long)) union lro_address { u_long raw[LRO_RAW_ADDRESS_MAX]; struct { uint8_t lro_type; /* internal */ #define LRO_TYPE_NONE 0 #define LRO_TYPE_IPV4_TCP 1 #define LRO_TYPE_IPV6_TCP 2 #define LRO_TYPE_IPV4_UDP 3 #define LRO_TYPE_IPV6_UDP 4 uint8_t lro_flags; #define LRO_FLAG_DECRYPTED 1 uint16_t vlan_id; /* VLAN identifier */ uint16_t s_port; /* source TCP/UDP port */ uint16_t d_port; /* destination TCP/UDP port */ uint32_t vxlan_vni; /* VXLAN virtual network identifier */ union { struct in_addr v4; struct in6_addr v6; } s_addr; /* source IPv4/IPv6 address */ union { struct in_addr v4; struct in6_addr v6; } d_addr; /* destination IPv4/IPv6 address */ }; }; _Static_assert(sizeof(union lro_address) == sizeof(u_long) * LRO_RAW_ADDRESS_MAX, "The raw field in the lro_address union does not cover the whole structure."); /* Optimize address comparison by comparing one unsigned long at a time: */ static inline bool lro_address_compare(const union lro_address *pa, const union lro_address *pb) { if (pa->lro_type == LRO_TYPE_NONE && pb->lro_type == LRO_TYPE_NONE) { return (true); } else for (unsigned i = 0; i < LRO_RAW_ADDRESS_MAX; i++) { if (pa->raw[i] != pb->raw[i]) return (false); } return (true); } struct lro_parser { union lro_address data; union { uint8_t *l3; struct ip *ip4; struct ip6_hdr *ip6; }; union { uint8_t *l4; struct tcphdr *tcp; struct udphdr *udp; }; uint16_t total_hdr_len; }; /* This structure is zeroed frequently, try to keep it small. */ struct lro_entry { LIST_ENTRY(lro_entry) next; LIST_ENTRY(lro_entry) hash_next; struct mbuf *m_head; struct mbuf *m_tail; struct mbuf *m_last_mbuf; struct lro_parser outer; struct lro_parser inner; uint32_t next_seq; /* tcp_seq */ uint32_t ack_seq; /* tcp_seq */ uint32_t tsval; uint32_t tsecr; uint16_t compressed; uint16_t uncompressed; uint16_t window; uint16_t flags : 12, /* 12 TCP header bits */ timestamp : 1, needs_merge : 1, reserved : 2; /* unused */ struct bintime alloc_time; /* time when entry was allocated */ }; LIST_HEAD(lro_head, lro_entry); struct lro_mbuf_sort { uint64_t seq; struct mbuf *mb; }; /* NB: This is part of driver structs. */ struct lro_ctrl { struct ifnet *ifp; struct lro_mbuf_sort *lro_mbuf_data; struct bintime lro_last_queue_time; /* last time data was queued */ uint64_t lro_queued; uint64_t lro_flushed; uint64_t lro_bad_csum; unsigned lro_cnt; unsigned lro_mbuf_count; unsigned lro_mbuf_max; unsigned short lro_ackcnt_lim; /* max # of aggregated ACKs */ unsigned short lro_cpu; /* Guess at the cpu we have affinity too */ unsigned lro_length_lim; /* max len of aggregated data */ u_long lro_hashsz; uint32_t lro_last_cpu; uint32_t lro_cnt_of_same_cpu; struct lro_head *lro_hash; struct lro_head lro_active; struct lro_head lro_free; uint8_t lro_cpu_is_set; /* Flag to say its ok to set the CPU on the inp */ }; struct tcp_ackent { uint64_t timestamp; /* hardware or sofware timestamp, valid if TSTMP_LRO or TSTMP_HDRW set */ uint32_t seq; /* th_seq value */ uint32_t ack; /* th_ack value */ uint32_t ts_value; /* If ts option value, valid if HAS_TSTMP is set */ uint32_t ts_echo; /* If ts option echo, valid if HAS_TSTMP is set */ uint16_t win; /* TCP window */ uint16_t flags; /* Flags to say if TS is present and type of timestamp and th_flags */ uint8_t codepoint; /* IP level codepoint including ECN bits */ uint8_t ack_val_set; /* Classification of ack used by the stack */ uint8_t pad[2]; /* To 32 byte boundary */ }; /* We use two M_PROTO on the mbuf */ #define M_ACKCMP M_PROTO4 /* Indicates LRO is sending in a Ack-compression mbuf */ #define M_LRO_EHDRSTRP M_PROTO6 /* Indicates that LRO has stripped the etherenet header */ #define TCP_LRO_LENGTH_MAX (65535 - 255) /* safe value with room for outer headers */ #define TCP_LRO_ACKCNT_MAX 65535 /* unlimited */ #define TCP_LRO_TS_OPTION ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |\ (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP) static inline struct tcphdr * tcp_lro_get_th(struct mbuf *m) { return ((struct tcphdr *)((char *)m->m_data + m->m_pkthdr.lro_tcp_h_off)); } extern long tcplro_stacks_wanting_mbufq; int tcp_lro_init(struct lro_ctrl *); int tcp_lro_init_args(struct lro_ctrl *, struct ifnet *, unsigned, unsigned); void tcp_lro_free(struct lro_ctrl *); void tcp_lro_flush_inactive(struct lro_ctrl *, const struct timeval *); void tcp_lro_flush(struct lro_ctrl *, struct lro_entry *); void tcp_lro_flush_all(struct lro_ctrl *); -int tcp_lro_flush_tcphpts(struct lro_ctrl *, struct lro_entry *); +extern int (*tcp_lro_flush_tcphpts)(struct lro_ctrl *, struct lro_entry *); int tcp_lro_rx(struct lro_ctrl *, struct mbuf *, uint32_t); void tcp_lro_queue_mbuf(struct lro_ctrl *, struct mbuf *); void tcp_lro_reg_mbufq(void); void tcp_lro_dereg_mbufq(void); #define TCP_LRO_NO_ENTRIES -2 #define TCP_LRO_CANNOT -1 #define TCP_LRO_NOT_SUPPORTED 1 #endif /* _TCP_LRO_H_ */ diff --git a/sys/netinet/tcp_lro_hpts.c b/sys/netinet/tcp_lro_hpts.c index 497da9cba40e..769c82a32391 100644 --- a/sys/netinet/tcp_lro_hpts.c +++ b/sys/netinet/tcp_lro_hpts.c @@ -1,577 +1,586 @@ /*- * Copyright (c) 2016-2018 Netflix, Inc. * Copyright (c) 2016-2021 Mellanox Technologies. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include "opt_inet.h" #include "opt_inet6.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void build_ack_entry(struct tcp_ackent *ae, struct tcphdr *th, struct mbuf *m, uint32_t *ts_ptr, uint16_t iptos) { /* * Given a TCP ACK, summarize it down into the small TCP ACK * entry. */ ae->timestamp = m->m_pkthdr.rcv_tstmp; ae->flags = 0; if (m->m_flags & M_TSTMP_LRO) ae->flags |= TSTMP_LRO; else if (m->m_flags & M_TSTMP) ae->flags |= TSTMP_HDWR; ae->seq = ntohl(th->th_seq); ae->ack = ntohl(th->th_ack); ae->flags |= tcp_get_flags(th); if (ts_ptr != NULL) { ae->ts_value = ntohl(ts_ptr[1]); ae->ts_echo = ntohl(ts_ptr[2]); ae->flags |= HAS_TSTMP; } ae->win = ntohs(th->th_win); ae->codepoint = iptos; } static inline bool tcp_lro_ack_valid(struct mbuf *m, struct tcphdr *th, uint32_t **ppts, bool *other_opts) { /* * This function returns two bits of valuable information. * a) Is what is present capable of being ack-compressed, * we can ack-compress if there is no options or just * a timestamp option, and of course the th_flags must * be correct as well. * b) Our other options present such as SACK. This is * used to determine if we want to wakeup or not. */ bool ret = true; switch (th->th_off << 2) { case (sizeof(*th) + TCPOLEN_TSTAMP_APPA): *ppts = (uint32_t *)(th + 1); /* Check if we have only one timestamp option. */ if (**ppts == TCP_LRO_TS_OPTION) *other_opts = false; else { *other_opts = true; ret = false; } break; case (sizeof(*th)): /* No options. */ *ppts = NULL; *other_opts = false; break; default: *ppts = NULL; *other_opts = true; ret = false; break; } /* For ACKCMP we only accept ACK, PUSH, ECE and CWR. */ if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH | TH_ECE | TH_CWR)) != 0) ret = false; /* If it has data on it we cannot compress it */ if (m->m_pkthdr.lro_tcp_d_len) ret = false; /* ACK flag must be set. */ if (!(tcp_get_flags(th) & TH_ACK)) ret = false; return (ret); } static bool tcp_lro_check_wake_status(struct tcpcb *tp) { if (tp->t_fb->tfb_early_wake_check != NULL) return ((tp->t_fb->tfb_early_wake_check)(tp)); return (false); } static void tcp_lro_log(struct tcpcb *tp, const struct lro_ctrl *lc, const struct lro_entry *le, const struct mbuf *m, int frm, int32_t tcp_data_len, uint32_t th_seq, uint32_t th_ack, uint16_t th_win) { if (tcp_bblogging_on(tp)) { union tcp_log_stackspecific log; struct timeval tv, btv; uint32_t cts; cts = tcp_get_usecs(&tv); memset(&log, 0, sizeof(union tcp_log_stackspecific)); log.u_bbr.flex8 = frm; log.u_bbr.flex1 = tcp_data_len; if (m) log.u_bbr.flex2 = m->m_pkthdr.len; else log.u_bbr.flex2 = 0; if (le->m_head) { log.u_bbr.flex3 = le->m_head->m_pkthdr.lro_nsegs; log.u_bbr.flex4 = le->m_head->m_pkthdr.lro_tcp_d_len; log.u_bbr.flex5 = le->m_head->m_pkthdr.len; log.u_bbr.delRate = le->m_head->m_flags; log.u_bbr.rttProp = le->m_head->m_pkthdr.rcv_tstmp; } log.u_bbr.inflight = th_seq; log.u_bbr.delivered = th_ack; log.u_bbr.timeStamp = cts; log.u_bbr.epoch = le->next_seq; log.u_bbr.lt_epoch = le->ack_seq; log.u_bbr.pacing_gain = th_win; log.u_bbr.cwnd_gain = le->window; log.u_bbr.lost = curcpu; log.u_bbr.cur_del_rate = (uintptr_t)m; log.u_bbr.bw_inuse = (uintptr_t)le->m_head; bintime2timeval(&lc->lro_last_queue_time, &btv); log.u_bbr.flex6 = tcp_tv_to_usectick(&btv); log.u_bbr.flex7 = le->compressed; log.u_bbr.pacing_gain = le->uncompressed; if (in_epoch(net_epoch_preempt)) log.u_bbr.inhpts = 1; else log.u_bbr.inhpts = 0; TCP_LOG_EVENTP(tp, NULL, &tptosocket(tp)->so_rcv, &tptosocket(tp)->so_snd, TCP_LOG_LRO, 0, 0, &log, false, &tv); } } static struct mbuf * tcp_lro_get_last_if_ackcmp(struct lro_ctrl *lc, struct lro_entry *le, struct tcpcb *tp, int32_t *new_m, bool can_append_old_cmp) { struct mbuf *m; /* Look at the last mbuf if any in queue */ if (can_append_old_cmp) { m = STAILQ_LAST(&tp->t_inqueue, mbuf, m_stailqpkt); if (m != NULL && (m->m_flags & M_ACKCMP) != 0) { if (M_TRAILINGSPACE(m) >= sizeof(struct tcp_ackent)) { tcp_lro_log(tp, lc, le, NULL, 23, 0, 0, 0, 0); *new_m = 0; counter_u64_add(tcp_extra_mbuf, 1); return (m); } else { /* Mark we ran out of space */ tp->t_flags2 |= TF2_MBUF_L_ACKS; } } } /* Decide mbuf size. */ tcp_lro_log(tp, lc, le, NULL, 21, 0, 0, 0, 0); if (tp->t_flags2 & TF2_MBUF_L_ACKS) m = m_getcl(M_NOWAIT, MT_DATA, M_ACKCMP | M_PKTHDR); else m = m_gethdr(M_NOWAIT, MT_DATA); if (__predict_false(m == NULL)) { counter_u64_add(tcp_would_have_but, 1); return (NULL); } counter_u64_add(tcp_comp_total, 1); m->m_pkthdr.rcvif = lc->ifp; m->m_flags |= M_ACKCMP; *new_m = 1; return (m); } /* * Do BPF tap for either ACK_CMP packets or MBUF QUEUE type packets * and strip all, but the IPv4/IPv6 header. */ static bool do_bpf_strip_and_compress(struct tcpcb *tp, struct lro_ctrl *lc, struct lro_entry *le, struct mbuf **pp, struct mbuf **cmp, struct mbuf **mv_to, bool *should_wake, bool bpf_req, bool lagg_bpf_req, struct ifnet *lagg_ifp, bool can_append_old_cmp) { union { void *ptr; struct ip *ip4; struct ip6_hdr *ip6; } l3; struct mbuf *m; struct mbuf *nm; struct tcphdr *th; struct tcp_ackent *ack_ent; uint32_t *ts_ptr; int32_t n_mbuf; bool other_opts, can_compress; uint8_t lro_type; uint16_t iptos; int tcp_hdr_offset; int idx; /* Get current mbuf. */ m = *pp; /* Let the BPF see the packet */ if (__predict_false(bpf_req)) ETHER_BPF_MTAP(lc->ifp, m); if (__predict_false(lagg_bpf_req)) ETHER_BPF_MTAP(lagg_ifp, m); tcp_hdr_offset = m->m_pkthdr.lro_tcp_h_off; lro_type = le->inner.data.lro_type; switch (lro_type) { case LRO_TYPE_NONE: lro_type = le->outer.data.lro_type; switch (lro_type) { case LRO_TYPE_IPV4_TCP: tcp_hdr_offset -= sizeof(*le->outer.ip4); m->m_pkthdr.lro_etype = ETHERTYPE_IP; break; case LRO_TYPE_IPV6_TCP: tcp_hdr_offset -= sizeof(*le->outer.ip6); m->m_pkthdr.lro_etype = ETHERTYPE_IPV6; break; default: goto compressed; } break; case LRO_TYPE_IPV4_TCP: tcp_hdr_offset -= sizeof(*le->outer.ip4); m->m_pkthdr.lro_etype = ETHERTYPE_IP; break; case LRO_TYPE_IPV6_TCP: tcp_hdr_offset -= sizeof(*le->outer.ip6); m->m_pkthdr.lro_etype = ETHERTYPE_IPV6; break; default: goto compressed; } MPASS(tcp_hdr_offset >= 0); m_adj(m, tcp_hdr_offset); m->m_flags |= M_LRO_EHDRSTRP; m->m_flags &= ~M_ACKCMP; m->m_pkthdr.lro_tcp_h_off -= tcp_hdr_offset; th = tcp_lro_get_th(m); th->th_sum = 0; /* TCP checksum is valid. */ /* Check if ACK can be compressed */ can_compress = tcp_lro_ack_valid(m, th, &ts_ptr, &other_opts); /* Now lets look at the should wake states */ if ((other_opts == true) && ((tp->t_flags2 & TF2_DONT_SACK_QUEUE) == 0)) { /* * If there are other options (SACK?) and the * tcp endpoint has not expressly told us it does * not care about SACKS, then we should wake up. */ *should_wake = true; } else if (*should_wake == false) { /* Wakeup override check if we are false here */ *should_wake = tcp_lro_check_wake_status(tp); } /* Is the ack compressable? */ if (can_compress == false) goto done; /* Does the TCP endpoint support ACK compression? */ if ((tp->t_flags2 & TF2_MBUF_ACKCMP) == 0) goto done; /* Lets get the TOS/traffic class field */ l3.ptr = mtod(m, void *); switch (lro_type) { case LRO_TYPE_IPV4_TCP: iptos = l3.ip4->ip_tos; break; case LRO_TYPE_IPV6_TCP: iptos = IPV6_TRAFFIC_CLASS(l3.ip6); break; default: iptos = 0; /* Keep compiler happy. */ break; } /* Now lets get space if we don't have some already */ if (*cmp == NULL) { new_one: nm = tcp_lro_get_last_if_ackcmp(lc, le, tp, &n_mbuf, can_append_old_cmp); if (__predict_false(nm == NULL)) goto done; *cmp = nm; if (n_mbuf) { /* * Link in the new cmp ack to our in-order place, * first set our cmp ack's next to where we are. */ nm->m_nextpkt = m; (*pp) = nm; /* * Set it up so mv_to is advanced to our * compressed ack. This way the caller can * advance pp to the right place. */ *mv_to = nm; /* * Advance it here locally as well. */ pp = &nm->m_nextpkt; } } else { /* We have one already we are working on */ nm = *cmp; if (M_TRAILINGSPACE(nm) < sizeof(struct tcp_ackent)) { /* We ran out of space */ tp->t_flags2 |= TF2_MBUF_L_ACKS; goto new_one; } } MPASS(M_TRAILINGSPACE(nm) >= sizeof(struct tcp_ackent)); counter_u64_add(tcp_inp_lro_compressed, 1); le->compressed++; /* We can add in to the one on the tail */ ack_ent = mtod(nm, struct tcp_ackent *); idx = (nm->m_len / sizeof(struct tcp_ackent)); build_ack_entry(&ack_ent[idx], th, m, ts_ptr, iptos); /* Bump the size of both pkt-hdr and len */ nm->m_len += sizeof(struct tcp_ackent); nm->m_pkthdr.len += sizeof(struct tcp_ackent); compressed: /* Advance to next mbuf before freeing. */ *pp = m->m_nextpkt; m->m_nextpkt = NULL; m_freem(m); return (true); done: counter_u64_add(tcp_uncomp_total, 1); le->uncompressed++; return (false); } static void tcp_queue_pkts(struct tcpcb *tp, struct lro_entry *le) { INP_WLOCK_ASSERT(tptoinpcb(tp)); STAILQ_HEAD(, mbuf) q = { le->m_head, &STAILQ_NEXT(le->m_last_mbuf, m_stailqpkt) }; STAILQ_CONCAT(&tp->t_inqueue, &q); le->m_head = NULL; le->m_last_mbuf = NULL; } static struct tcpcb * tcp_lro_lookup(struct ifnet *ifp, struct lro_parser *pa) { struct inpcb *inp; + CURVNET_SET(ifp->if_vnet); switch (pa->data.lro_type) { #ifdef INET6 case LRO_TYPE_IPV6_TCP: inp = in6_pcblookup(&V_tcbinfo, &pa->data.s_addr.v6, pa->data.s_port, &pa->data.d_addr.v6, pa->data.d_port, INPLOOKUP_WLOCKPCB, ifp); break; #endif #ifdef INET case LRO_TYPE_IPV4_TCP: inp = in_pcblookup(&V_tcbinfo, pa->data.s_addr.v4, pa->data.s_port, pa->data.d_addr.v4, pa->data.d_port, INPLOOKUP_WLOCKPCB, ifp); break; #endif default: + CURVNET_RESTORE(); return (NULL); } + CURVNET_RESTORE(); return (intotcpcb(inp)); } -int -tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le) +static int +_tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le) { struct tcpcb *tp; struct mbuf **pp, *cmp, *mv_to; struct ifnet *lagg_ifp; bool bpf_req, lagg_bpf_req, should_wake, can_append_old_cmp; /* Check if packet doesn't belongs to our network interface. */ if ((tcplro_stacks_wanting_mbufq == 0) || (le->outer.data.vlan_id != 0) || (le->inner.data.lro_type != LRO_TYPE_NONE)) return (TCP_LRO_CANNOT); #ifdef INET6 /* * Be proactive about unspecified IPv6 address in source. As * we use all-zero to indicate unbounded/unconnected pcb, * unspecified IPv6 address can be used to confuse us. * * Note that packets with unspecified IPv6 destination is * already dropped in ip6_input. */ if (__predict_false(le->outer.data.lro_type == LRO_TYPE_IPV6_TCP && IN6_IS_ADDR_UNSPECIFIED(&le->outer.data.s_addr.v6))) return (TCP_LRO_CANNOT); if (__predict_false(le->inner.data.lro_type == LRO_TYPE_IPV6_TCP && IN6_IS_ADDR_UNSPECIFIED(&le->inner.data.s_addr.v6))) return (TCP_LRO_CANNOT); #endif /* Lookup inp, if any. Returns locked TCP inpcb. */ tp = tcp_lro_lookup(lc->ifp, (le->inner.data.lro_type == LRO_TYPE_NONE) ? &le->outer : &le->inner); if (tp == NULL) return (TCP_LRO_CANNOT); counter_u64_add(tcp_inp_lro_locks_taken, 1); /* Check if the inp is dead, Jim. */ if (tp->t_state == TCPS_TIME_WAIT) { INP_WUNLOCK(tptoinpcb(tp)); return (TCP_LRO_CANNOT); } if (tp->t_lro_cpu == HPTS_CPU_NONE && lc->lro_cpu_is_set == 1) tp->t_lro_cpu = lc->lro_last_cpu; /* Check if the transport doesn't support the needed optimizations. */ if ((tp->t_flags2 & (TF2_SUPPORTS_MBUFQ | TF2_MBUF_ACKCMP)) == 0) { INP_WUNLOCK(tptoinpcb(tp)); return (TCP_LRO_CANNOT); } if (tp->t_flags2 & TF2_MBUF_QUEUE_READY) should_wake = false; else should_wake = true; /* Check if packets should be tapped to BPF. */ bpf_req = bpf_peers_present(lc->ifp->if_bpf); lagg_bpf_req = false; lagg_ifp = NULL; if (lc->ifp->if_type == IFT_IEEE8023ADLAG || lc->ifp->if_type == IFT_INFINIBANDLAG) { struct lagg_port *lp = lc->ifp->if_lagg; struct lagg_softc *sc = lp->lp_softc; lagg_ifp = sc->sc_ifp; if (lagg_ifp != NULL) lagg_bpf_req = bpf_peers_present(lagg_ifp->if_bpf); } /* Strip and compress all the incoming packets. */ can_append_old_cmp = true; cmp = NULL; for (pp = &le->m_head; *pp != NULL; ) { mv_to = NULL; if (do_bpf_strip_and_compress(tp, lc, le, pp, &cmp, &mv_to, &should_wake, bpf_req, lagg_bpf_req, lagg_ifp, can_append_old_cmp) == false) { /* Advance to next mbuf. */ pp = &(*pp)->m_nextpkt; /* * Once we have appended we can't look in the pending * inbound packets for a compressed ack to append to. */ can_append_old_cmp = false; /* * Once we append we also need to stop adding to any * compressed ack we were remembering. A new cmp * ack will be required. */ cmp = NULL; tcp_lro_log(tp, lc, le, NULL, 25, 0, 0, 0, 0); } else if (mv_to != NULL) { /* We are asked to move pp up */ pp = &mv_to->m_nextpkt; tcp_lro_log(tp, lc, le, NULL, 24, 0, 0, 0, 0); } else tcp_lro_log(tp, lc, le, NULL, 26, 0, 0, 0, 0); } /* Update "m_last_mbuf", if any. */ if (pp == &le->m_head) le->m_last_mbuf = *pp; else le->m_last_mbuf = __containerof(pp, struct mbuf, m_nextpkt); /* Check if any data mbufs left. */ if (le->m_head != NULL) { counter_u64_add(tcp_inp_lro_direct_queue, 1); tcp_lro_log(tp, lc, le, NULL, 22, 1, tp->t_flags2, 0, 1); tcp_queue_pkts(tp, le); } if (should_wake) { /* Wakeup */ counter_u64_add(tcp_inp_lro_wokeup_queue, 1); if ((*tp->t_fb->tfb_do_queued_segments)(tp, 0)) /* TCP cb gone and unlocked. */ return (0); } INP_WUNLOCK(tptoinpcb(tp)); return (0); /* Success. */ } + +void +tcp_lro_hpts_init(void) +{ + tcp_lro_flush_tcphpts = _tcp_lro_flush_tcphpts; +}