Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/tcp_hpts.h
Show All 22 Lines | |||||
* SUCH DAMAGE. | * SUCH DAMAGE. | ||||
* | * | ||||
* $FreeBSD$ | * $FreeBSD$ | ||||
*/ | */ | ||||
#ifndef __tcp_hpts_h__ | #ifndef __tcp_hpts_h__ | ||||
#define __tcp_hpts_h__ | #define __tcp_hpts_h__ | ||||
/* | |||||
* The hpts uses a 102400 wheel. The wheel | |||||
* defines the time in 10 usec increments (102400 x 10). | |||||
* This gives a range of 10usec - 1024ms to place | |||||
* an entry within. If the user requests more than | |||||
* 1.024 second, a remaineder is attached and the hpts | |||||
* when seeing the remainder will re-insert the | |||||
* inpcb forward in time from where it is until | |||||
* the remainder is zero. | |||||
*/ | |||||
#define NUM_OF_HPTSI_SLOTS 102400 | |||||
TAILQ_HEAD(hptsh, inpcb); | |||||
/* Number of useconds in a hpts tick */ | /* Number of useconds in a hpts tick */ | ||||
#define HPTS_TICKS_PER_SLOT 10 | #define HPTS_TICKS_PER_SLOT 10 | ||||
#define HPTS_MS_TO_SLOTS(x) ((x * 100) + 1) | #define HPTS_MS_TO_SLOTS(x) ((x * 100) + 1) | ||||
#define HPTS_USEC_TO_SLOTS(x) ((x+9) /10) | #define HPTS_USEC_TO_SLOTS(x) ((x+9) /10) | ||||
#define HPTS_USEC_IN_SEC 1000000 | #define HPTS_USEC_IN_SEC 1000000 | ||||
#define HPTS_MSEC_IN_SEC 1000 | #define HPTS_MSEC_IN_SEC 1000 | ||||
#define HPTS_USEC_IN_MSEC 1000 | #define HPTS_USEC_IN_MSEC 1000 | ||||
Show All 26 Lines | |||||
#define PACE_TMR_RXT 0x08 /* Retransmit timer running */ | #define PACE_TMR_RXT 0x08 /* Retransmit timer running */ | ||||
#define PACE_TMR_PERSIT 0x10 /* Persists timer running */ | #define PACE_TMR_PERSIT 0x10 /* Persists timer running */ | ||||
#define PACE_TMR_KEEP 0x20 /* Keep alive timer running */ | #define PACE_TMR_KEEP 0x20 /* Keep alive timer running */ | ||||
#define PACE_PKT_OUTPUT 0x40 /* Output Packets being paced */ | #define PACE_PKT_OUTPUT 0x40 /* Output Packets being paced */ | ||||
#define PACE_TMR_MASK (PACE_TMR_KEEP|PACE_TMR_PERSIT|PACE_TMR_RXT|PACE_TMR_TLP|PACE_TMR_RACK|PACE_TMR_DELACK) | #define PACE_TMR_MASK (PACE_TMR_KEEP|PACE_TMR_PERSIT|PACE_TMR_RXT|PACE_TMR_TLP|PACE_TMR_RACK|PACE_TMR_DELACK) | ||||
#define DEFAULT_CONNECTION_THESHOLD 100 | #define DEFAULT_CONNECTION_THESHOLD 100 | ||||
#ifdef _KERNEL | |||||
/* Each hpts has its own p_mtx which is used for locking */ | |||||
struct tcp_hpts_entry { | |||||
/* Cache line 0x00 */ | |||||
struct mtx p_mtx; /* Mutex for hpts */ | |||||
struct timeval p_mysleep; /* Our min sleep time */ | |||||
uint64_t syscall_cnt; | |||||
uint64_t sleeping; /* What the actual sleep was (if sleeping) */ | |||||
uint16_t p_hpts_active; /* Flag that says hpts is awake */ | |||||
uint8_t p_wheel_complete; /* have we completed the wheel arc walk? */ | |||||
uint32_t p_curtick; /* Tick in 10 us the hpts is going to */ | |||||
uint32_t p_runningslot; /* Current tick we are at if we are running */ | |||||
uint32_t p_prev_slot; /* Previous slot we were on */ | |||||
uint32_t p_cur_slot; /* Current slot in wheel hpts is draining */ | |||||
uint32_t p_nxt_slot; /* The next slot outside the current range of | |||||
* slots that the hpts is running on. */ | |||||
int32_t p_on_queue_cnt; /* Count on queue in this hpts */ | |||||
uint32_t p_lasttick; /* Last tick before the current one */ | |||||
uint8_t p_direct_wake :1, /* boolean */ | |||||
p_on_min_sleep:1, /* boolean */ | |||||
p_hpts_wake_scheduled:1, /* boolean */ | |||||
p_avail:5; | |||||
uint8_t p_fill[3]; /* Fill to 32 bits */ | |||||
/* Cache line 0x40 */ | |||||
void *p_inp; | |||||
struct hptsh p_input; /* For the tcp-input runner */ | |||||
/* Hptsi wheel */ | |||||
struct hptsh *p_hptss; | |||||
int32_t p_on_inqueue_cnt; /* Count on input queue in this hpts */ | |||||
uint32_t p_hpts_sleep_time; /* Current sleep interval having a max | |||||
* of 255ms */ | |||||
uint32_t overidden_sleep; /* what was overrided by min-sleep for logging */ | |||||
uint32_t saved_lasttick; /* for logging */ | |||||
uint32_t saved_curtick; /* for logging */ | |||||
uint32_t saved_curslot; /* for logging */ | |||||
uint32_t saved_prev_slot; /* for logging */ | |||||
uint32_t p_delayed_by; /* How much were we delayed by */ | |||||
/* Cache line 0x80 */ | |||||
struct sysctl_ctx_list hpts_ctx; | |||||
struct sysctl_oid *hpts_root; | |||||
struct intr_event *ie; | |||||
void *ie_cookie; | |||||
uint16_t p_num; /* The hpts number one per cpu */ | |||||
uint16_t p_cpu; /* The hpts CPU */ | |||||
/* There is extra space in here */ | |||||
/* Cache line 0x100 */ | |||||
struct callout co __aligned(CACHE_LINE_SIZE); | |||||
} __aligned(CACHE_LINE_SIZE); | |||||
struct tcp_hptsi { | |||||
struct proc *rp_proc; /* Process structure for hpts */ | |||||
struct tcp_hpts_entry **rp_ent; /* Array of hptss */ | |||||
uint32_t *cts_last_ran; | |||||
uint32_t rp_num_hptss; /* Number of hpts threads */ | |||||
}; | |||||
#endif | |||||
#define HPTS_REMOVE_INPUT 0x01 | |||||
#define HPTS_REMOVE_OUTPUT 0x02 | |||||
#define HPTS_REMOVE_ALL (HPTS_REMOVE_INPUT | HPTS_REMOVE_OUTPUT) | |||||
/* | /* | ||||
* When using the hpts, a TCP stack must make sure | * When using the hpts, a TCP stack must make sure | ||||
* that once a INP_DROPPED flag is applied to a INP | * that once a INP_DROPPED flag is applied to a INP | ||||
* that it does not expect tcp_output() to ever be | * that it does not expect tcp_output() to ever be | ||||
* called by the hpts. The hpts will *not* call | * called by the hpts. The hpts will *not* call | ||||
* any output (or input) functions on a TCB that | * any output (or input) functions on a TCB that | ||||
* is in the DROPPED state. | * is in the DROPPED state. | ||||
* | * | ||||
Show All 28 Lines | |||||
* or 10000 ticks). If we were that late, the actual sleep time | * or 10000 ticks). If we were that late, the actual sleep time | ||||
* is adjusted down by 50%. If the ticks_ran is less than | * is adjusted down by 50%. If the ticks_ran is less than | ||||
* ticks_indicate_more_sleep (100 ticks or 1000usecs). | * ticks_indicate_more_sleep (100 ticks or 1000usecs). | ||||
* | * | ||||
*/ | */ | ||||
#ifdef _KERNEL | #ifdef _KERNEL | ||||
#define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED) | |||||
struct tcp_hpts_entry *tcp_hpts_lock(struct inpcb *inp); | |||||
struct tcp_hpts_entry *tcp_input_lock(struct inpcb *inp); | |||||
int __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line); | |||||
#define tcp_queue_to_hpts_immediate(a)__tcp_queue_to_hpts_immediate(a, __LINE__) | |||||
struct tcp_hpts_entry *tcp_cur_hpts(struct inpcb *inp); | |||||
#define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__) | #define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__) | ||||
void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line); | void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line); | ||||
#define HPTS_REMOVE_INPUT 0x01 | |||||
#define HPTS_REMOVE_OUTPUT 0x02 | |||||
#define HPTS_REMOVE_ALL (HPTS_REMOVE_INPUT | HPTS_REMOVE_OUTPUT) | |||||
static inline bool | static inline bool | ||||
tcp_in_hpts(struct inpcb *inp) | tcp_in_hpts(struct inpcb *inp) | ||||
{ | { | ||||
return (inp->inp_in_hpts > 0); | return (inp->inp_in_hpts > 0); | ||||
} | } | ||||
Show All 22 Lines | |||||
* you should already have the INP_WLOCK(). | * you should already have the INP_WLOCK(). | ||||
*/ | */ | ||||
uint32_t __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line); | uint32_t __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line); | ||||
#define tcp_hpts_insert(a, b) __tcp_hpts_insert(a, b, __LINE__) | #define tcp_hpts_insert(a, b) __tcp_hpts_insert(a, b, __LINE__) | ||||
uint32_t | uint32_t | ||||
tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag); | tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag); | ||||
int | |||||
__tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line); | |||||
#define tcp_queue_to_input_locked(a, b) __tcp_queue_to_input_locked(a, b, __LINE__); | |||||
int | |||||
__tcp_queue_to_input(struct inpcb *inp, int32_t line); | |||||
#define tcp_queue_to_input(a) __tcp_queue_to_input(a, __LINE__) | |||||
uint16_t tcp_hpts_delayedby(struct inpcb *inp); | |||||
void __tcp_set_hpts(struct inpcb *inp, int32_t line); | void __tcp_set_hpts(struct inpcb *inp, int32_t line); | ||||
#define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__) | #define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__) | ||||
void __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line); | void __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line); | ||||
#define tcp_set_inp_to_drop(a, b) __tcp_set_inp_to_drop(a, b, __LINE__) | #define tcp_set_inp_to_drop(a, b) __tcp_set_inp_to_drop(a, b, __LINE__) | ||||
void tcp_run_hpts(void); | void tcp_run_hpts(void); | ||||
uint16_t hpts_random_cpu(struct inpcb *inp); | |||||
extern int32_t tcp_min_hptsi_time; | extern int32_t tcp_min_hptsi_time; | ||||
#endif /* _KERNEL */ | #endif /* _KERNEL */ | ||||
/* | /* | ||||
* The following functions should also be available | * The following functions should also be available | ||||
* to userspace as well. | * to userspace as well. | ||||
*/ | */ | ||||
Show All 17 Lines | |||||
static __inline uint64_t | static __inline uint64_t | ||||
tcp_tv_to_lusectick(const struct timeval *sv) | tcp_tv_to_lusectick(const struct timeval *sv) | ||||
{ | { | ||||
return ((uint64_t)((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec)); | return ((uint64_t)((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec)); | ||||
} | } | ||||
#ifdef _KERNEL | #ifdef _KERNEL | ||||
static __inline void | |||||
tcp_hpts_unlock(struct tcp_hpts_entry *hpts) | |||||
{ | |||||
mtx_unlock(&hpts->p_mtx); | |||||
} | |||||
static __inline uint32_t | static __inline uint32_t | ||||
tcp_gethptstick(struct timeval *sv) | tcp_gethptstick(struct timeval *sv) | ||||
{ | { | ||||
struct timeval tv; | struct timeval tv; | ||||
if (sv == NULL) | if (sv == NULL) | ||||
sv = &tv; | sv = &tv; | ||||
microuptime(sv); | microuptime(sv); | ||||
Show All 16 Lines |