Changeset View
Changeset View
Standalone View
Standalone View
sys/netinet/tcp_hpts.c
Show First 20 Lines • Show All 179 Lines • ▼ Show 20 Lines | |||||
TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads); | TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads); | ||||
static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG; | static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG; | ||||
TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size); | TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size); | ||||
static struct tcp_hptsi tcp_pace; | static struct tcp_hptsi tcp_pace; | ||||
static int | |||||
tcp_hptsi_lock_inpinfo(struct inpcb *inp, | |||||
struct tcpcb **tp); | |||||
static void tcp_wakehpts(struct tcp_hpts_entry *p); | static void tcp_wakehpts(struct tcp_hpts_entry *p); | ||||
static void tcp_wakeinput(struct tcp_hpts_entry *p); | static void tcp_wakeinput(struct tcp_hpts_entry *p); | ||||
static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv); | static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv); | ||||
static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick); | static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick); | ||||
static void tcp_hpts_thread(void *ctx); | static void tcp_hpts_thread(void *ctx); | ||||
static void tcp_init_hptsi(void *st); | static void tcp_init_hptsi(void *st); | ||||
int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP; | int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP; | ||||
▲ Show 20 Lines • Show All 295 Lines • ▼ Show 20 Lines | sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS) | ||||
logging_on = logging_was; | logging_on = logging_was; | ||||
return error; | return error; | ||||
} | } | ||||
SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, | SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, | ||||
0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log"); | 0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log"); | ||||
/* | |||||
* Try to get the INP_INFO lock. | |||||
* | |||||
* This function always succeeds in getting the lock. It will clear | |||||
* *tpp and return (1) if something critical changed while the inpcb | |||||
* was unlocked. Otherwise, it will leave *tpp unchanged and return (0). | |||||
* | |||||
* This function relies on the fact that the hpts always holds a | |||||
* reference on the inpcb while the segment is on the hptsi wheel and | |||||
* in the input queue. | |||||
* | |||||
*/ | |||||
static int | |||||
tcp_hptsi_lock_inpinfo(struct inpcb *inp, struct tcpcb **tpp) | |||||
{ | |||||
struct tcp_function_block *tfb; | |||||
struct tcpcb *tp; | |||||
void *ptr; | |||||
/* Try the easy way. */ | |||||
if (INP_INFO_TRY_RLOCK(&V_tcbinfo)) | |||||
return (0); | |||||
/* | |||||
* OK, let's try the hard way. We'll save the function pointer block | |||||
* to make sure that doesn't change while we aren't holding the | |||||
* lock. | |||||
*/ | |||||
tp = *tpp; | |||||
tfb = tp->t_fb; | |||||
ptr = tp->t_fb_ptr; | |||||
INP_WUNLOCK(inp); | |||||
INP_INFO_RLOCK(&V_tcbinfo); | |||||
INP_WLOCK(inp); | |||||
/* If the session went away, return an error. */ | |||||
if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || | |||||
(inp->inp_flags2 & INP_FREED)) { | |||||
*tpp = NULL; | |||||
return (1); | |||||
} | |||||
/* | |||||
* If the function block or stack-specific data block changed, | |||||
* report an error. | |||||
*/ | |||||
tp = intotcpcb(inp); | |||||
if ((tp->t_fb != tfb) && (tp->t_fb_ptr != ptr)) { | |||||
*tpp = NULL; | |||||
return (1); | |||||
} | |||||
return (0); | |||||
} | |||||
static void | static void | ||||
tcp_wakehpts(struct tcp_hpts_entry *hpts) | tcp_wakehpts(struct tcp_hpts_entry *hpts) | ||||
{ | { | ||||
HPTS_MTX_ASSERT(hpts); | HPTS_MTX_ASSERT(hpts); | ||||
swi_sched(hpts->ie_cookie, 0); | swi_sched(hpts->ie_cookie, 0); | ||||
if (hpts->p_hpts_active == 2) { | if (hpts->p_hpts_active == 2) { | ||||
/* Rare sleeping on a ENOBUF */ | /* Rare sleeping on a ENOBUF */ | ||||
wakeup_one(hpts); | wakeup_one(hpts); | ||||
▲ Show 20 Lines • Show All 723 Lines • ▼ Show 20 Lines | if (set_cpu) { | ||||
tcp_set_hpts(inp); | tcp_set_hpts(inp); | ||||
} | } | ||||
m = tp->t_in_pkt; | m = tp->t_in_pkt; | ||||
n = NULL; | n = NULL; | ||||
if (m != NULL && | if (m != NULL && | ||||
(m->m_pkthdr.pace_lock == TI_RLOCKED || | (m->m_pkthdr.pace_lock == TI_RLOCKED || | ||||
tp->t_state != TCPS_ESTABLISHED)) { | tp->t_state != TCPS_ESTABLISHED)) { | ||||
ti_locked = TI_RLOCKED; | ti_locked = TI_RLOCKED; | ||||
if (tcp_hptsi_lock_inpinfo(inp, &tp)) { | INP_INFO_RLOCK(&V_tcbinfo); | ||||
CURVNET_RESTORE(); | |||||
goto out; | |||||
} | |||||
m = tp->t_in_pkt; | m = tp->t_in_pkt; | ||||
} | } | ||||
if (in_newts_every_tcb) { | if (in_newts_every_tcb) { | ||||
if (in_ts_percision) | if (in_ts_percision) | ||||
microuptime(tv); | microuptime(tv); | ||||
else | else | ||||
getmicrouptime(tv); | getmicrouptime(tv); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | if ((m != NULL) && (m == tp->t_in_pkt)) { | ||||
INP_WLOCK(inp); | INP_WLOCK(inp); | ||||
/* | /* | ||||
* Since we have an opening here we must | * Since we have an opening here we must | ||||
* re-check if the tcb went away while we | * re-check if the tcb went away while we | ||||
* were getting the lock(s). | * were getting the lock(s). | ||||
*/ | */ | ||||
if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || | if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || | ||||
(inp->inp_flags2 & INP_FREED)) { | (inp->inp_flags2 & INP_FREED)) { | ||||
out_free: | |||||
while (m) { | while (m) { | ||||
m_freem(m); | m_freem(m); | ||||
m = n; | m = n; | ||||
if (m) | if (m) | ||||
n = m->m_nextpkt; | n = m->m_nextpkt; | ||||
} | } | ||||
goto out; | goto out; | ||||
} | } | ||||
/* | /* | ||||
* Now that we hold the INP lock, check if | * Now that we hold the INP lock, check if | ||||
* we need to upgrade our lock. | * we need to upgrade our lock. | ||||
*/ | */ | ||||
if (ti_locked == TI_UNLOCKED && | if (ti_locked == TI_UNLOCKED && | ||||
(tp->t_state != TCPS_ESTABLISHED)) { | (tp->t_state != TCPS_ESTABLISHED)) { | ||||
ti_locked = TI_RLOCKED; | ti_locked = TI_RLOCKED; | ||||
if (tcp_hptsi_lock_inpinfo(inp, &tp)) | INP_INFO_RLOCK(&V_tcbinfo); | ||||
goto out_free; | |||||
} | } | ||||
} /** end while(m) */ | } /** end while(m) */ | ||||
} /** end if ((m != NULL) && (m == tp->t_in_pkt)) */ | } /** end if ((m != NULL) && (m == tp->t_in_pkt)) */ | ||||
if (in_pcbrele_wlocked(inp) == 0) | if (in_pcbrele_wlocked(inp) == 0) | ||||
INP_WUNLOCK(inp); | INP_WUNLOCK(inp); | ||||
if (ti_locked == TI_RLOCKED) | if (ti_locked == TI_RLOCKED) | ||||
INP_INFO_RUNLOCK(&V_tcbinfo); | INP_INFO_RUNLOCK(&V_tcbinfo); | ||||
INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); | INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); | ||||
▲ Show 20 Lines • Show All 575 Lines • Show Last 20 Lines |