Differential D15686 Diff 44054 head/sys/netinet/tcp_hpts.c

Changeset View

Standalone View

head/sys/netinet/tcp_hpts.c

Show First 20 Lines • Show All 178 Lines • ▼ Show 20 Lines
TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads);		TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads);

static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG;		static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG;

TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size);		TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size);

static struct tcp_hptsi tcp_pace;		static struct tcp_hptsi tcp_pace;

static int
tcp_hptsi_lock_inpinfo(struct inpcb *inp,
struct tcpcb **tp);
static void tcp_wakehpts(struct tcp_hpts_entry *p);		static void tcp_wakehpts(struct tcp_hpts_entry *p);
static void tcp_wakeinput(struct tcp_hpts_entry *p);		static void tcp_wakeinput(struct tcp_hpts_entry *p);
static void tcp_input_data(struct tcp_hpts_entry hpts, struct timeval tv);		static void tcp_input_data(struct tcp_hpts_entry hpts, struct timeval tv);
static void tcp_hptsi(struct tcp_hpts_entry hpts, struct timeval ctick);		static void tcp_hptsi(struct tcp_hpts_entry hpts, struct timeval ctick);
static void tcp_hpts_thread(void *ctx);		static void tcp_hpts_thread(void *ctx);
static void tcp_init_hptsi(void *st);		static void tcp_init_hptsi(void *st);

int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP;		int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP;
▲ Show 20 Lines • Show All 295 Lines • ▼ Show 20 Lines	sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS)
logging_on = logging_was;		logging_on = logging_was;
return error;		return error;
}		}

SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE,		SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE,
0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log");		0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log");


/*
* Try to get the INP_INFO lock.
*
* This function always succeeds in getting the lock. It will clear
* *tpp and return (1) if something critical changed while the inpcb
* was unlocked. Otherwise, it will leave *tpp unchanged and return (0).
*
* This function relies on the fact that the hpts always holds a
* reference on the inpcb while the segment is on the hptsi wheel and
* in the input queue.
*
*/
static int
tcp_hptsi_lock_inpinfo(struct inpcb inp, struct tcpcb *tpp)
{
struct tcp_function_block *tfb;
struct tcpcb *tp;
void *ptr;

/* Try the easy way. */
if (INP_INFO_TRY_RLOCK(&V_tcbinfo))
return (0);

/*
* OK, let's try the hard way. We'll save the function pointer block
* to make sure that doesn't change while we aren't holding the
* lock.
*/
tp = *tpp;
tfb = tp->t_fb;
ptr = tp->t_fb_ptr;
INP_WUNLOCK(inp);
INP_INFO_RLOCK(&V_tcbinfo);
INP_WLOCK(inp);
/* If the session went away, return an error. */
if ((inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) \|\|
(inp->inp_flags2 & INP_FREED)) {
*tpp = NULL;
return (1);
}
/*
* If the function block or stack-specific data block changed,
* report an error.
*/
tp = intotcpcb(inp);
if ((tp->t_fb != tfb) && (tp->t_fb_ptr != ptr)) {
*tpp = NULL;
return (1);
}
return (0);
}


static void		static void
tcp_wakehpts(struct tcp_hpts_entry *hpts)		tcp_wakehpts(struct tcp_hpts_entry *hpts)
{		{
HPTS_MTX_ASSERT(hpts);		HPTS_MTX_ASSERT(hpts);
swi_sched(hpts->ie_cookie, 0);		swi_sched(hpts->ie_cookie, 0);
if (hpts->p_hpts_active == 2) {		if (hpts->p_hpts_active == 2) {
/* Rare sleeping on a ENOBUF */		/* Rare sleeping on a ENOBUF */
wakeup_one(hpts);		wakeup_one(hpts);
▲ Show 20 Lines • Show All 723 Lines • ▼ Show 20 Lines	if (set_cpu) {
tcp_set_hpts(inp);		tcp_set_hpts(inp);
}		}
m = tp->t_in_pkt;		m = tp->t_in_pkt;
n = NULL;		n = NULL;
if (m != NULL &&		if (m != NULL &&
(m->m_pkthdr.pace_lock == TI_RLOCKED \|\|		(m->m_pkthdr.pace_lock == TI_RLOCKED \|\|
tp->t_state != TCPS_ESTABLISHED)) {		tp->t_state != TCPS_ESTABLISHED)) {
ti_locked = TI_RLOCKED;		ti_locked = TI_RLOCKED;
if (tcp_hptsi_lock_inpinfo(inp, &tp)) {		INP_INFO_RLOCK(&V_tcbinfo);
CURVNET_RESTORE();
goto out;
}
m = tp->t_in_pkt;		m = tp->t_in_pkt;
}		}
if (in_newts_every_tcb) {		if (in_newts_every_tcb) {
if (in_ts_percision)		if (in_ts_percision)
microuptime(tv);		microuptime(tv);
else		else
getmicrouptime(tv);		getmicrouptime(tv);
}		}
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	if ((m != NULL) && (m == tp->t_in_pkt)) {
INP_WLOCK(inp);		INP_WLOCK(inp);
/*		/*
* Since we have an opening here we must		* Since we have an opening here we must
* re-check if the tcb went away while we		* re-check if the tcb went away while we
* were getting the lock(s).		* were getting the lock(s).
*/		*/
if ((inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) \|\|		if ((inp->inp_flags & (INP_TIMEWAIT \| INP_DROPPED)) \|\|
(inp->inp_flags2 & INP_FREED)) {		(inp->inp_flags2 & INP_FREED)) {
out_free:
while (m) {		while (m) {
m_freem(m);		m_freem(m);
m = n;		m = n;
if (m)		if (m)
n = m->m_nextpkt;		n = m->m_nextpkt;
}		}
goto out;		goto out;
}		}
/*		/*
* Now that we hold the INP lock, check if		* Now that we hold the INP lock, check if
* we need to upgrade our lock.		* we need to upgrade our lock.
*/		*/
if (ti_locked == TI_UNLOCKED &&		if (ti_locked == TI_UNLOCKED &&
(tp->t_state != TCPS_ESTABLISHED)) {		(tp->t_state != TCPS_ESTABLISHED)) {
ti_locked = TI_RLOCKED;		ti_locked = TI_RLOCKED;
if (tcp_hptsi_lock_inpinfo(inp, &tp))		INP_INFO_RLOCK(&V_tcbinfo);
goto out_free;
}		}
} /** end while(m) */		} /** end while(m) */
} /** end if ((m != NULL) && (m == tp->t_in_pkt)) */		} /** end if ((m != NULL) && (m == tp->t_in_pkt)) */
if (in_pcbrele_wlocked(inp) == 0)		if (in_pcbrele_wlocked(inp) == 0)
INP_WUNLOCK(inp);		INP_WUNLOCK(inp);
if (ti_locked == TI_RLOCKED)		if (ti_locked == TI_RLOCKED)
INP_INFO_RUNLOCK(&V_tcbinfo);		INP_INFO_RUNLOCK(&V_tcbinfo);
INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);		INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
▲ Show 20 Lines • Show All 575 Lines • Show Last 20 Lines