Index: sys/netinet/tcp_subr.c
===================================================================
--- sys/netinet/tcp_subr.c	(revision 277315)
+++ sys/netinet/tcp_subr.c	(working copy)
@@ -790,11 +790,16 @@
 		V_tcp_mssdflt;
 
 	/* Set up our timeouts. */
-	callout_init(&tp->t_timers->tt_rexmt, CALLOUT_MPSAFE);
-	callout_init(&tp->t_timers->tt_persist, CALLOUT_MPSAFE);
-	callout_init(&tp->t_timers->tt_keep, CALLOUT_MPSAFE);
-	callout_init(&tp->t_timers->tt_2msl, CALLOUT_MPSAFE);
-	callout_init(&tp->t_timers->tt_delack, CALLOUT_MPSAFE);
+	callout_init_rw(&tp->t_timers->tt_rexmt,
+	    &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
+	callout_init_rw(&tp->t_timers->tt_persist,
+	    &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
+	callout_init_rw(&tp->t_timers->tt_keep,
+	    &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
+	callout_init_rw(&tp->t_timers->tt_2msl,
+	    &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
+	callout_init_rw(&tp->t_timers->tt_delack,
+	    &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
 
 	if (V_tcp_do_rfc1323)
 		tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
@@ -888,6 +893,32 @@
 }
 
 /*
+ * Switch locks to avoid LOR
+ * Return values:
+ * 0: No lock switching needed
+ * 1: Success
+ * 2: Failure and all locks dropped
+ */
+int
+tcp_switch_locks(struct inpcb *inp)
+{
+	INP_WLOCK_ASSERT(inp);
+
+	if (rw_wowned(&V_tcbinfo.ipi_lock))
+		return (0);
+
+	in_pcbref(inp);
+	INP_WUNLOCK(inp);
+	INP_INFO_WLOCK(&V_tcbinfo);
+	INP_WLOCK(inp);
+	if (in_pcbrele_wlocked(inp)) {
+		INP_INFO_WUNLOCK(&V_tcbinfo);
+		return (2);
+	}
+	return (1);
+}
+
+/*
  * Drop a TCP connection, reporting
  * the specified error.  If connection is synchronized,
  * then send a RST to peer.
@@ -896,10 +927,17 @@
 tcp_drop(struct tcpcb *tp, int errno)
 {
 	struct socket *so = tp->t_inpcb->inp_socket;
+	struct tcpcb *retval;
+	int do_unlock;
 
-	INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
+	do_unlock = tcp_switch_locks(tp->t_inpcb);
+	if (do_unlock == 2)
+		return (NULL);
+
+	INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
+	
 	if (TCPS_HAVERCVDSYN(tp->t_state)) {
 		tcp_state_change(tp, TCPS_CLOSED);
 		(void) tcp_output(tp);
@@ -909,9 +947,34 @@
 	if (errno == ETIMEDOUT && tp->t_softerror)
 		errno = tp->t_softerror;
 	so->so_error = errno;
-	return (tcp_close(tp));
+	retval = tcp_close(tp);
+	if (do_unlock)
+		INP_INFO_WUNLOCK(&V_tcbinfo);
+	return (retval);
 }
 
+static void
+tcp_callout_drain(void *arg)
+{
+	struct tcpcb *tp = arg;
+	struct inpcb *inp;
+
+	inp = tp->t_inpcb;
+	
+	INP_WLOCK(inp);
+	if (--(tp->t_calloutref) != 0) {
+	  	INP_WUNLOCK(inp);
+		return;
+	}
+	inp->inp_ppcb = NULL;
+	if (in_pcbrele_wlocked(inp) == 0)
+		INP_WUNLOCK(inp);
+
+	CC_ALGO(tp) = NULL;
+	tp->t_inpcb = NULL;
+	uma_zfree(V_tcpcb_zone, tp);
+}
+
 void
 tcp_discardcb(struct tcpcb *tp)
 {
@@ -924,26 +987,25 @@
 	INP_WLOCK_ASSERT(inp);
 
 	/*
-	 * Make sure that all of our timers are stopped before we delete the
-	 * PCB.
-	 *
-	 * XXXRW: Really, we would like to use callout_drain() here in order
-	 * to avoid races experienced in tcp_timer.c where a timer is already
-	 * executing at this point.  However, we can't, both because we're
-	 * running in a context where we can't sleep, and also because we
-	 * hold locks required by the timers.  What we instead need to do is
-	 * test to see if callout_drain() is required, and if so, defer some
-	 * portion of the remainder of tcp_discardcb() to an asynchronous
-	 * context that can callout_drain() and then continue.  Some care
-	 * will be required to ensure that no further processing takes place
-	 * on the tcpcb, even though it hasn't been freed (a flag?).
+	 * Make sure that all of our timers are stopped before we
+	 * delete the PCB:
 	 */
-	callout_stop(&tp->t_timers->tt_rexmt);
-	callout_stop(&tp->t_timers->tt_persist);
-	callout_stop(&tp->t_timers->tt_keep);
-	callout_stop(&tp->t_timers->tt_2msl);
-	callout_stop(&tp->t_timers->tt_delack);
+	tp->t_calloutref = 6;	/* set initial value */
+	
+	if (callout_drain_async(&tp->t_timers->tt_rexmt, &tcp_callout_drain, tp) == 0)
+		tp->t_calloutref--;
+	if (callout_drain_async(&tp->t_timers->tt_persist, &tcp_callout_drain, tp) == 0)
+		tp->t_calloutref--;
+	if (callout_drain_async(&tp->t_timers->tt_keep, &tcp_callout_drain, tp) == 0)
+		tp->t_calloutref--;
+	if (callout_drain_async(&tp->t_timers->tt_2msl, &tcp_callout_drain, tp) == 0)
+		tp->t_calloutref--;
+	if (callout_drain_async(&tp->t_timers->tt_delack, &tcp_callout_drain, tp) == 0)
+		tp->t_calloutref--;
 
+	/* this ref is taken by the final tcp_callout_drain() */
+	in_pcbref(inp);
+
 	/*
 	 * If we got enough samples through the srtt filter,
 	 * save the rtt and rttvar in the routing entry.
@@ -1017,10 +1079,7 @@
 
 	khelp_destroy_osd(tp->osd);
 
-	CC_ALGO(tp) = NULL;
-	inp->inp_ppcb = NULL;
-	tp->t_inpcb = NULL;
-	uma_zfree(V_tcpcb_zone, tp);
+	tcp_callout_drain(tp);
 }
 
 /*
@@ -1032,10 +1091,16 @@
 {
 	struct inpcb *inp = tp->t_inpcb;
 	struct socket *so;
+	int do_unlock;
 
-	INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
 	INP_WLOCK_ASSERT(inp);
 
+	do_unlock = tcp_switch_locks(tp->t_inpcb);
+	if (do_unlock == 2)
+		return (NULL);
+
+	INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
+
 #ifdef TCP_OFFLOAD
 	if (tp->t_state == TCPS_LISTEN)
 		tcp_offload_listen_stop(tp);
@@ -1054,8 +1119,10 @@
 		SOCK_LOCK(so);
 		so->so_state &= ~SS_PROTOREF;
 		sofree(so);
-		return (NULL);
+		tp = NULL;
 	}
+	if (do_unlock)
+		INP_INFO_WUNLOCK(&V_tcbinfo);
 	return (tp);
 }
 
Index: sys/netinet/tcp_timer.c
===================================================================
--- sys/netinet/tcp_timer.c	(revision 277315)
+++ sys/netinet/tcp_timer.c	(working copy)
@@ -273,25 +273,7 @@
 	CURVNET_SET(tp->t_vnet);
 
 	inp = tp->t_inpcb;
-	/*
-	 * XXXRW: While this assert is in fact correct, bugs in the tcpcb
-	 * tear-down mean we need it as a work-around for races between
-	 * timers and tcp_discardcb().
-	 *
-	 * KASSERT(inp != NULL, ("tcp_timer_delack: inp == NULL"));
-	 */
-	if (inp == NULL) {
-		tcp_timer_race++;
-		CURVNET_RESTORE();
-		return;
-	}
-	INP_WLOCK(inp);
-	if (callout_pending(&tp->t_timers->tt_delack) ||
-	    !callout_active(&tp->t_timers->tt_delack)) {
-		INP_WUNLOCK(inp);
-		CURVNET_RESTORE();
-		return;
-	}
+
 	callout_deactivate(&tp->t_timers->tt_delack);
 	if ((inp->inp_flags & INP_DROPPED) != 0) {
 		INP_WUNLOCK(inp);
@@ -317,37 +299,11 @@
 
 	ostate = tp->t_state;
 #endif
-	/*
-	 * XXXRW: Does this actually happen?
-	 */
-	INP_INFO_WLOCK(&V_tcbinfo);
 	inp = tp->t_inpcb;
-	/*
-	 * XXXRW: While this assert is in fact correct, bugs in the tcpcb
-	 * tear-down mean we need it as a work-around for races between
-	 * timers and tcp_discardcb().
-	 *
-	 * KASSERT(inp != NULL, ("tcp_timer_2msl: inp == NULL"));
-	 */
-	if (inp == NULL) {
-		tcp_timer_race++;
-		INP_INFO_WUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
-	INP_WLOCK(inp);
 	tcp_free_sackholes(tp);
-	if (callout_pending(&tp->t_timers->tt_2msl) ||
-	    !callout_active(&tp->t_timers->tt_2msl)) {
-		INP_WUNLOCK(tp->t_inpcb);
-		INP_INFO_WUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
 	callout_deactivate(&tp->t_timers->tt_2msl);
 	if ((inp->inp_flags & INP_DROPPED) != 0) {
 		INP_WUNLOCK(inp);
-		INP_INFO_WUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 		return;
 	}
@@ -383,7 +339,6 @@
 #endif
 	if (tp != NULL)
 		INP_WUNLOCK(inp);
-	INP_INFO_WUNLOCK(&V_tcbinfo);
 	CURVNET_RESTORE();
 }
 
@@ -399,33 +354,10 @@
 
 	ostate = tp->t_state;
 #endif
-	INP_INFO_WLOCK(&V_tcbinfo);
 	inp = tp->t_inpcb;
-	/*
-	 * XXXRW: While this assert is in fact correct, bugs in the tcpcb
-	 * tear-down mean we need it as a work-around for races between
-	 * timers and tcp_discardcb().
-	 *
-	 * KASSERT(inp != NULL, ("tcp_timer_keep: inp == NULL"));
-	 */
-	if (inp == NULL) {
-		tcp_timer_race++;
-		INP_INFO_WUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
-	INP_WLOCK(inp);
-	if (callout_pending(&tp->t_timers->tt_keep) ||
-	    !callout_active(&tp->t_timers->tt_keep)) {
-		INP_WUNLOCK(inp);
-		INP_INFO_WUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
 	callout_deactivate(&tp->t_timers->tt_keep);
 	if ((inp->inp_flags & INP_DROPPED) != 0) {
 		INP_WUNLOCK(inp);
-		INP_INFO_WUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 		return;
 	}
@@ -472,7 +404,6 @@
 			  PRU_SLOWTIMO);
 #endif
 	INP_WUNLOCK(inp);
-	INP_INFO_WUNLOCK(&V_tcbinfo);
 	CURVNET_RESTORE();
 	return;
 
@@ -487,7 +418,6 @@
 #endif
 	if (tp != NULL)
 		INP_WUNLOCK(tp->t_inpcb);
-	INP_INFO_WUNLOCK(&V_tcbinfo);
 	CURVNET_RESTORE();
 }
 
@@ -502,33 +432,11 @@
 
 	ostate = tp->t_state;
 #endif
-	INP_INFO_WLOCK(&V_tcbinfo);
 	inp = tp->t_inpcb;
-	/*
-	 * XXXRW: While this assert is in fact correct, bugs in the tcpcb
-	 * tear-down mean we need it as a work-around for races between
-	 * timers and tcp_discardcb().
-	 *
-	 * KASSERT(inp != NULL, ("tcp_timer_persist: inp == NULL"));
-	 */
-	if (inp == NULL) {
-		tcp_timer_race++;
-		INP_INFO_WUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
-	INP_WLOCK(inp);
-	if (callout_pending(&tp->t_timers->tt_persist) ||
-	    !callout_active(&tp->t_timers->tt_persist)) {
-		INP_WUNLOCK(inp);
-		INP_INFO_WUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
+
 	callout_deactivate(&tp->t_timers->tt_persist);
 	if ((inp->inp_flags & INP_DROPPED) != 0) {
 		INP_WUNLOCK(inp);
-		INP_INFO_WUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 		return;
 	}
@@ -573,7 +481,6 @@
 #endif
 	if (tp != NULL)
 		INP_WUNLOCK(inp);
-	INP_INFO_WUNLOCK(&V_tcbinfo);
 	CURVNET_RESTORE();
 }
 
@@ -583,7 +490,7 @@
 	struct tcpcb *tp = xtp;
 	CURVNET_SET(tp->t_vnet);
 	int rexmt;
-	int headlocked;
+	int do_unlock = 0;
 	struct inpcb *inp;
 #ifdef TCPDEBUG
 	int ostate;
@@ -590,34 +497,10 @@
 
 	ostate = tp->t_state;
 #endif
-
-	INP_INFO_RLOCK(&V_tcbinfo);
 	inp = tp->t_inpcb;
-	/*
-	 * XXXRW: While this assert is in fact correct, bugs in the tcpcb
-	 * tear-down mean we need it as a work-around for races between
-	 * timers and tcp_discardcb().
-	 *
-	 * KASSERT(inp != NULL, ("tcp_timer_rexmt: inp == NULL"));
-	 */
-	if (inp == NULL) {
-		tcp_timer_race++;
-		INP_INFO_RUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
-	INP_WLOCK(inp);
-	if (callout_pending(&tp->t_timers->tt_rexmt) ||
-	    !callout_active(&tp->t_timers->tt_rexmt)) {
-		INP_WUNLOCK(inp);
-		INP_INFO_RUNLOCK(&V_tcbinfo);
-		CURVNET_RESTORE();
-		return;
-	}
 	callout_deactivate(&tp->t_timers->tt_rexmt);
 	if ((inp->inp_flags & INP_DROPPED) != 0) {
 		INP_WUNLOCK(inp);
-		INP_INFO_RUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 		return;
 	}
@@ -630,30 +513,18 @@
 	if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) {
 		tp->t_rxtshift = TCP_MAXRXTSHIFT;
 		TCPSTAT_INC(tcps_timeoutdrop);
-		in_pcbref(inp);
-		INP_INFO_RUNLOCK(&V_tcbinfo);
-		INP_WUNLOCK(inp);
-		INP_INFO_WLOCK(&V_tcbinfo);
-		INP_WLOCK(inp);
-		if (in_pcbrele_wlocked(inp)) {
-			INP_INFO_WUNLOCK(&V_tcbinfo);
+		do_unlock = tcp_switch_locks(inp);
+		if (do_unlock == 2) {
 			CURVNET_RESTORE();
 			return;
 		}
-		if (inp->inp_flags & INP_DROPPED) {
-			INP_WUNLOCK(inp);
-			INP_INFO_WUNLOCK(&V_tcbinfo);
-			CURVNET_RESTORE();
-			return;
-		}
+		if (inp->inp_flags & INP_DROPPED)
+			goto out;
 
 		tp = tcp_drop(tp, tp->t_softerror ?
 			      tp->t_softerror : ETIMEDOUT);
-		headlocked = 1;
 		goto out;
 	}
-	INP_INFO_RUNLOCK(&V_tcbinfo);
-	headlocked = 0;
 	if (tp->t_state == TCPS_SYN_SENT) {
 		/*
 		 * If the SYN was retransmitted, indicate CWND to be
@@ -843,7 +714,7 @@
 #endif
 	if (tp != NULL)
 		INP_WUNLOCK(inp);
-	if (headlocked)
+	if (do_unlock)
 		INP_INFO_WUNLOCK(&V_tcbinfo);
 	CURVNET_RESTORE();
 }
@@ -856,6 +727,8 @@
 	struct inpcb *inp = tp->t_inpcb;
 	int cpu = inp_to_cpuid(inp);
 
+	INP_WLOCK_ASSERT(tp->t_inpcb);
+	
 #ifdef TCP_OFFLOAD
 	if (tp->t_flags & TF_TOE)
 		return;
Index: sys/netinet/tcp_var.h
===================================================================
--- sys/netinet/tcp_var.h	(revision 277315)
+++ sys/netinet/tcp_var.h	(working copy)
@@ -153,6 +153,7 @@
 	u_long	max_sndwnd;		/* largest window peer has offered */
 
 	int	t_softerror;		/* possible error not yet reported */
+	u_char	t_calloutref;		/* callout drain references */
 /* out-of-band data */
 	char	t_oobflags;		/* have some */
 	char	t_iobc;			/* input character */
@@ -654,6 +655,7 @@
 void	 tcp_twclose(struct tcptw *, int);
 void	 tcp_ctlinput(int, struct sockaddr *, void *);
 int	 tcp_ctloutput(struct socket *, struct sockopt *);
+int	 tcp_switch_locks(struct inpcb *inp);
 struct tcpcb *
 	 tcp_drop(struct tcpcb *, int);
 void	 tcp_drain(void);