diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -2864,12 +2864,12 @@
 	}
 	while ((inp = inp_next(&inpi)) != NULL)
 		if (inp->inp_gencnt == params->sop_id) {
-			if (inp->inp_flags & INP_DROPPED) {
+			if ((inp->inp_flags & INP_DROPPED) ||
+			    inp->inp_socket == NULL) {
 				INP_WUNLOCK(inp);
 				return (ECONNRESET);
 			}
 			so = inp->inp_socket;
-			KASSERT(so != NULL, ("inp_socket == NULL"));
 			soref(so);
 			error = (*ctloutput_set)(inp, &sopt);
 			sorele(so);
diff --git a/sys/netinet/siftr.c b/sys/netinet/siftr.c
--- a/sys/netinet/siftr.c
+++ b/sys/netinet/siftr.c
@@ -896,9 +896,10 @@
 
 	/*
 	 * If we can't find the TCP control block (happens occasionaly for a
-	 * packet sent during the shutdown phase of a TCP connection), bail
+	 * packet sent during the shutdown phase of a TCP connection),
+	 * or we're in the timewait state, bail
 	 */
-	if (tp == NULL) {
+	if (tp == NULL || tp->t_state == TCPS_TIME_WAIT) {
 		if (dir == PFIL_IN)
 			ss->nskip_in_tcpcb++;
 		else
@@ -1080,9 +1081,10 @@
 
 	/*
 	 * If we can't find the TCP control block (happens occasionaly for a
-	 * packet sent during the shutdown phase of a TCP connection), bail
+	 * packet sent during the shutdown phase of a TCP connection),
+	 * or we're in the timewait state, bail.
 	 */
-	if (tp == NULL) {
+	if (tp == NULL || tp->t_state == TCPS_TIME_WAIT) {
 		if (dir == PFIL_IN)
 			ss->nskip_in_tcpcb++;
 		else
diff --git a/sys/netinet/tcp_lro.c b/sys/netinet/tcp_lro.c
--- a/sys/netinet/tcp_lro.c
+++ b/sys/netinet/tcp_lro.c
@@ -1359,7 +1359,8 @@
 	tp = intotcpcb(inp);
 
 	/* Check if the inp is dead, Jim. */
-	if (tp == NULL || (inp->inp_flags & INP_DROPPED)) {
+	if (tp == NULL || (inp->inp_flags & INP_DROPPED) ||
+	    tp->t_state == TCPS_TIME_WAIT) {
 		INP_WUNLOCK(inp);
 		return (TCP_LRO_CANNOT);
 	}
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -2563,18 +2563,29 @@
 #endif
 	in_pcbdrop(inp);
 	TCPSTAT_INC(tcps_closed);
-	if (tp->t_state != TCPS_CLOSED)
+	/*
+	 * Connections in TIME_WAIT may have the socket already detached,
+	 * but tcp_use_detach() didn't free pcb.  In this case it our job
+	 * to free it.
+	 */
+	if ((so = inp->inp_socket) != NULL) {
+		if (tp->t_state != TCPS_CLOSED)
+			tcp_state_change(tp, TCPS_CLOSED);
+		soisdisconnected(so);
+		if (inp->inp_flags & INP_SOCKREF) {
+			inp->inp_flags &= ~INP_SOCKREF;
+			INP_WUNLOCK(inp);
+			sorele(so);
+			return (NULL);
+		} else
+			return (tp);	/* locked */
+	} else {
+		MPASS(tp->t_state == TCPS_TIME_WAIT);
 		tcp_state_change(tp, TCPS_CLOSED);
-	KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
-	so = inp->inp_socket;
-	soisdisconnected(so);
-	if (inp->inp_flags & INP_SOCKREF) {
-		inp->inp_flags &= ~INP_SOCKREF;
-		INP_WUNLOCK(inp);
-		sorele(so);
+		tcp_discardcb(tp);
+		in_pcbfree(inp);
 		return (NULL);
 	}
-	return (tp);
 }
 
 /*
@@ -3693,10 +3704,12 @@
 #endif
 	}
 	if (inp != NULL) {
-		if ((inp->inp_flags & INP_DROPPED) == 0 &&
-		    !SOLISTENING(inp->inp_socket)) {
-			tp = intotcpcb(inp);
-			tp = tcp_drop(tp, ECONNABORTED);
+		tp = intotcpcb(inp);
+		if ((inp->inp_flags & INP_DROPPED) == 0) {
+			if (tp->t_state == TCPS_TIME_WAIT)
+				tp = tcp_close(tp);
+			else if (!SOLISTENING(inp->inp_socket))
+				tp = tcp_drop(tp, ECONNABORTED);
 			if (tp != NULL)
 				INP_WUNLOCK(inp);
 		} else
diff --git a/sys/netinet/tcp_timewait.c b/sys/netinet/tcp_timewait.c
--- a/sys/netinet/tcp_timewait.c
+++ b/sys/netinet/tcp_timewait.c
@@ -116,6 +116,7 @@
 tcp_twstart(struct tcpcb *tp)
 {
 	struct inpcb *inp = tp->t_inpcb;
+	struct socket *so = inp->inp_socket;
 #ifdef INET6
 	bool isipv6 = inp->inp_inc.inc_flags & INC_ISIPV6;
 #endif
@@ -128,7 +129,7 @@
 	    "(inp->inp_flags & INP_DROPPED) != 0"));
 
 	tcp_state_change(tp, TCPS_TIME_WAIT);
-	soisdisconnected(inp->inp_socket);
+	soisdisconnected(so);
 
 	if (tp->t_flags & TF_ACKNOW)
 		tcp_output(tp);
@@ -149,7 +150,23 @@
 	}
 
 	tcp_timer_activate(tp, TT_2MSL, 2 * V_tcp_msl);
-	INP_WUNLOCK(inp);
+
+	/*
+	 * INP_SOCKREF means that the socket had been close(2)d by the
+	 * userland and it is the inpcb that keeps socket in memory.
+	 * To preserve memory, we want to release the socket now, not
+	 * when the TIME_WAIT state expires.  Note that tcp_usr_detach()
+	 * has a special case to handle a call from here.
+	 */
+	if (inp->inp_flags & INP_SOCKREF) {
+		inp->inp_flags &= ~INP_SOCKREF;
+#ifdef TCPHPTS
+		tcp_hpts_remove(tp->t_inpcb);
+#endif
+		INP_WUNLOCK(inp);
+		sorele(so);
+	} else
+		INP_WUNLOCK(inp);
 }
 
 /*
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -216,13 +216,27 @@
 
 	tp = intotcpcb(inp);
 
-	KASSERT(inp->inp_flags & INP_DROPPED ||
-	    tp->t_state < TCPS_SYN_SENT,
-	    ("%s: inp %p not dropped or embryonic", __func__, inp));
-
-	tcp_discardcb(tp);
-	in_pcbdetach(inp);
-	in_pcbfree(inp);
+	/*
+	 * Note on connections in TIME_WAIT.
+	 *
+	 * There are two cases here:
+	 * - We went through tcp_close() in our call stack, and the connection
+	 *   shall be processed normally.  Such pcbs are always marked with
+	 *   INP_DROPPED.
+	 * - We are detaching pcb from the socket in tcp_twstart(), for memory
+	 *   saving purposes.  The pcb shall remain.
+	 */
+	if (inp->inp_flags & INP_DROPPED) {
+		KASSERT(tp->t_state < TCPS_SYN_SENT,
+		    ("%s: dropped inp %p not embryonic", __func__, inp));
+		tcp_discardcb(tp);
+		in_pcbdetach(inp);
+		in_pcbfree(inp);
+	} else {
+		MPASS(tp->t_state == TCPS_TIME_WAIT);
+		in_pcbdetach(inp);
+		INP_WUNLOCK(inp);
+	}
 }
 
 #ifdef INET