diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -707,6 +707,9 @@
 	struct in6_addr *laddr6, *faddr6;
 #endif
 
+	KASSERT(fsa == NULL || fsa->sa_family == lsa->sa_family,
+	    ("%s: address family mismatch", __func__));
+
 	pcbinfo = inp->inp_pcbinfo;
 
 	/*
@@ -746,22 +749,31 @@
 
 #ifdef INET
 	laddr.s_addr = INADDR_ANY;	/* used by INET6+INET below too */
-	if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
-		if (lsa != NULL)
-			laddr = ((struct sockaddr_in *)lsa)->sin_addr;
-		if (fsa != NULL)
-			faddr = ((struct sockaddr_in *)fsa)->sin_addr;
-	}
 #endif
 #ifdef INET6
 	laddr6 = NULL;
-	if ((inp->inp_vflag & INP_IPV6) != 0) {
-		if (lsa != NULL)
+#endif
+
+	if (lsa != NULL) {
+#ifdef INET
+		if (lsa->sa_family == AF_INET)
+			laddr = ((struct sockaddr_in *)lsa)->sin_addr;
+#endif
+#ifdef INET6
+		if (lsa->sa_family == AF_INET6)
 			laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
-		if (fsa != NULL)
-			faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
+#endif
 	}
+	if (fsa != NULL) {
+#ifdef INET
+		if (fsa->sa_family == AF_INET)
+			faddr = ((struct sockaddr_in *)fsa)->sin_addr;
 #endif
+#ifdef INET6
+		else if (fsa->sa_family == AF_INET6)
+			faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
+#endif
+	}
 
 	tmpinp = NULL;
 	lport = *lportp;