diff --git a/sys/kern/kern_jail.c b/sys/kern/kern_jail.c
--- a/sys/kern/kern_jail.c
+++ b/sys/kern/kern_jail.c
@@ -777,7 +777,7 @@
 
 /*
  * Restrict a prison's IP address list with its parent's, possibly replacing
- * it.  Return true if the replacement buffer was used (or would have been).
+ * it.  Return true if the replacement buffer was used (or should redo).
  * kern_jail_set() helper.
  */
 static bool
@@ -789,7 +789,7 @@
 	int (*const cmp)(const void *, const void *) = pr_families[af].cmp;
 	const size_t size = pr_families[af].size;
 	uint32_t ips;
-	bool alloced;
+	bool alloced, used;
 
 	mtx_assert(&pr->pr_mtx, MA_OWNED);
 
@@ -800,28 +800,44 @@
 	 * screw up sorting, and in case of IPv6 we can't even atomically write
 	 * one.
 	 */
-	ips = (pr->pr_flags & pr_families[af].ip_flag) ? pip->ips : ppip->ips;
-	if (ips == 0) {
-		prison_ip_set(pr, af, NULL);
+	if (ppip == NULL) {
+		if (pip != NULL)
+			prison_ip_set(pr, af, NULL);
 		return (false);
 	}
-	if (new == NULL) {
-		new = prison_ip_alloc(af, ips, M_NOWAIT);
-		if (new == NULL)
-			return (true);
-		alloced = true;
-	} else
-		alloced = false;
+
 	if (!(pr->pr_flags & pr_families[af].ip_flag)) {
+		if (new == NULL) {
+			new = prison_ip_alloc(af, ppip->ips, M_NOWAIT);
+			if (new == NULL)
+				return (true); /* redo */
+			used = false;
+		} else
+			used = true;
 		/* This has no user settings, so just copy the parent's list. */
-		bcopy(ppip + 1, new + 1, ips * size);
-	} else {
+		MPASS(new->ips == ppip->ips);
+		bcopy(ppip + 1, new + 1, ppip->ips * size);
+		prison_ip_set(pr, af, new);
+		return (used);
+	} else if (pip != NULL) {
 		/* Remove addresses that aren't in the parent. */
 		int i;
 
 		i = 0; /* index in pip */
 		ips = 0; /* index in new */
 
+		used = true;
+		if (new == NULL) {
+			new = prison_ip_alloc(af, pip->ips, M_NOWAIT);
+			if (new == NULL)
+				return (true); /* redo */
+			used = false;
+			alloced = true;
+		} else {
+			used = true;
+			alloced = false;
+		}
+
 		for (int pi = 0; pi < ppip->ips; pi++)
 			if (cmp(PR_IP(pip, 0), PR_IP(ppip, pi)) == 0) {
 				/* Found our primary address in parent. */
@@ -860,10 +876,17 @@
 			if (alloced)
 				prison_ip_free(new);
 			new = NULL;
+			used = false;
+		} else {
+			/* Shrink to real size */
+			KASSERT((new->ips >= ips),
+			    ("Out-of-bounds write to prison_ip %p", new));
+			new->ips = ips;
 		}
+		prison_ip_set(pr, af, new);
+		return (used);
 	}
-	prison_ip_set(pr, af, new);
-	return (new != NULL ? true : false);
+	return (false);
 }
 
 /*