diff --git a/sys/netpfil/pf/if_pfsync.c b/sys/netpfil/pf/if_pfsync.c
--- a/sys/netpfil/pf/if_pfsync.c
+++ b/sys/netpfil/pf/if_pfsync.c
@@ -185,7 +185,6 @@
 struct pfsync_deferral {
 	struct pfsync_softc		*pd_sc;
 	TAILQ_ENTRY(pfsync_deferral)	pd_entry;
-	u_int				pd_refs;
 	struct callout			pd_tmo;
 
 	struct pf_kstate		*pd_st;
@@ -421,10 +420,8 @@
 			if (ret > 0) {
 				pfsync_undefer(pd, 1);
 			} else {
-				pd->pd_refs++;
 				callout_drain(&pd->pd_tmo);
 			}
-			free(pd, M_PFSYNC);
 			PFSYNC_BUCKET_LOCK(b);
 		}
 		MPASS(b->b_deferred == 0);
@@ -1780,7 +1777,6 @@
 	st->state_flags |= PFSTATE_ACK;
 
 	pd->pd_sc = sc;
-	pd->pd_refs = 0;
 	pd->pd_st = st;
 	pf_ref_state(st);
 	pd->pd_m = m;
@@ -1831,21 +1827,21 @@
 
 	PFSYNC_BUCKET_LOCK_ASSERT(b);
 
+	TAILQ_REMOVE(&b->b_deferrals, pd, pd_entry);
+	b->b_deferred--;
+	pd->pd_st->state_flags &= ~PFSTATE_ACK;	/* XXX: locking! */
+	PFSYNC_BUCKET_UNLOCK(b);
+	free(pd, M_PFSYNC);
+
 	if (sc->sc_sync_if == NULL) {
-		PFSYNC_BUCKET_UNLOCK(b);
+		pf_release_state(st);
+		m_freem(m);
 		return;
 	}
 
 	NET_EPOCH_ENTER(et);
 	CURVNET_SET(sc->sc_sync_if->if_vnet);
 
-	TAILQ_REMOVE(&b->b_deferrals, pd, pd_entry);
-	b->b_deferred--;
-	pd->pd_st->state_flags &= ~PFSTATE_ACK;	/* XXX: locking! */
-	if (pd->pd_refs == 0)
-		free(pd, M_PFSYNC);
-	PFSYNC_BUCKET_UNLOCK(b);
-
 	pfsync_tx(sc, m);
 
 	pf_release_state(st);