diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
--- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -36,6 +36,7 @@
 #include <machine/atomic.h>
 
 #include <net/debugnet.h>
+#include <netinet/tcp_ratelimit.h>
 #include <netipsec/keydb.h>
 #include <netipsec/ipsec_offload.h>
 
@@ -4876,7 +4877,12 @@
 
 #ifdef RATELIMIT
 	/*
-	 * The kernel can have reference(s) via the m_snd_tag's into
+	 * Tell the TCP ratelimit code to release the rate-sets attached
+	 * to our ifnet.
+	 */
+	tcp_rl_release_ifnet(ifp);
+	/*
+	 * The kernel can still have reference(s) via the m_snd_tag's into
 	 * the ratelimit channels, and these must go away before
 	 * detaching:
 	 */
diff --git a/sys/netinet/tcp_ratelimit.h b/sys/netinet/tcp_ratelimit.h
--- a/sys/netinet/tcp_ratelimit.h
+++ b/sys/netinet/tcp_ratelimit.h
@@ -94,6 +94,8 @@
 #ifndef ETHERNET_SEGMENT_SIZE
 #define ETHERNET_SEGMENT_SIZE 1514
 #endif
+struct tcpcb;
+
 #ifdef RATELIMIT
 #define DETAILED_RATELIMIT_SYSCTL 1	/*
 					 * Undefine this if you don't want
@@ -131,6 +133,9 @@
 void
 tcp_rl_log_enobuf(const struct tcp_hwrate_limit_table *rte);
 
+void
+tcp_rl_release_ifnet(struct ifnet *ifp);
+
 #else
 static inline const struct tcp_hwrate_limit_table *
 tcp_set_pacing_rate(struct tcpcb *tp, struct ifnet *ifp,
@@ -218,6 +223,10 @@
 {
 }
 
+static inline void
+tcp_rl_release_ifnet(struct ifnet *ifp)
+{
+}
 #endif
 
 /*
diff --git a/sys/netinet/tcp_ratelimit.c b/sys/netinet/tcp_ratelimit.c
--- a/sys/netinet/tcp_ratelimit.c
+++ b/sys/netinet/tcp_ratelimit.c
@@ -1298,6 +1298,12 @@
 	NET_EPOCH_EXIT(et);
 }
 
+void
+tcp_rl_release_ifnet(struct ifnet *ifp)
+{
+	tcp_rl_ifnet_departure(NULL, ifp);
+}
+
 static void
 tcp_rl_shutdown(void *arg __unused, int howto __unused)
 {