diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -854,6 +854,7 @@
 static int release_clip_addr(struct adapter *, struct t4_clip_addr *);
 #ifdef TCP_OFFLOAD
 static int toe_capability(struct vi_info *, bool);
+static int t4_deactivate_all_uld(struct adapter *);
 static void t4_async_event(struct adapter *);
 #endif
 #ifdef KERN_TLS
@@ -1686,6 +1687,15 @@
 
 	sc = device_get_softc(dev);
 
+#ifdef TCP_OFFLOAD
+	rc = t4_deactivate_all_uld(sc);
+	if (rc) {
+		device_printf(dev,
+		    "failed to detach upper layer drivers: %d\n", rc);
+		return (rc);
+	}
+#endif
+
 	if (sc->cdev) {
 		destroy_dev(sc->cdev);
 		sc->cdev = NULL;
@@ -12591,6 +12601,34 @@
 	return (rc);
 }
 
+static int
+t4_deactivate_all_uld(struct adapter *sc)
+{
+	int rc;
+	struct uld_info *ui;
+
+	rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4detuld");
+	if (rc != 0)
+		return (ENXIO);
+
+	sx_slock(&t4_uld_list_lock);
+
+	SLIST_FOREACH(ui, &t4_uld_list, link) {
+		if (isset(&sc->active_ulds, ui->uld_id)) {
+			rc = ui->deactivate(sc);
+			if (rc != 0)
+				break;
+			clrbit(&sc->active_ulds, ui->uld_id);
+			ui->refcount--;
+		}
+	}
+
+	sx_sunlock(&t4_uld_list_lock);
+	end_synchronized_op(sc, 0);
+
+	return (rc);
+}
+
 static void
 t4_async_event(struct adapter *sc)
 {