diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -106,7 +106,8 @@
 linker_file_t linker_kernel_file;
 
 static struct sx kld_sx;	/* kernel linker lock */
-static bool kld_busy;
+static u_int kld_busy;
+static struct thread *kld_busy_owner;
 
 /*
  * Load counter used by clients to determine if a linker file has been
@@ -1065,7 +1066,9 @@
 
 	if ((flags & LINKER_UB_LOCKED) == 0)
 		sx_xlock(&kld_sx);
-	while (kld_busy) {
+	while (kld_busy > 0) {
+		if (kld_busy_owner == curthread)
+			break;
 		error = sx_sleep(&kld_busy, &kld_sx,
 		    (flags & LINKER_UB_PCATCH) != 0 ? PCATCH : 0,
 		    "kldbusy", 0);
@@ -1075,7 +1078,8 @@
 			return (error);
 		}
 	}
-	kld_busy = true;
+	kld_busy++;
+	kld_busy_owner = curthread;
 	if ((flags & LINKER_UB_UNLOCK) != 0)
 		sx_xunlock(&kld_sx);
 	return (0);
@@ -1090,9 +1094,15 @@
 
 	if ((flags & LINKER_UB_LOCKED) == 0)
 		sx_xlock(&kld_sx);
-	MPASS(kld_busy);
-	kld_busy = false;
-	wakeup(&kld_busy);
+	MPASS(kld_busy > 0);
+	if (kld_busy_owner != curthread)
+		panic("linker_kldload_unbusy done by not owning thread %p",
+		    kld_busy_owner);
+	kld_busy--;
+	if (kld_busy == 0) {
+		kld_busy_owner = NULL;
+		wakeup(&kld_busy);
+	}
 	sx_xunlock(&kld_sx);
 }