diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -1280,6 +1280,8 @@
 			printf("kldunload: attempt to unload file that was"
 			    " loaded by the kernel\n");
 			error = EBUSY;
+		} else if (lf->refs > 1) {
+			error = EBUSY;
 		} else {
 			lf->userrefs--;
 			error = linker_file_unload(lf, flags);