diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk
--- a/sys/conf/kern.mk
+++ b/sys/conf/kern.mk
@@ -410,3 +410,10 @@
 LD_EMULATION_powerpc64le= elf64lppc_fbsd
 LD_EMULATION_riscv64= elf64lriscv
 LD_EMULATION=${LD_EMULATION_${MACHINE_ARCH}}
+
+_kldxref: .PHONY
+	${KLDXREF_CMD} ${DESTDIR}${KMODDIR}
+.if defined(NO_ROOT) && defined(METALOG)
+	echo ".${DISTBASE}${KMODDIR}/linker.hints type=file uname=root gname=wheel mode=0644" | \
+	    cat -l >> ${METALOG}
+.endif
diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk
--- a/sys/conf/kern.post.mk
+++ b/sys/conf/kern.post.mk
@@ -154,6 +154,14 @@
 
 .ORDER: kernel-install modules-install
 
+# Calling kldxref(8) for each module is expensive and, in the case of a
+# parallel install, risky if it manages to catch a module in the middle
+# of being installed.
+MKMODULESENV+=	NO_XREF=1
+.ORDER:	modules-install _kldxref
+install: _kldxref
+KMODDIR=	${KODIR}
+
 beforebuild: .PHONY
 kernel-all: beforebuild .WAIT ${KERNEL_KO} ${KERNEL_EXTRA}
 
diff --git a/sys/conf/kmod.mk b/sys/conf/kmod.mk
--- a/sys/conf/kmod.mk
+++ b/sys/conf/kmod.mk
@@ -382,12 +382,6 @@
 afterinstall: _kldxref
 .ORDER: realinstall _kldxref
 .ORDER: _installlinks _kldxref
-_kldxref: .PHONY
-	${KLDXREF_CMD} ${DESTDIR}${KMODDIR}
-.if defined(NO_ROOT) && defined(METALOG)
-	echo ".${DISTBASE}${KMODDIR}/linker.hints type=file uname=root gname=wheel mode=0644" | \
-	    cat -l >> ${METALOG}
-.endif
 .endif
 .endif # !target(realinstall)
 
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -965,17 +965,6 @@
 
 .endif # MODULES_OVERRIDE -- Keep last
 
-# Calling kldxref(8) for each module is expensive.
-.if !defined(NO_XREF)
-.MAKEFLAGS+=	-DNO_XREF
-afterinstall: .PHONY
-	${KLDXREF_CMD} ${DESTDIR}${KMODDIR}
-.if defined(NO_ROOT) && defined(METALOG)
-	echo ".${DISTBASE}${KMODDIR}/linker.hints type=file uname=root gname=wheel mode=0644" | \
-	    cat -l >> ${METALOG}
-.endif
-.endif
-
 SUBDIR:= ${SUBDIR:u:O}
 
 .include <bsd.subdir.mk>