Index: UPDATING
===================================================================
--- UPDATING
+++ UPDATING
@@ -26,6 +26,11 @@
 	disable the most expensive debugging functionality run
 	"ln -s 'abort:false,junk:false' /etc/malloc.conf".)
 
+202001xx:
+	The size of struct vnet and the magic cookie have changed.
+	Users need to recompile modules using VIMAGE together with kernel
+	as well as libkvm.
+
 20200108:
 	Clang/LLVM is now the default compiler and LLD the default
 	linker for riscv64.
Index: lib/libkvm/kvm.c
===================================================================
--- lib/libkvm/kvm.c
+++ lib/libkvm/kvm.c
@@ -48,6 +48,7 @@
 #include <sys/stat.h>
 #include <sys/mman.h>
 
+#include <stdbool.h>
 #include <net/vnet.h>
 
 #include <fcntl.h>
Index: lib/libkvm/kvm_private.c
===================================================================
--- lib/libkvm/kvm_private.c
+++ lib/libkvm/kvm_private.c
@@ -45,6 +45,7 @@
 #include <sys/stat.h>
 #include <sys/mman.h>
 
+#include <stdbool.h>
 #include <net/vnet.h>
 
 #include <assert.h>
Index: lib/libkvm/kvm_vnet.c
===================================================================
--- lib/libkvm/kvm_vnet.c
+++ lib/libkvm/kvm_vnet.c
@@ -43,6 +43,7 @@
 #include <sys/proc.h>
 #include <sys/types.h>
 
+#include <stdbool.h>
 #include <net/vnet.h>
 
 #include <kvm.h>
Index: sys/net/if.c
===================================================================
--- sys/net/if.c
+++ sys/net/if.c
@@ -1078,7 +1078,8 @@
 #ifdef VIMAGE
 	bool shutdown;
 
-	shutdown = ifp->if_vnet->vnet_shutdown;
+	shutdown = (ifp->if_vnet->vnet_shutdown &&
+	    ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE);
 #endif
 	IFNET_WLOCK();
 	CK_STAILQ_FOREACH(iter, &V_ifnet, if_link)
@@ -1337,6 +1338,7 @@
 	struct prison *pr;
 	struct ifnet *difp;
 	int error;
+	bool shutdown;
 
 	/* Try to find the prison within our visibility. */
 	sx_slock(&allprison_lock);
@@ -1364,7 +1366,9 @@
 	}
 
 	/* Make sure the VNET is stable. */
-	if (ifp->if_vnet->vnet_shutdown) {
+	shutdown = (ifp->if_vnet->vnet_shutdown &&
+	    ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE);
+	if (shutdown) {
 		CURVNET_RESTORE();
 		prison_free(pr);
 		return (EBUSY);
@@ -1389,6 +1393,7 @@
 	struct vnet *vnet_dst;
 	struct ifnet *ifp;
 	int error;
+ 	bool shutdown;
 
 	/* Try to find the prison within our visibility. */
 	sx_slock(&allprison_lock);
@@ -1417,7 +1422,9 @@
 	}
 
 	/* Make sure the VNET is stable. */
-	if (ifp->if_vnet->vnet_shutdown) {
+	shutdown = (ifp->if_vnet->vnet_shutdown &&
+	    ifp->if_vnet->vnet_state < SI_SUB_VNET_DONE);
+	if (shutdown) {
 		CURVNET_RESTORE();
 		prison_free(pr);
 		return (EBUSY);
@@ -2948,11 +2955,16 @@
 	struct ifreq *ifr;
 	int error;
 	int oif_flags;
+#ifdef VIMAGE
+	bool shutdown;
+#endif
 
 	CURVNET_SET(so->so_vnet);
 #ifdef VIMAGE
 	/* Make sure the VNET is stable. */
-	if (so->so_vnet->vnet_shutdown) {
+	shutdown = (so->so_vnet->vnet_shutdown &&
+	    so->so_vnet->vnet_state < SI_SUB_VNET_DONE);
+	if (shutdown) {
 		CURVNET_RESTORE();
 		return (EBUSY);
 	}
Index: sys/net/vnet.h
===================================================================
--- sys/net/vnet.h
+++ sys/net/vnet.h
@@ -72,11 +72,12 @@
 	u_int			 vnet_magic_n;
 	u_int			 vnet_ifcnt;
 	u_int			 vnet_sockcnt;
-	u_int			 vnet_shutdown; /* Shutdown in progress. */
+	u_int			 vnet_state;	/* SI_SUB_* */
 	void			*vnet_data_mem;
 	uintptr_t		 vnet_data_base;
-};
-#define	VNET_MAGIC_N	0x3e0d8f29
+	bool			 vnet_shutdown;	/* Shutdown in progress. */
+} __aligned(CACHE_LINE_SIZE);
+#define	VNET_MAGIC_N	0x5e31a7bd
 
 /*
  * These two virtual network stack allocator definitions are also required
Index: sys/net/vnet.c
===================================================================
--- sys/net/vnet.c
+++ sys/net/vnet.c
@@ -279,6 +279,9 @@
 	LIST_REMOVE(vnet, vnet_le);
 	VNET_LIST_WUNLOCK();
 
+	/* Signal that VNET is being shutdown. */
+	vnet->vnet_shutdown = true;
+
 	CURVNET_SET_QUIET(vnet);
 	vnet_sysuninit();
 	CURVNET_RESTORE();
@@ -350,15 +353,15 @@
 }
 SYSINIT(vnet_data, SI_SUB_KLD, SI_ORDER_FIRST, vnet_data_startup, NULL);
 
+/* Dummy VNET_SYSINIT to make sure we always reach the final end state. */
 static void
-vnet_sysuninit_shutdown(void *unused __unused)
+vnet_sysinit_done(void *unused __unused)
 {
 
-	/* Signal that VNET is being shutdown. */
-	curvnet->vnet_shutdown = 1;
+	return;
 }
-VNET_SYSUNINIT(vnet_sysuninit_shutdown, SI_SUB_VNET_DONE, SI_ORDER_FIRST,
-    vnet_sysuninit_shutdown, NULL);
+VNET_SYSINIT(vnet_sysinit_done, SI_SUB_VNET_DONE, SI_ORDER_ANY,
+    vnet_sysinit_done, NULL);
 
 /*
  * When a module is loaded and requires storage for a virtualized global
@@ -572,8 +575,10 @@
 	struct vnet_sysinit *vs;
 
 	VNET_SYSINIT_RLOCK();
-	TAILQ_FOREACH(vs, &vnet_constructors, link)
+	TAILQ_FOREACH(vs, &vnet_constructors, link) {
+		curvnet->vnet_state = vs->subsystem;
 		vs->func(vs->arg);
+	}
 	VNET_SYSINIT_RUNLOCK();
 }
 
@@ -589,8 +594,10 @@
 
 	VNET_SYSINIT_RLOCK();
 	TAILQ_FOREACH_REVERSE(vs, &vnet_destructors, vnet_sysuninit_head,
-	    link)
+	    link) {
+		curvnet->vnet_state = vs->subsystem;
 		vs->func(vs->arg);
+	}
 	VNET_SYSINIT_RUNLOCK();
 }
 
@@ -704,7 +711,8 @@
 	db_printf(" vnet_data_mem  = %p\n", vnet->vnet_data_mem);
 	db_printf(" vnet_data_base = %#jx\n",
 	    (uintmax_t)vnet->vnet_data_base);
-	db_printf(" vnet_shutdown  = %#08x\n", vnet->vnet_shutdown);
+	db_printf(" vnet_state     = %#08x\n", vnet->vnet_state);
+	db_printf(" vnet_shutdown  = %#03x\n", vnet->vnet_shutdown);
 	db_printf("\n");
 }