Index: sys/x86/x86/mp_x86.c
===================================================================
--- sys/x86/x86/mp_x86.c
+++ sys/x86/x86/mp_x86.c
@@ -1425,16 +1425,33 @@
 #else
 		npxsuspend(susppcbs[cpu]->sp_fpususpend);
 #endif
-		wbinvd();
-		CPU_SET_ATOMIC(cpu, &suspended_cpus);
 		/*
-		 * Hack for xen, which does not use resumectx() so never
-		 * uses the next clause: set resuming_cpus early so that
-		 * resume_cpus() can wait on the same bitmap for acpi and
-		 * xen.  resuming_cpus now means eventually_resumable_cpus.
+		 * suspended_cpus gets cleared shortly after an AP is restarted
+		 * via Startup IPI, so that the BSP could proceed to restarting
+		 * the next AP.
+		 * resuming_cpus gets cleared when the AP completes
+		 * initialization after having been released by the BSP.
+		 *
+		 * Note that suspended_cpus is meaningful only for ACPI
+		 * suspend, it's not really used for Xen suspend as the APs are
+		 * automatically restored to the running state and the correct
+		 * context.  For the same reason resumectx is never called in
+		 * that case.
 		 */
+		CPU_SET_ATOMIC(cpu, &suspended_cpus);
 		CPU_SET_ATOMIC(cpu, &resuming_cpus);
+
+		/*
+		 * Invalidate the cache after setting the global status bits.
+		 * The last AP to set its bit may end up being an Owner of
+		 * the corresponding cache line in MOESI protocol.
+		 * The AP may be stopped before the cache line is written
+		 * to the main memory.
+		 */
+		wbinvd();
 	} else {
+		/* Indicate that we are resuming */
+		CPU_CLR_ATOMIC(cpu, &suspended_cpus);
 #ifdef __amd64__
 		fpuresume(susppcbs[cpu]->sp_fpususpend);
 #else
@@ -1444,9 +1461,6 @@
 		initializecpu();
 		PCPU_SET(switchtime, 0);
 		PCPU_SET(switchticks, ticks);
-
-		/* Indicate that we are resuming */
-		CPU_CLR_ATOMIC(cpu, &suspended_cpus);
 	}
 
 	/* Wait for resume directive */