Differential D25510 Diff 74459 head/sys/x86/x86/mp_x86.c

Changeset View

Standalone View

head/sys/x86/x86/mp_x86.c

Show First 20 Lines • Show All 1,587 Lines • ▼ Show 20 Lines	#endif
lapic_setup(0);		lapic_setup(0);

/* Indicate that we are resumed */		/* Indicate that we are resumed */
CPU_CLR_ATOMIC(cpu, &resuming_cpus);		CPU_CLR_ATOMIC(cpu, &resuming_cpus);
CPU_CLR_ATOMIC(cpu, &suspended_cpus);		CPU_CLR_ATOMIC(cpu, &suspended_cpus);
CPU_CLR_ATOMIC(cpu, &toresume_cpus);		CPU_CLR_ATOMIC(cpu, &toresume_cpus);
}		}


void
invlcache_handler(void)
{
uint32_t generation;

#ifdef COUNT_IPIS
(*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */

/*		/*
* Reading the generation here allows greater parallelism
* since wbinvd is a serializing instruction. Without the
* temporary, we'd wait for wbinvd to complete, then the read
* would execute, then the dependent write, which must then
* complete before return from interrupt.
*/
generation = smp_tlb_generation;
wbinvd();
PCPU_SET(smp_tlb_done, generation);
}

/*
* This is called once the rest of the system is up and running and we're		* This is called once the rest of the system is up and running and we're
* ready to let the AP's out of the pen.		* ready to let the AP's out of the pen.
*/		*/
static void		static void
release_aps(void *dummy __unused)		release_aps(void *dummy __unused)
{		{

if (mp_ncpus == 1)		if (mp_ncpus == 1)
Show All 30 Lines	CPU_FOREACH(i) {
snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i);		snprintf(buf, sizeof(buf), "cpu%d:rendezvous", i);
intrcnt_add(buf, &ipi_rendezvous_counts[i]);		intrcnt_add(buf, &ipi_rendezvous_counts[i]);
snprintf(buf, sizeof(buf), "cpu%d:hardclock", i);		snprintf(buf, sizeof(buf), "cpu%d:hardclock", i);
intrcnt_add(buf, &ipi_hardclock_counts[i]);		intrcnt_add(buf, &ipi_hardclock_counts[i]);
}		}
}		}
SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL);		SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL);
#endif		#endif

/*
* Flush the TLB on other CPU's
*/

/* Variables needed for SMP tlb shootdown. */
vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
pmap_t smp_tlb_pmap;
volatile uint32_t smp_tlb_generation;

#ifdef __amd64__
#define read_eflags() read_rflags()
#endif

/*
* Used by pmap to request invalidation of TLB or cache on local and
* remote processors. Mask provides the set of remote CPUs which are
* to be signalled with the IPI specified by vector. The curcpu_cb
* callback is invoked on the calling CPU while waiting for remote
* CPUs to complete the operation.
*
* The callback function is called unconditionally on the caller's
* underlying processor, even when this processor is not set in the
* mask. So, the callback function must be prepared to handle such
* spurious invocations.
*/
static void
smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,
vm_offset_t addr1, vm_offset_t addr2, smp_invl_cb_t curcpu_cb)
{
cpuset_t other_cpus;
volatile uint32_t *p_cpudone;
uint32_t generation;
int cpu;

/*
* It is not necessary to signal other CPUs while booting or
* when in the debugger.
*/
if (kdb_active \|\| KERNEL_PANICKED() \|\| !smp_started) {
curcpu_cb(pmap, addr1, addr2);
return;
}

sched_pin();

/*
* Check for other cpus. Return if none.
*/
if (CPU_ISFULLSET(&mask)) {
if (mp_ncpus <= 1)
goto nospinexit;
} else {
CPU_CLR(PCPU_GET(cpuid), &mask);
if (CPU_EMPTY(&mask))
goto nospinexit;
}

if (!(read_eflags() & PSL_I))
panic("%s: interrupts disabled", __func__);
mtx_lock_spin(&smp_ipi_mtx);
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
smp_tlb_pmap = pmap;
generation = ++smp_tlb_generation;
if (CPU_ISFULLSET(&mask)) {
ipi_all_but_self(vector);
other_cpus = all_cpus;
CPU_CLR(PCPU_GET(cpuid), &other_cpus);
} else {
other_cpus = mask;
while ((cpu = CPU_FFS(&mask)) != 0) {
cpu--;
CPU_CLR(cpu, &mask);
CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__,
cpu, vector);
ipi_send_cpu(cpu, vector);
}
}
curcpu_cb(pmap, addr1, addr2);
while ((cpu = CPU_FFS(&other_cpus)) != 0) {
cpu--;
CPU_CLR(cpu, &other_cpus);
p_cpudone = &cpuid_to_pcpu[cpu]->pc_smp_tlb_done;
while (*p_cpudone != generation)
ia32_pause();
}
mtx_unlock_spin(&smp_ipi_mtx);
sched_unpin();
return;

nospinexit:
curcpu_cb(pmap, addr1, addr2);
sched_unpin();
}

void
smp_masked_invltlb(cpuset_t mask, pmap_t pmap, smp_invl_cb_t curcpu_cb)
{

smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0, curcpu_cb);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
#endif
}

void
smp_masked_invlpg(cpuset_t mask, vm_offset_t addr, pmap_t pmap,
smp_invl_cb_t curcpu_cb)
{

smp_targeted_tlb_shootdown(mask, IPI_INVLPG, pmap, addr, 0, curcpu_cb);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
#endif
}

void
smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2,
pmap_t pmap, smp_invl_cb_t curcpu_cb)
{

smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, pmap, addr1, addr2,
curcpu_cb);
#ifdef COUNT_XINVLTLB_HITS
ipi_range++;
ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
#endif
}

void
smp_cache_flush(smp_invl_cb_t curcpu_cb)
{

smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL, 0, 0,
curcpu_cb);
}

/*
* Handlers for TLB related IPIs
*/
void
invltlb_handler(void)
{
uint32_t generation;

#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */

/*
* Reading the generation here allows greater parallelism
* since invalidating the TLB is a serializing operation.
*/
generation = smp_tlb_generation;
if (smp_tlb_pmap == kernel_pmap)
invltlb_glob();
#ifdef __amd64__
else
invltlb();
#endif
PCPU_SET(smp_tlb_done, generation);
}

void
invlpg_handler(void)
{
uint32_t generation;

#ifdef COUNT_XINVLTLB_HITS
xhits_pg[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */

generation = smp_tlb_generation; /* Overlap with serialization */
#ifdef __i386__
if (smp_tlb_pmap == kernel_pmap)
#endif
invlpg(smp_tlb_addr1);
PCPU_SET(smp_tlb_done, generation);
}

void
invlrng_handler(void)
{
vm_offset_t addr, addr2;
uint32_t generation;

#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */

addr = smp_tlb_addr1;
addr2 = smp_tlb_addr2;
generation = smp_tlb_generation; /* Overlap with serialization */
#ifdef __i386__
if (smp_tlb_pmap == kernel_pmap)
#endif
do {
invlpg(addr);
addr += PAGE_SIZE;
} while (addr < addr2);

PCPU_SET(smp_tlb_done, generation);
}