Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/mp_machdep.c
| Show First 20 Lines • Show All 612 Lines • ▼ Show 20 Lines | |||||
| * | * | ||||
| * Function must be called with the thread pinned, and it unpins on | * Function must be called with the thread pinned, and it unpins on | ||||
| * completion. | * completion. | ||||
| */ | */ | ||||
| static void | static void | ||||
| smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1, | smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1, | ||||
| vm_offset_t addr2, smp_invl_cb_t curcpu_cb, enum invl_op_codes op) | vm_offset_t addr2, smp_invl_cb_t curcpu_cb, enum invl_op_codes op) | ||||
| { | { | ||||
| cpuset_t other_cpus, mask1; | cpuset_t other_cpus; | ||||
| uint32_t generation, *p_cpudone; | uint32_t generation, *p_cpudone; | ||||
| int cpu; | int cpu; | ||||
| bool is_all; | bool is_all; | ||||
| /* | /* | ||||
| * It is not necessary to signal other CPUs while booting or | * It is not necessary to signal other CPUs while booting or | ||||
| * when in the debugger. | * when in the debugger. | ||||
| */ | */ | ||||
| Show All 27 Lines | smp_targeted_tlb_shootdown(cpuset_t mask, pmap_t pmap, vm_offset_t addr1, | ||||
| generation = PCPU_GET(smp_tlb_gen); | generation = PCPU_GET(smp_tlb_gen); | ||||
| if (++generation == 0) | if (++generation == 0) | ||||
| generation = 1; | generation = 1; | ||||
| PCPU_SET(smp_tlb_gen, generation); | PCPU_SET(smp_tlb_gen, generation); | ||||
| PCPU_SET(smp_tlb_op, op); | PCPU_SET(smp_tlb_op, op); | ||||
| /* Fence between filling smp_tlb fields and clearing scoreboard. */ | /* Fence between filling smp_tlb fields and clearing scoreboard. */ | ||||
| atomic_thread_fence_rel(); | atomic_thread_fence_rel(); | ||||
| mask1 = mask; | CPU_FOREACH_ISSET(cpu, &mask) { | ||||
| while ((cpu = CPU_FFS(&mask1)) != 0) { | |||||
| cpu--; | |||||
| CPU_CLR(cpu, &mask1); | |||||
| KASSERT(*invl_scoreboard_slot(cpu) != 0, | KASSERT(*invl_scoreboard_slot(cpu) != 0, | ||||
| ("IPI scoreboard is zero, initiator %d target %d", | ("IPI scoreboard is zero, initiator %d target %d", | ||||
| PCPU_GET(cpuid), cpu)); | PCPU_GET(cpuid), cpu)); | ||||
| *invl_scoreboard_slot(cpu) = 0; | *invl_scoreboard_slot(cpu) = 0; | ||||
| } | } | ||||
| /* | /* | ||||
| * IPI acts as a fence between writing to the scoreboard above | * IPI acts as a fence between writing to the scoreboard above | ||||
| * (zeroing slot) and reading from it below (wait for | * (zeroing slot) and reading from it below (wait for | ||||
| * acknowledgment). | * acknowledgment). | ||||
| */ | */ | ||||
| if (is_all) { | if (is_all) { | ||||
| ipi_all_but_self(IPI_INVLOP); | ipi_all_but_self(IPI_INVLOP); | ||||
| other_cpus = all_cpus; | other_cpus = all_cpus; | ||||
| CPU_CLR(PCPU_GET(cpuid), &other_cpus); | CPU_CLR(PCPU_GET(cpuid), &other_cpus); | ||||
| } else { | } else { | ||||
| other_cpus = mask; | other_cpus = mask; | ||||
| ipi_selected(mask, IPI_INVLOP); | ipi_selected(mask, IPI_INVLOP); | ||||
| } | } | ||||
| curcpu_cb(pmap, addr1, addr2); | curcpu_cb(pmap, addr1, addr2); | ||||
| while ((cpu = CPU_FFS(&other_cpus)) != 0) { | CPU_FOREACH_ISSET(cpu, &other_cpus) { | ||||
| cpu--; | |||||
| CPU_CLR(cpu, &other_cpus); | |||||
| p_cpudone = invl_scoreboard_slot(cpu); | p_cpudone = invl_scoreboard_slot(cpu); | ||||
| while (atomic_load_int(p_cpudone) != generation) | while (atomic_load_int(p_cpudone) != generation) | ||||
| ia32_pause(); | ia32_pause(); | ||||
| } | } | ||||
| /* | /* | ||||
| * Unpin before leaving critical section. If the thread owes | * Unpin before leaving critical section. If the thread owes | ||||
| * preemption, this allows scheduler to select thread on any | * preemption, this allows scheduler to select thread on any | ||||
| ▲ Show 20 Lines • Show All 409 Lines • Show Last 20 Lines | |||||