Page MenuHomeFreeBSD
Differential D36919 Diff 111621 sys/amd64/amd64/mp_machdep.c

Changeset View

Standalone View

View Options

sys/amd64/amd64/mp_machdep.c

Show First 20 Lines • Show All 897 Lines • ▼ Show 20 Lines if (smp_tlb_pmap == PCPU_GET(curpmap) &&
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1); pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1);
} }
} }
static void static void
invlrng_handler(vm_offset_t smp_tlb_addr1, vm_offset_t smp_tlb_addr2) invlrng_handler(vm_offset_t smp_tlb_addr1, vm_offset_t smp_tlb_addr2)
{ {
vm_offset_t addr, addr2; vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS #ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++; xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */ #endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS #ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++; (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */ #endif /* COUNT_IPIS */
addr = smp_tlb_addr1; addr = smp_tlb_addr1;
addr2 = smp_tlb_addr2;
do { do {
invlpg(addr); invlpg(addr);
addr += PAGE_SIZE; addr += PAGE_SIZE;
} while (addr < addr2); } while (addr < smp_tlb_addr2);
} }
static void static void
invlrng_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1, invlrng_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
vm_offset_t smp_tlb_addr2) vm_offset_t smp_tlb_addr2)
{ {
struct invpcid_descr d; struct invpcid_descr d;
vm_offset_t addr, addr2; vm_offset_t addr;
#ifdef COUNT_XINVLTLB_HITS #ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++; xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */ #endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS #ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++; (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */ #endif /* COUNT_IPIS */
addr = smp_tlb_addr1; addr = smp_tlb_addr1;
addr2 = smp_tlb_addr2;
do { do {
invlpg(addr); invlpg(addr);
addr += PAGE_SIZE; addr += PAGE_SIZE;
} while (addr < addr2); } while (addr < smp_tlb_addr2);
if (smp_tlb_pmap == PCPU_GET(curpmap) && if (smp_tlb_pmap == PCPU_GET(curpmap) &&
smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 && smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid | d.pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid |
PMAP_PCID_USER_PT; PMAP_PCID_USER_PT;
d.pad = 0; d.pad = 0;
d.addr = smp_tlb_addr1; d.addr = smp_tlb_addr1;
do { do {
invpcid(&d, INVPCID_ADDR); invpcid(&d, INVPCID_ADDR);
d.addr += PAGE_SIZE; d.addr += PAGE_SIZE;
} while (d.addr < addr2); } while (d.addr < smp_tlb_addr2);
} }
} }
static void static void
invlrng_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1, invlrng_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
vm_offset_t smp_tlb_addr2) vm_offset_t smp_tlb_addr2)
{ {
vm_offset_t addr, addr2; vm_offset_t addr;
uint64_t kcr3, ucr3; uint64_t kcr3, ucr3;
uint32_t pcid; uint32_t pcid;
#ifdef COUNT_XINVLTLB_HITS #ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++; xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */ #endif /* COUNT_XINVLTLB_HITS */
#ifdef COUNT_IPIS #ifdef COUNT_IPIS
(*ipi_invlrng_counts[PCPU_GET(cpuid)])++; (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */ #endif /* COUNT_IPIS */
addr = smp_tlb_addr1; addr = smp_tlb_addr1;
addr2 = smp_tlb_addr2;
do { do {
invlpg(addr); invlpg(addr);
addr += PAGE_SIZE; addr += PAGE_SIZE;
} while (addr < addr2); } while (addr < smp_tlb_addr2);
if (smp_tlb_pmap == PCPU_GET(curpmap) && if (smp_tlb_pmap == PCPU_GET(curpmap) &&
(ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 && (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid;
kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, addr2); pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2);
} }
} }
static void static void
invlcache_handler(void) invlcache_handler(void)
{ {
#ifdef COUNT_IPIS #ifdef COUNT_IPIS
(*ipi_invlcache_counts[PCPU_GET(cpuid)])++; (*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
▲ Show 20 Lines • Show All 106 Lines • Show Last 20 Lines