diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -903,7 +903,7 @@ static void invlrng_handler(vm_offset_t smp_tlb_addr1, vm_offset_t smp_tlb_addr2) { - vm_offset_t addr, addr2; + vm_offset_t addr; #ifdef COUNT_XINVLTLB_HITS xhits_rng[PCPU_GET(cpuid)]++; @@ -913,11 +913,10 @@ #endif /* COUNT_IPIS */ addr = smp_tlb_addr1; - addr2 = smp_tlb_addr2; do { invlpg(addr); addr += PAGE_SIZE; - } while (addr < addr2); + } while (addr < smp_tlb_addr2); } static void @@ -925,7 +924,7 @@ vm_offset_t smp_tlb_addr2) { struct invpcid_descr d; - vm_offset_t addr, addr2; + vm_offset_t addr; #ifdef COUNT_XINVLTLB_HITS xhits_rng[PCPU_GET(cpuid)]++; @@ -935,11 +934,10 @@ #endif /* COUNT_IPIS */ addr = smp_tlb_addr1; - addr2 = smp_tlb_addr2; do { invlpg(addr); addr += PAGE_SIZE; - } while (addr < addr2); + } while (addr < smp_tlb_addr2); if (smp_tlb_pmap == PCPU_GET(curpmap) && smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 && PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { @@ -950,7 +948,7 @@ do { invpcid(&d, INVPCID_ADDR); d.addr += PAGE_SIZE; - } while (d.addr < addr2); + } while (d.addr < smp_tlb_addr2); } } @@ -958,7 +956,7 @@ invlrng_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1, vm_offset_t smp_tlb_addr2) { - vm_offset_t addr, addr2; + vm_offset_t addr; uint64_t kcr3, ucr3; uint32_t pcid; @@ -970,18 +968,17 @@ #endif /* COUNT_IPIS */ addr = smp_tlb_addr1; - addr2 = smp_tlb_addr2; do { invlpg(addr); addr += PAGE_SIZE; - } while (addr < addr2); + } while (addr < smp_tlb_addr2); if (smp_tlb_pmap == PCPU_GET(curpmap) && (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 && PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) { pcid = smp_tlb_pmap->pm_pcids[PCPU_GET(cpuid)].pm_pcid; kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE; ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; - pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, addr2); + pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2); } } diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -2903,8 +2903,16 @@ * page table, and INVPCID(INVPCID_CTXGLOB)/invltlb_glob() for a * user space page table(s). * - * If the INVPCID instruction is available, it is used to flush entries - * from the kernel page table. + * If the INVPCID instruction is available, it is used to flush user + * entries from the kernel page table. + * + * When PCID is enabled, the INVLPG instruction invalidates all TLB + * entries for the given page that either match the current PCID or + * are global. Since TLB entries for the same page under different + * PCIDs are unaffected, kernel pages which reside in all address + * spaces could be problematic. We avoid the problem by creating + * all kernel PTEs with the global flag (PG_G) set, when PTI is + * disabled. * * * mode: PTI disabled, PCID present. The kernel reserves PCID 0 for its * address space, all other 4095 PCIDs are used for user mode spaces