Page MenuHomeFreeBSD
Differential D9665 Diff 25723 head/sys/i386/i386/pmap.c

Changeset View

Standalone View

View Options

head/sys/i386/i386/pmap.c

Show First 20 Lines • Show All 295 Lines • ▼ Show 20 Lines
static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot); vm_prot_t prot);
static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, vm_page_t mpte); vm_page_t m, vm_prot_t prot, vm_page_t mpte);
static void pmap_flush_page(vm_page_t m); static void pmap_flush_page(vm_page_t m);
static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
pd_entry_t pde);
static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
static boolean_t pmap_is_modified_pvh(struct md_page *pvh); static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
static boolean_t pmap_is_referenced_pvh(struct md_page *pvh); static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde); static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits); static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
▲ Show 20 Lines • Show All 942 Lines • ▼ Show 20 Lines if (pmap == kernel_pmap)
pmap_kenter_pde(va, newpde); pmap_kenter_pde(va, newpde);
else else
pde_store(pde, newpde); pde_store(pde, newpde);
if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
pmap_update_pde_invalidate(va, newpde); pmap_update_pde_invalidate(va, newpde);
} }
#endif /* !SMP */ #endif /* !SMP */
static void
pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
{
/*
* When the PDE has PG_PROMOTED set, the 2- or 4MB page mapping was
* created by a promotion that did not invalidate the 512 or 1024 4KB
* page mappings that might exist in the TLB. Consequently, at this
* point, the TLB may hold both 4KB and 2- or 4MB page mappings for
* the address range [va, va + NBPDR). Therefore, the entire range
* must be invalidated here. In contrast, when PG_PROMOTED is clear,
* the TLB will not hold any 4KB page mappings for the address range
* [va, va + NBPDR), and so a single INVLPG suffices to invalidate the
* 2- or 4MB page mapping from the TLB.
*/
if ((pde & PG_PROMOTED) != 0)
pmap_invalidate_range(pmap, va, va + NBPDR - 1);
else
pmap_invalidate_page(pmap, va);
}
#define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024)
void void
pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force) pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)
{ {
if (force) { if (force) {
sva &= ~(vm_offset_t)cpu_clflush_line_size; sva &= ~(vm_offset_t)cpu_clflush_line_size;
▲ Show 20 Lines • Show All 1,374 Lines • ▼ Show 20 Lines if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
* allocation of the new page table page fails. * allocation of the new page table page fails.
*/ */
if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL |
VM_ALLOC_WIRED)) == NULL) { VM_ALLOC_WIRED)) == NULL) {
SLIST_INIT(&free); SLIST_INIT(&free);
sva = trunc_4mpage(va); sva = trunc_4mpage(va);
pmap_remove_pde(pmap, pde, sva, &free); pmap_remove_pde(pmap, pde, sva, &free);
pmap_invalidate_range(pmap, sva, sva + NBPDR - 1); if ((oldpde & PG_G) == 0)
pmap_invalidate_pde_page(pmap, sva, oldpde);
pmap_free_zero_pages(&free); pmap_free_zero_pages(&free);
CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x" CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
" in pmap %p", va, pmap); " in pmap %p", va, pmap);
return (FALSE); return (FALSE);
} }
if (va < VM_MAXUSER_ADDRESS) if (va < VM_MAXUSER_ADDRESS)
pmap->pm_stats.resident_count++; pmap->pm_stats.resident_count++;
} }
▲ Show 20 Lines • Show All 153 Lines • ▼ Show 20 Lines KASSERT((sva & PDRMASK) == 0,
("pmap_remove_pde: sva is not 4mpage aligned")); ("pmap_remove_pde: sva is not 4mpage aligned"));
oldpde = pte_load_clear(pdq); oldpde = pte_load_clear(pdq);
if (oldpde & PG_W) if (oldpde & PG_W)
pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
/* /*
* Machines that don't support invlpg, also don't support * Machines that don't support invlpg, also don't support
* PG_G. * PG_G.
*
* When workaround_erratum383 is false, a promotion to a 2M/4M
* page mapping does not invalidate the 512/1024 4K page mappings
* from the TLB. Consequently, at this point, the TLB may
* hold both 4K and 2M/4M page mappings. Therefore, the entire
* range of addresses must be invalidated here. In contrast,
* when workaround_erratum383 is true, a promotion does
* invalidate the 512/1024 4K page mappings, and so a single INVLPG
* suffices to invalidate the 2M/4M page mapping.
*/ */
if ((oldpde & PG_G) != 0) { if ((oldpde & PG_G) != 0)
if (workaround_erratum383) pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
pmap_invalidate_page(kernel_pmap, sva);
else
pmap_invalidate_range(kernel_pmap, sva,
sva + NBPDR - 1);
}
pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
if (oldpde & PG_MANAGED) { if (oldpde & PG_MANAGED) {
pvh = pa_to_pvh(oldpde & PG_PS_FRAME); pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
pmap_pvh_free(pvh, pmap, sva); pmap_pvh_free(pvh, pmap, sva);
eva = sva + NBPDR; eva = sva + NBPDR;
for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
va < eva; va += PAGE_SIZE, m++) { va < eva; va += PAGE_SIZE, m++) {
▲ Show 20 Lines • Show All 291 Lines • ▼ Show 20 Linesretry:
} }
if ((prot & VM_PROT_WRITE) == 0) if ((prot & VM_PROT_WRITE) == 0)
newpde &= ~(PG_RW | PG_M); newpde &= ~(PG_RW | PG_M);
#if defined(PAE) || defined(PAE_TABLES) #if defined(PAE) || defined(PAE_TABLES)
if ((prot & VM_PROT_EXECUTE) == 0) if ((prot & VM_PROT_EXECUTE) == 0)
newpde |= pg_nx; newpde |= pg_nx;
#endif #endif
if (newpde != oldpde) { if (newpde != oldpde) {
if (!pde_cmpset(pde, oldpde, newpde)) /*
* As an optimization to future operations on this PDE, clear
* PG_PROMOTED. The impending invalidation will remove any
* lingering 4KB page mappings from the TLB.
*/
if (!pde_cmpset(pde, oldpde, newpde & ~PG_PROMOTED))
goto retry; goto retry;
if (oldpde & PG_G) { if ((oldpde & PG_G) != 0)
/* See pmap_remove_pde() for explanation. */ pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
if (workaround_erratum383)
pmap_invalidate_page(kernel_pmap, sva);
else else
pmap_invalidate_range(kernel_pmap, sva,
sva + NBPDR - 1);
} else
anychanged = TRUE; anychanged = TRUE;
} }
return (anychanged); return (anychanged);
} }
/* /*
* Set the physical protection on the * Set the physical protection on the
* specified range of this map as requested. * specified range of this map as requested.
▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines if ((newpde & PG_PTE_PAT) != 0)
newpde ^= PG_PDE_PAT | PG_PTE_PAT; newpde ^= PG_PDE_PAT | PG_PTE_PAT;
/* /*
* Map the superpage. * Map the superpage.
*/ */
if (workaround_erratum383) if (workaround_erratum383)
pmap_update_pde(pmap, va, pde, PG_PS | newpde); pmap_update_pde(pmap, va, pde, PG_PS | newpde);
else if (pmap == kernel_pmap) else if (pmap == kernel_pmap)
pmap_kenter_pde(va, PG_PS | newpde); pmap_kenter_pde(va, PG_PROMOTED | PG_PS | newpde);
else else
pde_store(pde, PG_PS | newpde); pde_store(pde, PG_PROMOTED | PG_PS | newpde);
pmap_pde_promotions++; pmap_pde_promotions++;
CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x" CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x"
" in pmap %p", va, pmap); " in pmap %p", va, pmap);
} }
/* /*
* Insert the given physical page (p) at * Insert the given physical page (p) at
▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines if (va < VM_MAXUSER_ADDRESS)
newpde |= PG_U; newpde |= PG_U;
/* /*
* Increment counters. * Increment counters.
*/ */
pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE; pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
/* /*
* Map the superpage. * Map the superpage. (This is not a promoted mapping; there will not
* be any lingering 4KB page mappings in the TLB.)
*/ */
pde_store(pde, newpde); pde_store(pde, newpde);
pmap_pde_mappings++; pmap_pde_mappings++;
CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
" in pmap %p", va, pmap); " in pmap %p", va, pmap);
return (TRUE); return (TRUE);
} }
▲ Show 20 Lines • Show All 1,942 Lines • Show Last 20 Lines