Page MenuHomeFreeBSD
Differential D26513 Diff 77392 head/sys/amd64/amd64/pmap.c

Changeset View

Standalone View

View Options

head/sys/amd64/amd64/pmap.c

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 6,473 Lines • ▼ Show 20 Lines
static int static int
pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags,
int psind) int psind)
{ {
vm_page_t mp; vm_page_t mp;
pt_entry_t origpte, *pml4e, *pdpe, *pde, pten, PG_V; pt_entry_t origpte, *pml4e, *pdpe, *pde, pten, PG_V;
PMAP_LOCK_ASSERT(pmap, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT(psind > 0 && psind < MAXPAGESIZES, KASSERT(psind > 0 && psind < MAXPAGESIZES && pagesizes[psind] != 0,
("psind %d unexpected", psind)); ("psind %d unexpected", psind));
KASSERT(((newpte & PG_FRAME) & (pagesizes[psind] - 1)) == 0, KASSERT(((newpte & PG_FRAME) & (pagesizes[psind] - 1)) == 0,
("unaligned phys address %#lx newpte %#lx psind %d", ("unaligned phys address %#lx newpte %#lx psind %d",
newpte & PG_FRAME, newpte, psind)); newpte & PG_FRAME, newpte, psind));
KASSERT((va & (pagesizes[psind] - 1)) == 0, KASSERT((va & (pagesizes[psind] - 1)) == 0,
("unaligned va %#lx psind %d", va, psind)); ("unaligned va %#lx psind %d", va, psind));
KASSERT(va < VM_MAXUSER_ADDRESS, KASSERT(va < VM_MAXUSER_ADDRESS,
("kernel mode non-transparent superpage")); /* XXXKIB */ ("kernel mode non-transparent superpage")); /* XXXKIB */
KASSERT(va + pagesizes[psind] < VM_MAXUSER_ADDRESS, KASSERT(va + pagesizes[psind] < VM_MAXUSER_ADDRESS,
("overflowing user map va %#lx psind %d", va, psind)); /* XXXKIB */ ("overflowing user map va %#lx psind %d", va, psind)); /* XXXKIB */
PG_V = pmap_valid_bit(pmap); PG_V = pmap_valid_bit(pmap);
restart: restart:
if (!pmap_pkru_same(pmap, va, va + pagesizes[psind]))
return (KERN_PROTECTION_FAILURE);
pten = newpte; pten = newpte;
if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86)
pten |= pmap_pkru_get(pmap, va); pten |= pmap_pkru_get(pmap, va);
if (psind == 2) { /* 1G */ if (psind == 2) { /* 1G */
if (!pmap_pkru_same(pmap, va, va + NBPDP))
return (KERN_PROTECTION_FAILURE);
pml4e = pmap_pml4e(pmap, va); pml4e = pmap_pml4e(pmap, va);
if (pml4e == NULL || (*pml4e & PG_V) == 0) { if (pml4e == NULL || (*pml4e & PG_V) == 0) {
mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va), mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va),
NULL, va); NULL, va);
if (mp == NULL) { if (mp == NULL)
if ((flags & PMAP_ENTER_NOSLEEP) != 0) goto allocf;
return (KERN_RESOURCE_SHORTAGE); pdpe = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp));
PMAP_UNLOCK(pmap); pdpe = &pdpe[pmap_pdpe_index(va)];
vm_wait(NULL);
PMAP_LOCK(pmap);
/*
* Restart at least to recalcuate the pkru
* key. Our caller must keep the map locked
* so no paging structure can be validated
* under us.
*/
goto restart;
}
pdpe = pmap_pdpe(pmap, va);
KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va));
origpte = *pdpe; origpte = *pdpe;
MPASS(origpte == 0); MPASS(origpte == 0);
} else { } else {
pdpe = pmap_pdpe(pmap, va); pdpe = pmap_pml4e_to_pdpe(pml4e, va);
KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va));
origpte = *pdpe; origpte = *pdpe;
if ((origpte & PG_V) == 0) { if ((origpte & PG_V) == 0) {
mp = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME); mp = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME);
mp->ref_count++; mp->ref_count++;
} }
} }
KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 &&
(origpte & PG_FRAME) == (pten & PG_FRAME)),
("va %#lx changing 1G phys page pdpe %#lx pten %#lx",
va, origpte, pten));
if ((pten & PG_W) != 0 && (origpte & PG_W) == 0)
pmap->pm_stats.wired_count += NBPDP / PAGE_SIZE;
else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0)
pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE;
*pdpe = pten; *pdpe = pten;
} else /* (psind == 1) */ { /* 2M */ } else /* (psind == 1) */ { /* 2M */
if (!pmap_pkru_same(pmap, va, va + NBPDR))
return (KERN_PROTECTION_FAILURE);
pde = pmap_pde(pmap, va); pde = pmap_pde(pmap, va);
if (pde == NULL) { if (pde == NULL) {
mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va), mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va),
NULL, va); NULL, va);
if (mp == NULL) { if (mp == NULL)
if ((flags & PMAP_ENTER_NOSLEEP) != 0) goto allocf;
return (KERN_RESOURCE_SHORTAGE);
PMAP_UNLOCK(pmap);
vm_wait(NULL);
PMAP_LOCK(pmap);
goto restart;
}
pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp));
pde = &pde[pmap_pde_index(va)]; pde = &pde[pmap_pde_index(va)];
origpte = *pde; origpte = *pde;
MPASS(origpte == 0); MPASS(origpte == 0);
} else { } else {
pdpe = pmap_pdpe(pmap, va);
MPASS(pdpe != NULL && (*pdpe & PG_V) != 0);
origpte = *pde; origpte = *pde;
if ((origpte & PG_V) == 0) { if ((origpte & PG_V) == 0) {
pdpe = pmap_pdpe(pmap, va);
MPASS(pdpe != NULL && (*pdpe & PG_V) != 0);
mp = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); mp = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
mp->ref_count++; mp->ref_count++;
} }
} }
*pde = pten;
}
KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 &&
(origpte & PG_FRAME) == (pten & PG_FRAME)), (origpte & PG_PS_FRAME) == (pten & PG_PS_FRAME)),
("va %#lx changing 2M phys page pde %#lx pten %#lx", ("va %#lx changing %s phys page origpte %#lx pten %#lx",
va, origpte, pten)); va, psind == 2 ? "1G" : "2M", origpte, pten));
if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) if ((pten & PG_W) != 0 && (origpte & PG_W) == 0)
pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE; pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE;
else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0)
pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE;
*pde = pten;
}
if ((origpte & PG_V) == 0) if ((origpte & PG_V) == 0)
pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE); pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE);
return (KERN_SUCCESS); return (KERN_SUCCESS);
allocf:
if ((flags & PMAP_ENTER_NOSLEEP) != 0)
return (KERN_RESOURCE_SHORTAGE);
PMAP_UNLOCK(pmap);
vm_wait(NULL);
PMAP_LOCK(pmap);
goto restart;
} }
/* /*
* Insert the given physical page (p) at * Insert the given physical page (p) at
* the specified virtual address (v) in the * the specified virtual address (v) in the
* target physical map with the protection requested. * target physical map with the protection requested.
* *
* If specified, the page will be wired down, meaning * If specified, the page will be wired down, meaning
▲ Show 20 Lines • Show All 4,854 Lines • Show Last 20 Lines