Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 464 Lines • ▼ Show 20 Lines | pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) | ||||
pd_entry_t l1, *l2p; | pd_entry_t l1, *l2p; | ||||
l1 = pmap_load(l1p); | l1 = pmap_load(l1p); | ||||
/* | /* | ||||
* The valid bit may be clear if pmap_update_entry() is concurrently | * The valid bit may be clear if pmap_update_entry() is concurrently | ||||
* modifying the entry, so for KVA only the entry type may be checked. | * modifying the entry, so for KVA only the entry type may be checked. | ||||
*/ | */ | ||||
KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0, | KASSERT(ADDR_IS_CANONICAL(va), | ||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
KASSERT(ADDR_IS_KERNEL(va) || (l1 & ATTR_DESCR_VALID) != 0, | |||||
("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); | ("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); | ||||
alc: I would suggest moving this KASSERT() before the above comment. Otherwise, this KASSERT() sits… | |||||
KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, | KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, | ||||
("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); | ("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); | ||||
l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK); | l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK); | ||||
return (&l2p[pmap_l2_index(va)]); | return (&l2p[pmap_l2_index(va)]); | ||||
} | } | ||||
static __inline pd_entry_t * | static __inline pd_entry_t * | ||||
pmap_l2(pmap_t pmap, vm_offset_t va) | pmap_l2(pmap_t pmap, vm_offset_t va) | ||||
Show All 14 Lines | pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) | ||||
pt_entry_t *l3p; | pt_entry_t *l3p; | ||||
l2 = pmap_load(l2p); | l2 = pmap_load(l2p); | ||||
/* | /* | ||||
* The valid bit may be clear if pmap_update_entry() is concurrently | * The valid bit may be clear if pmap_update_entry() is concurrently | ||||
* modifying the entry, so for KVA only the entry type may be checked. | * modifying the entry, so for KVA only the entry type may be checked. | ||||
*/ | */ | ||||
KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0, | KASSERT(ADDR_IS_CANONICAL(va), | ||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
KASSERT(ADDR_IS_KERNEL(va) || (l2 & ATTR_DESCR_VALID) != 0, | |||||
("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); | ("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); | ||||
Not Done Inline ActionsDitto. alc: Ditto. | |||||
KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, | KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, | ||||
("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); | ("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); | ||||
l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK); | l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK); | ||||
return (&l3p[pmap_l3_index(va)]); | return (&l3p[pmap_l3_index(va)]); | ||||
} | } | ||||
/* | /* | ||||
* Returns the lowest valid pde for a given virtual address. | * Returns the lowest valid pde for a given virtual address. | ||||
▲ Show 20 Lines • Show All 1,060 Lines • ▼ Show 20 Lines | |||||
*/ | */ | ||||
void | void | ||||
pmap_qremove(vm_offset_t sva, int count) | pmap_qremove(vm_offset_t sva, int count) | ||||
{ | { | ||||
pt_entry_t *pte; | pt_entry_t *pte; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
int lvl; | int lvl; | ||||
KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", sva)); | KASSERT(ADDR_IS_CANONICAL(sva), | ||||
("%s: Address not in canonical form: %lx", __func__, sva)); | |||||
KASSERT(ADDR_IS_KERNEL(sva), ("usermode va %lx", sva)); | |||||
va = sva; | va = sva; | ||||
while (count-- > 0) { | while (count-- > 0) { | ||||
pte = pmap_pte(kernel_pmap, va, &lvl); | pte = pmap_pte(kernel_pmap, va, &lvl); | ||||
KASSERT(lvl == 3, | KASSERT(lvl == 3, | ||||
("Invalid device pagetable level: %d != 3", lvl)); | ("Invalid device pagetable level: %d != 3", lvl)); | ||||
if (pte != NULL) { | if (pte != NULL) { | ||||
pmap_clear(pte); | pmap_clear(pte); | ||||
▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | |||||
* conditionally free the page, and manage the reference count. | * conditionally free the page, and manage the reference count. | ||||
*/ | */ | ||||
static int | static int | ||||
pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, | ||||
struct spglist *free) | struct spglist *free) | ||||
{ | { | ||||
vm_page_t mpte; | vm_page_t mpte; | ||||
if (va >= VM_MAXUSER_ADDRESS) | KASSERT(ADDR_IS_CANONICAL(va), | ||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
if (ADDR_IS_KERNEL(va)) | |||||
return (0); | return (0); | ||||
KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); | KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); | ||||
mpte = PHYS_TO_VM_PAGE(ptepde & ~ATTR_MASK); | mpte = PHYS_TO_VM_PAGE(ptepde & ~ATTR_MASK); | ||||
return (pmap_unwire_l3(pmap, va, mpte, free)); | return (pmap_unwire_l3(pmap, va, mpte, free)); | ||||
} | } | ||||
/* | /* | ||||
* Release a page table page reference after a failed attempt to create a | * Release a page table page reference after a failed attempt to create a | ||||
▲ Show 20 Lines • Show All 237 Lines • ▼ Show 20 Lines | |||||
static pd_entry_t * | static pd_entry_t * | ||||
pmap_alloc_l2(pmap_t pmap, vm_offset_t va, vm_page_t *l2pgp, | pmap_alloc_l2(pmap_t pmap, vm_offset_t va, vm_page_t *l2pgp, | ||||
struct rwlock **lockp) | struct rwlock **lockp) | ||||
{ | { | ||||
pd_entry_t *l1, *l2; | pd_entry_t *l1, *l2; | ||||
vm_page_t l2pg; | vm_page_t l2pg; | ||||
vm_pindex_t l2pindex; | vm_pindex_t l2pindex; | ||||
KASSERT(ADDR_IS_CANONICAL(va), | |||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
retry: | retry: | ||||
l1 = pmap_l1(pmap, va); | l1 = pmap_l1(pmap, va); | ||||
if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) { | if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) { | ||||
l2 = pmap_l1_to_l2(l1, va); | l2 = pmap_l1_to_l2(l1, va); | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (!ADDR_IS_KERNEL(va)) { | ||||
/* Add a reference to the L2 page. */ | /* Add a reference to the L2 page. */ | ||||
l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK); | l2pg = PHYS_TO_VM_PAGE(pmap_load(l1) & ~ATTR_MASK); | ||||
l2pg->ref_count++; | l2pg->ref_count++; | ||||
} else | } else | ||||
l2pg = NULL; | l2pg = NULL; | ||||
} else if (va < VM_MAXUSER_ADDRESS) { | } else if (!ADDR_IS_KERNEL(va)) { | ||||
/* Allocate a L2 page. */ | /* Allocate a L2 page. */ | ||||
l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT; | l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT; | ||||
l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp); | l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp); | ||||
if (l2pg == NULL) { | if (l2pg == NULL) { | ||||
if (lockp != NULL) | if (lockp != NULL) | ||||
goto retry; | goto retry; | ||||
else | else | ||||
return (NULL); | return (NULL); | ||||
▲ Show 20 Lines • Show All 949 Lines • ▼ Show 20 Lines | pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva, | ||||
vm_offset_t eva, struct spglist *free, struct rwlock **lockp) | vm_offset_t eva, struct spglist *free, struct rwlock **lockp) | ||||
{ | { | ||||
struct md_page *pvh; | struct md_page *pvh; | ||||
struct rwlock *new_lock; | struct rwlock *new_lock; | ||||
pt_entry_t *l3, old_l3; | pt_entry_t *l3, old_l3; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
vm_page_t l3pg, m; | vm_page_t l3pg, m; | ||||
KASSERT(ADDR_IS_CANONICAL(sva), | |||||
("%s: Start address not in canonical form: %lx", __func__, sva)); | |||||
KASSERT(ADDR_IS_CANONICAL(eva) || eva == VM_MAX_USER_ADDRESS, | |||||
("%s: End address not in canonical form: %lx", __func__, eva)); | |||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
KASSERT(rounddown2(sva, L2_SIZE) + L2_SIZE == roundup2(eva, L2_SIZE), | KASSERT(rounddown2(sva, L2_SIZE) + L2_SIZE == roundup2(eva, L2_SIZE), | ||||
("pmap_remove_l3_range: range crosses an L3 page table boundary")); | ("pmap_remove_l3_range: range crosses an L3 page table boundary")); | ||||
l3pg = sva < VM_MAXUSER_ADDRESS ? PHYS_TO_VM_PAGE(l2e & ~ATTR_MASK) : | l3pg = !ADDR_IS_KERNEL(sva) ? PHYS_TO_VM_PAGE(l2e & ~ATTR_MASK) : NULL; | ||||
NULL; | |||||
va = eva; | va = eva; | ||||
for (l3 = pmap_l2_to_l3(&l2e, sva); sva != eva; l3++, sva += L3_SIZE) { | for (l3 = pmap_l2_to_l3(&l2e, sva); sva != eva; l3++, sva += L3_SIZE) { | ||||
if (!pmap_l3_valid(pmap_load(l3))) { | if (!pmap_l3_valid(pmap_load(l3))) { | ||||
if (va != eva) { | if (va != eva) { | ||||
pmap_invalidate_range(pmap, va, sva); | pmap_invalidate_range(pmap, va, sva); | ||||
va = eva; | va = eva; | ||||
} | } | ||||
continue; | continue; | ||||
▲ Show 20 Lines • Show All 744 Lines • ▼ Show 20 Lines | pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
pt_entry_t new_l3, orig_l3; | pt_entry_t new_l3, orig_l3; | ||||
pt_entry_t *l2, *l3; | pt_entry_t *l2, *l3; | ||||
pv_entry_t pv; | pv_entry_t pv; | ||||
vm_paddr_t opa, pa; | vm_paddr_t opa, pa; | ||||
vm_page_t mpte, om; | vm_page_t mpte, om; | ||||
boolean_t nosleep; | boolean_t nosleep; | ||||
int lvl, rv; | int lvl, rv; | ||||
KASSERT(ADDR_IS_CANONICAL(va), | |||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
va = trunc_page(va); | va = trunc_page(va); | ||||
if ((m->oflags & VPO_UNMANAGED) == 0) | if ((m->oflags & VPO_UNMANAGED) == 0) | ||||
VM_PAGE_OBJECT_BUSY_ASSERT(m); | VM_PAGE_OBJECT_BUSY_ASSERT(m); | ||||
pa = VM_PAGE_TO_PHYS(m); | pa = VM_PAGE_TO_PHYS(m); | ||||
new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | L3_PAGE); | new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | L3_PAGE); | ||||
new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr); | new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr); | ||||
new_l3 |= pmap_pte_prot(pmap, prot); | new_l3 |= pmap_pte_prot(pmap, prot); | ||||
if ((flags & PMAP_ENTER_WIRED) != 0) | if ((flags & PMAP_ENTER_WIRED) != 0) | ||||
new_l3 |= ATTR_SW_WIRED; | new_l3 |= ATTR_SW_WIRED; | ||||
if (pmap->pm_stage == PM_STAGE1) { | if (pmap->pm_stage == PM_STAGE1) { | ||||
if (va < VM_MAXUSER_ADDRESS) | if (!ADDR_IS_KERNEL(va)) | ||||
new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | ||||
else | else | ||||
new_l3 |= ATTR_S1_UXN; | new_l3 |= ATTR_S1_UXN; | ||||
if (pmap != kernel_pmap) | if (pmap != kernel_pmap) | ||||
new_l3 |= ATTR_S1_nG; | new_l3 |= ATTR_S1_nG; | ||||
} else { | } else { | ||||
/* | /* | ||||
* Clear the access flag on executable mappings, this will be | * Clear the access flag on executable mappings, this will be | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
/* | /* | ||||
* In the case that a page table page is not | * In the case that a page table page is not | ||||
* resident, we are creating it here. | * resident, we are creating it here. | ||||
*/ | */ | ||||
retry: | retry: | ||||
pde = pmap_pde(pmap, va, &lvl); | pde = pmap_pde(pmap, va, &lvl); | ||||
if (pde != NULL && lvl == 2) { | if (pde != NULL && lvl == 2) { | ||||
l3 = pmap_l2_to_l3(pde, va); | l3 = pmap_l2_to_l3(pde, va); | ||||
if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { | if (!ADDR_IS_KERNEL(va) && mpte == NULL) { | ||||
mpte = PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK); | mpte = PHYS_TO_VM_PAGE(pmap_load(pde) & ~ATTR_MASK); | ||||
mpte->ref_count++; | mpte->ref_count++; | ||||
} | } | ||||
goto havel3; | goto havel3; | ||||
} else if (pde != NULL && lvl == 1) { | } else if (pde != NULL && lvl == 1) { | ||||
l2 = pmap_l1_to_l2(pde, va); | l2 = pmap_l1_to_l2(pde, va); | ||||
if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK && | if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK && | ||||
(l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) { | (l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) { | ||||
l3 = &l3[pmap_l3_index(va)]; | l3 = &l3[pmap_l3_index(va)]; | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (!ADDR_IS_KERNEL(va)) { | ||||
mpte = PHYS_TO_VM_PAGE( | mpte = PHYS_TO_VM_PAGE( | ||||
pmap_load(l2) & ~ATTR_MASK); | pmap_load(l2) & ~ATTR_MASK); | ||||
mpte->ref_count++; | mpte->ref_count++; | ||||
} | } | ||||
goto havel3; | goto havel3; | ||||
} | } | ||||
/* We need to allocate an L3 table. */ | /* We need to allocate an L3 table. */ | ||||
} | } | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (!ADDR_IS_KERNEL(va)) { | ||||
nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; | ||||
/* | /* | ||||
* We use _pmap_alloc_l3() instead of pmap_alloc_l3() in order | * We use _pmap_alloc_l3() instead of pmap_alloc_l3() in order | ||||
* to handle the possibility that a superpage mapping for "va" | * to handle the possibility that a superpage mapping for "va" | ||||
* was created while we slept. | * was created while we slept. | ||||
*/ | */ | ||||
mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), | mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), | ||||
▲ Show 20 Lines • Show All 208 Lines • ▼ Show 20 Lines | |||||
static bool | static bool | ||||
pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
struct rwlock **lockp) | struct rwlock **lockp) | ||||
{ | { | ||||
pd_entry_t new_l2; | pd_entry_t new_l2; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PMAP_ASSERT_STAGE1(pmap); | PMAP_ASSERT_STAGE1(pmap); | ||||
KASSERT(ADDR_IS_CANONICAL(va), | |||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
new_l2 = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | | new_l2 = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | | ||||
ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | | ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | | ||||
L2_BLOCK); | L2_BLOCK); | ||||
if ((m->oflags & VPO_UNMANAGED) == 0) { | if ((m->oflags & VPO_UNMANAGED) == 0) { | ||||
new_l2 |= ATTR_SW_MANAGED; | new_l2 |= ATTR_SW_MANAGED; | ||||
new_l2 &= ~ATTR_AF; | new_l2 &= ~ATTR_AF; | ||||
} | } | ||||
if ((prot & VM_PROT_EXECUTE) == 0 || | if ((prot & VM_PROT_EXECUTE) == 0 || | ||||
m->md.pv_memattr == VM_MEMATTR_DEVICE) | m->md.pv_memattr == VM_MEMATTR_DEVICE) | ||||
new_l2 |= ATTR_S1_XN; | new_l2 |= ATTR_S1_XN; | ||||
if (va < VM_MAXUSER_ADDRESS) | if (!ADDR_IS_KERNEL(va)) | ||||
new_l2 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | new_l2 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | ||||
else | else | ||||
new_l2 |= ATTR_S1_UXN; | new_l2 |= ATTR_S1_UXN; | ||||
if (pmap != kernel_pmap) | if (pmap != kernel_pmap) | ||||
new_l2 |= ATTR_S1_nG; | new_l2 |= ATTR_S1_nG; | ||||
return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP | | return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP | | ||||
PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == | PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == | ||||
KERN_SUCCESS); | KERN_SUCCESS); | ||||
Show All 32 Lines | |||||
pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags, | pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags, | ||||
vm_page_t m, struct rwlock **lockp) | vm_page_t m, struct rwlock **lockp) | ||||
{ | { | ||||
struct spglist free; | struct spglist free; | ||||
pd_entry_t *l2, old_l2; | pd_entry_t *l2, old_l2; | ||||
vm_page_t l2pg, mt; | vm_page_t l2pg, mt; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
KASSERT(ADDR_IS_CANONICAL(va), | |||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
if ((l2 = pmap_alloc_l2(pmap, va, &l2pg, (flags & | if ((l2 = pmap_alloc_l2(pmap, va, &l2pg, (flags & | ||||
PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp)) == NULL) { | PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp)) == NULL) { | ||||
CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p", | CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p", | ||||
va, pmap); | va, pmap); | ||||
return (KERN_RESOURCE_SHORTAGE); | return (KERN_RESOURCE_SHORTAGE); | ||||
} | } | ||||
/* | /* | ||||
* If there are existing mappings, either abort or remove them. | * If there are existing mappings, either abort or remove them. | ||||
*/ | */ | ||||
if ((old_l2 = pmap_load(l2)) != 0) { | if ((old_l2 = pmap_load(l2)) != 0) { | ||||
KASSERT(l2pg == NULL || l2pg->ref_count > 1, | KASSERT(l2pg == NULL || l2pg->ref_count > 1, | ||||
("pmap_enter_l2: l2pg's ref count is too low")); | ("pmap_enter_l2: l2pg's ref count is too low")); | ||||
if ((flags & PMAP_ENTER_NOREPLACE) != 0 && (va < | if ((flags & PMAP_ENTER_NOREPLACE) != 0 && | ||||
VM_MAXUSER_ADDRESS || (old_l2 & ATTR_DESCR_MASK) == | (!ADDR_IS_KERNEL(va) || | ||||
L2_BLOCK || !pmap_every_pte_zero(old_l2 & ~ATTR_MASK))) { | (old_l2 & ATTR_DESCR_MASK) == L2_BLOCK || | ||||
!pmap_every_pte_zero(old_l2 & ~ATTR_MASK))) { | |||||
if (l2pg != NULL) | if (l2pg != NULL) | ||||
l2pg->ref_count--; | l2pg->ref_count--; | ||||
CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (KERN_FAILURE); | return (KERN_FAILURE); | ||||
} | } | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) | if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) | ||||
(void)pmap_remove_l2(pmap, l2, va, | (void)pmap_remove_l2(pmap, l2, va, | ||||
pmap_load(pmap_l1(pmap, va)), &free, lockp); | pmap_load(pmap_l1(pmap, va)), &free, lockp); | ||||
else | else | ||||
pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE, | pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE, | ||||
&free, lockp); | &free, lockp); | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (!ADDR_IS_KERNEL(va)) { | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
KASSERT(pmap_load(l2) == 0, | KASSERT(pmap_load(l2) == 0, | ||||
("pmap_enter_l2: non-zero L2 entry %p", l2)); | ("pmap_enter_l2: non-zero L2 entry %p", l2)); | ||||
} else { | } else { | ||||
KASSERT(SLIST_EMPTY(&free), | KASSERT(SLIST_EMPTY(&free), | ||||
("pmap_enter_l2: freed kernel page table page")); | ("pmap_enter_l2: freed kernel page table page")); | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
int lvl; | int lvl; | ||||
KASSERT(!VA_IS_CLEANMAP(va) || | KASSERT(!VA_IS_CLEANMAP(va) || | ||||
(m->oflags & VPO_UNMANAGED) != 0, | (m->oflags & VPO_UNMANAGED) != 0, | ||||
("pmap_enter_quick_locked: managed mapping within the clean submap")); | ("pmap_enter_quick_locked: managed mapping within the clean submap")); | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PMAP_ASSERT_STAGE1(pmap); | PMAP_ASSERT_STAGE1(pmap); | ||||
KASSERT(ADDR_IS_CANONICAL(va), | |||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va); | CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va); | ||||
/* | /* | ||||
* In the case that a page table page is not | * In the case that a page table page is not | ||||
* resident, we are creating it here. | * resident, we are creating it here. | ||||
*/ | */ | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (!ADDR_IS_KERNEL(va)) { | ||||
vm_pindex_t l2pindex; | vm_pindex_t l2pindex; | ||||
/* | /* | ||||
* Calculate pagetable page index | * Calculate pagetable page index | ||||
*/ | */ | ||||
l2pindex = pmap_l2_pindex(va); | l2pindex = pmap_l2_pindex(va); | ||||
if (mpte && (mpte->pindex == l2pindex)) { | if (mpte && (mpte->pindex == l2pindex)) { | ||||
mpte->ref_count++; | mpte->ref_count++; | ||||
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
pa = VM_PAGE_TO_PHYS(m); | pa = VM_PAGE_TO_PHYS(m); | ||||
l3_val = pa | ATTR_DEFAULT | ATTR_S1_IDX(m->md.pv_memattr) | | l3_val = pa | ATTR_DEFAULT | ATTR_S1_IDX(m->md.pv_memattr) | | ||||
ATTR_S1_AP(ATTR_S1_AP_RO) | L3_PAGE; | ATTR_S1_AP(ATTR_S1_AP_RO) | L3_PAGE; | ||||
if ((prot & VM_PROT_EXECUTE) == 0 || | if ((prot & VM_PROT_EXECUTE) == 0 || | ||||
m->md.pv_memattr == VM_MEMATTR_DEVICE) | m->md.pv_memattr == VM_MEMATTR_DEVICE) | ||||
l3_val |= ATTR_S1_XN; | l3_val |= ATTR_S1_XN; | ||||
if (va < VM_MAXUSER_ADDRESS) | if (!ADDR_IS_KERNEL(va)) | ||||
l3_val |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | l3_val |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | ||||
else | else | ||||
l3_val |= ATTR_S1_UXN; | l3_val |= ATTR_S1_UXN; | ||||
if (pmap != kernel_pmap) | if (pmap != kernel_pmap) | ||||
l3_val |= ATTR_S1_nG; | l3_val |= ATTR_S1_nG; | ||||
/* | /* | ||||
* Now validate mapping with RO protection | * Now validate mapping with RO protection | ||||
▲ Show 20 Lines • Show All 1,729 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
pt_entry_t *l3, newl3, oldl2; | pt_entry_t *l3, newl3, oldl2; | ||||
vm_offset_t tmpl2; | vm_offset_t tmpl2; | ||||
vm_paddr_t l3phys; | vm_paddr_t l3phys; | ||||
vm_page_t ml3; | vm_page_t ml3; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PMAP_ASSERT_STAGE1(pmap); | PMAP_ASSERT_STAGE1(pmap); | ||||
KASSERT(ADDR_IS_CANONICAL(va), | |||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
l3 = NULL; | l3 = NULL; | ||||
oldl2 = pmap_load(l2); | oldl2 = pmap_load(l2); | ||||
KASSERT((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK, | KASSERT((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK, | ||||
("pmap_demote_l2: Demoting a non-block entry")); | ("pmap_demote_l2: Demoting a non-block entry")); | ||||
va &= ~L2_OFFSET; | va &= ~L2_OFFSET; | ||||
tmpl2 = 0; | tmpl2 = 0; | ||||
if (va <= (vm_offset_t)l2 && va + L2_SIZE > (vm_offset_t)l2) { | if (va <= (vm_offset_t)l2 && va + L2_SIZE > (vm_offset_t)l2) { | ||||
Show All 23 Lines | if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) { | ||||
/* | /* | ||||
* If the page table page is missing and the mapping | * If the page table page is missing and the mapping | ||||
* is for a kernel address, the mapping must belong to | * is for a kernel address, the mapping must belong to | ||||
* the direct map. Page table pages are preallocated | * the direct map. Page table pages are preallocated | ||||
* for every other part of the kernel address space, | * for every other part of the kernel address space, | ||||
* so the direct map region is the only part of the | * so the direct map region is the only part of the | ||||
* kernel address space that must be handled here. | * kernel address space that must be handled here. | ||||
*/ | */ | ||||
KASSERT(va < VM_MAXUSER_ADDRESS || VIRT_IN_DMAP(va), | KASSERT(!ADDR_IS_KERNEL(va) || VIRT_IN_DMAP(va), | ||||
("pmap_demote_l2: No saved mpte for va %#lx", va)); | ("pmap_demote_l2: No saved mpte for va %#lx", va)); | ||||
/* | /* | ||||
* If the 2MB page mapping belongs to the direct map | * If the 2MB page mapping belongs to the direct map | ||||
* region of the kernel's address space, then the page | * region of the kernel's address space, then the page | ||||
* allocation request specifies the highest possible | * allocation request specifies the highest possible | ||||
* priority (VM_ALLOC_INTERRUPT). Otherwise, the | * priority (VM_ALLOC_INTERRUPT). Otherwise, the | ||||
* priority is normal. | * priority is normal. | ||||
*/ | */ | ||||
ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va), | ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va), | ||||
(VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); | ||||
/* | /* | ||||
* If the allocation of the new page table page fails, | * If the allocation of the new page table page fails, | ||||
* invalidate the 2MB page mapping and return "failure". | * invalidate the 2MB page mapping and return "failure". | ||||
*/ | */ | ||||
if (ml3 == NULL) { | if (ml3 == NULL) { | ||||
pmap_demote_l2_abort(pmap, va, l2, lockp); | pmap_demote_l2_abort(pmap, va, l2, lockp); | ||||
CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
if (va < VM_MAXUSER_ADDRESS) { | if (!ADDR_IS_KERNEL(va)) { | ||||
ml3->ref_count = NL3PG; | ml3->ref_count = NL3PG; | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
} | } | ||||
l3phys = VM_PAGE_TO_PHYS(ml3); | l3phys = VM_PAGE_TO_PHYS(ml3); | ||||
l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); | l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); | ||||
newl3 = (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE; | newl3 = (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE; | ||||
KASSERT((oldl2 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) != | KASSERT((oldl2 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) != | ||||
▲ Show 20 Lines • Show All 346 Lines • ▼ Show 20 Lines | pmap_switch(struct thread *old __unused, struct thread *new) | ||||
return (pcb); | return (pcb); | ||||
} | } | ||||
void | void | ||||
pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz) | pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz) | ||||
{ | { | ||||
PMAP_ASSERT_STAGE1(pmap); | PMAP_ASSERT_STAGE1(pmap); | ||||
if (va >= VM_MIN_KERNEL_ADDRESS) { | KASSERT(ADDR_IS_CANONICAL(va), | ||||
("%s: Address not in canonical form: %lx", __func__, va)); | |||||
if (ADDR_IS_KERNEL(va)) { | |||||
cpu_icache_sync_range(va, sz); | cpu_icache_sync_range(va, sz); | ||||
} else { | } else { | ||||
u_int len, offset; | u_int len, offset; | ||||
vm_paddr_t pa; | vm_paddr_t pa; | ||||
/* Find the length of data in this page to flush */ | /* Find the length of data in this page to flush */ | ||||
offset = va & PAGE_MASK; | offset = va & PAGE_MASK; | ||||
len = imin(PAGE_SIZE - offset, sz); | len = imin(PAGE_SIZE - offset, sz); | ||||
▲ Show 20 Lines • Show All 499 Lines • Show Last 20 Lines |
I would suggest moving this KASSERT() before the above comment. Otherwise, this KASSERT() sits between the comment and the KASSERT() immediately below that the comment explains.