Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 4,413 Lines • ▼ Show 20 Lines | |||||
out: | out: | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
return (rv); | return (rv); | ||||
} | } | ||||
/* | /* | ||||
* Tries to create a read- and/or execute-only 2MB page mapping. Returns true | * Tries to create a read- and/or execute-only 2MB page mapping. Returns | ||||
* if successful. Returns false if (1) a page table page cannot be allocated | * KERN_SUCCESS if the mapping was created. Otherwise, returns an error | ||||
* without sleeping, (2) a mapping already exists at the specified virtual | * value. See pmap_enter_l2() for the possible error values when "no sleep", | ||||
* address, or (3) a PV entry cannot be allocated without reclaiming another | * "no replace", and "no reclaim" are specified. | ||||
* PV entry. | |||||
*/ | */ | ||||
static bool | static int | ||||
pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, | ||||
struct rwlock **lockp) | struct rwlock **lockp) | ||||
{ | { | ||||
pd_entry_t new_l2; | pd_entry_t new_l2; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
PMAP_ASSERT_STAGE1(pmap); | PMAP_ASSERT_STAGE1(pmap); | ||||
KASSERT(ADDR_IS_CANONICAL(va), | KASSERT(ADDR_IS_CANONICAL(va), | ||||
Show All 11 Lines | if ((prot & VM_PROT_EXECUTE) == 0 || | ||||
new_l2 |= ATTR_S1_XN; | new_l2 |= ATTR_S1_XN; | ||||
if (!ADDR_IS_KERNEL(va)) | if (!ADDR_IS_KERNEL(va)) | ||||
new_l2 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | new_l2 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; | ||||
else | else | ||||
new_l2 |= ATTR_S1_UXN; | new_l2 |= ATTR_S1_UXN; | ||||
if (pmap != kernel_pmap) | if (pmap != kernel_pmap) | ||||
new_l2 |= ATTR_S1_nG; | new_l2 |= ATTR_S1_nG; | ||||
return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP | | return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP | | ||||
PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, m, lockp) == | PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, m, lockp)); | ||||
KERN_SUCCESS); | |||||
} | } | ||||
/* | /* | ||||
* Returns true if every page table entry in the specified page table is | * Returns true if every page table entry in the specified page table is | ||||
* zero. | * zero. | ||||
*/ | */ | ||||
static bool | static bool | ||||
pmap_every_pte_zero(vm_paddr_t pa) | pmap_every_pte_zero(vm_paddr_t pa) | ||||
{ | { | ||||
pt_entry_t *pt_end, *pte; | pt_entry_t *pt_end, *pte; | ||||
KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); | KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); | ||||
pte = (pt_entry_t *)PHYS_TO_DMAP(pa); | pte = (pt_entry_t *)PHYS_TO_DMAP(pa); | ||||
for (pt_end = pte + Ln_ENTRIES; pte < pt_end; pte++) { | for (pt_end = pte + Ln_ENTRIES; pte < pt_end; pte++) { | ||||
if (*pte != 0) | if (*pte != 0) | ||||
return (false); | return (false); | ||||
} | } | ||||
return (true); | return (true); | ||||
} | } | ||||
/* | /* | ||||
* Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if | * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if | ||||
* the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE | * the mapping was created, and one of KERN_FAILURE, KERN_NO_SPACE, or | ||||
* otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and | * KERN_RESOURCE_FAILURE otherwise. Returns KERN_FAILURE if | ||||
* a mapping already exists at the specified virtual address. Returns | * PMAP_ENTER_NOREPLACE was specified and a 4KB page mapping already exists | ||||
* KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table | * within the 2MB virtual address range starting at the specified virtual | ||||
* page allocation failed. Returns KERN_RESOURCE_SHORTAGE if | * address. Returns KERN_NO_SPACE if PMAP_ENTER_NOREPLACE was specified and a | ||||
* PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. | * 2MB page mapping already exists at the specified virtual address. Returns | ||||
* KERN_RESOURCE_SHORTAGE if either (1) PMAP_ENTER_NOSLEEP was specified and a | |||||
* page table page allocation failed or (2) PMAP_ENTER_NORECLAIM was specified | |||||
* and a PV entry allocation failed. | |||||
*/ | */ | ||||
static int | static int | ||||
pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags, | pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags, | ||||
vm_page_t m, struct rwlock **lockp) | vm_page_t m, struct rwlock **lockp) | ||||
{ | { | ||||
struct spglist free; | struct spglist free; | ||||
pd_entry_t *l2, old_l2; | pd_entry_t *l2, old_l2; | ||||
vm_page_t l2pg, mt; | vm_page_t l2pg, mt; | ||||
Show All 10 Lines | pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags, | ||||
} | } | ||||
/* | /* | ||||
* If there are existing mappings, either abort or remove them. | * If there are existing mappings, either abort or remove them. | ||||
*/ | */ | ||||
if ((old_l2 = pmap_load(l2)) != 0) { | if ((old_l2 = pmap_load(l2)) != 0) { | ||||
KASSERT(l2pg == NULL || l2pg->ref_count > 1, | KASSERT(l2pg == NULL || l2pg->ref_count > 1, | ||||
("pmap_enter_l2: l2pg's ref count is too low")); | ("pmap_enter_l2: l2pg's ref count is too low")); | ||||
if ((flags & PMAP_ENTER_NOREPLACE) != 0 && | if ((flags & PMAP_ENTER_NOREPLACE) != 0) { | ||||
(!ADDR_IS_KERNEL(va) || | if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) { | ||||
(old_l2 & ATTR_DESCR_MASK) == L2_BLOCK || | |||||
!pmap_every_pte_zero(old_l2 & ~ATTR_MASK))) { | |||||
if (l2pg != NULL) | if (l2pg != NULL) | ||||
l2pg->ref_count--; | l2pg->ref_count--; | ||||
CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx" | CTR2(KTR_PMAP, | ||||
"pmap_enter_l2: no space for va %#lx" | |||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (KERN_NO_SPACE); | |||||
} else if (!ADDR_IS_KERNEL(va) || | |||||
!pmap_every_pte_zero(old_l2 & ~ATTR_MASK)) { | |||||
if (l2pg != NULL) | |||||
l2pg->ref_count--; | |||||
CTR2(KTR_PMAP, | |||||
"pmap_enter_l2: failure for va %#lx" | |||||
" in pmap %p", va, pmap); | |||||
return (KERN_FAILURE); | return (KERN_FAILURE); | ||||
} | } | ||||
} | |||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) | if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) | ||||
(void)pmap_remove_l2(pmap, l2, va, | (void)pmap_remove_l2(pmap, l2, va, | ||||
pmap_load(pmap_l1(pmap, va)), &free, lockp); | pmap_load(pmap_l1(pmap, va)), &free, lockp); | ||||
else | else | ||||
pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE, | pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE, | ||||
&free, lockp); | &free, lockp); | ||||
if (!ADDR_IS_KERNEL(va)) { | if (!ADDR_IS_KERNEL(va)) { | ||||
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines | |||||
void | void | ||||
pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, | pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, | ||||
vm_page_t m_start, vm_prot_t prot) | vm_page_t m_start, vm_prot_t prot) | ||||
{ | { | ||||
struct rwlock *lock; | struct rwlock *lock; | ||||
vm_offset_t va; | vm_offset_t va; | ||||
vm_page_t m, mpte; | vm_page_t m, mpte; | ||||
vm_pindex_t diff, psize; | vm_pindex_t diff, psize; | ||||
int rv; | |||||
VM_OBJECT_ASSERT_LOCKED(m_start->object); | VM_OBJECT_ASSERT_LOCKED(m_start->object); | ||||
psize = atop(end - start); | psize = atop(end - start); | ||||
mpte = NULL; | mpte = NULL; | ||||
m = m_start; | m = m_start; | ||||
lock = NULL; | lock = NULL; | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { | while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { | ||||
va = start + ptoa(diff); | va = start + ptoa(diff); | ||||
if ((va & L2_OFFSET) == 0 && va + L2_SIZE <= end && | if ((va & L2_OFFSET) == 0 && va + L2_SIZE <= end && | ||||
m->psind == 1 && pmap_ps_enabled(pmap) && | m->psind == 1 && pmap_ps_enabled(pmap) && | ||||
pmap_enter_2mpage(pmap, va, m, prot, &lock)) | ((rv = pmap_enter_2mpage(pmap, va, m, prot, &lock)) == | ||||
KERN_SUCCESS || rv == KERN_NO_SPACE)) | |||||
m = &m[L2_SIZE / PAGE_SIZE - 1]; | m = &m[L2_SIZE / PAGE_SIZE - 1]; | ||||
else | else | ||||
mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte, | mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte, | ||||
&lock); | &lock); | ||||
m = TAILQ_NEXT(m, listq); | m = TAILQ_NEXT(m, listq); | ||||
} | } | ||||
if (lock != NULL) | if (lock != NULL) | ||||
rw_wunlock(lock); | rw_wunlock(lock); | ||||
▲ Show 20 Lines • Show All 3,017 Lines • Show Last 20 Lines |