Page MenuHomeFreeBSD
Differential D20538 Diff 58423 head/sys/arm64/arm64/pmap.c

Changeset View

Standalone View

View Options

head/sys/arm64/arm64/pmap.c

Show First 20 Lines • Show All 2,395 Lines • ▼ Show 20 Linespmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va)
ml3 = pmap_remove_pt_page(pmap, va); ml3 = pmap_remove_pt_page(pmap, va);
if (ml3 == NULL) if (ml3 == NULL)
panic("pmap_remove_kernel_l2: Missing pt page"); panic("pmap_remove_kernel_l2: Missing pt page");
ml3pa = VM_PAGE_TO_PHYS(ml3); ml3pa = VM_PAGE_TO_PHYS(ml3);
newl2 = ml3pa | L2_TABLE; newl2 = ml3pa | L2_TABLE;
/* /*
* Initialize the page table page. * If this page table page was unmapped by a promotion, then it
* contains valid mappings. Zero it to invalidate those mappings.
*/ */
if (ml3->valid != 0)
pagezero((void *)PHYS_TO_DMAP(ml3pa)); pagezero((void *)PHYS_TO_DMAP(ml3pa));
/* /*
* Demote the mapping. The caller must have already invalidated the * Demote the mapping. The caller must have already invalidated the
* mapping (i.e., the "break" in break-before-make). * mapping (i.e., the "break" in break-before-make).
*/ */
oldl2 = pmap_load_store(l2, newl2); oldl2 = pmap_load_store(l2, newl2);
KASSERT(oldl2 == 0, ("%s: found existing mapping at %p: %#lx", KASSERT(oldl2 == 0, ("%s: found existing mapping at %p: %#lx",
__func__, l2, oldl2)); __func__, l2, oldl2));
Show All 36 Lines for (va = sva, m = PHYS_TO_VM_PAGE(old_l2 & ~ATTR_MASK);
vm_page_aflag_clear(m, PGA_WRITEABLE); vm_page_aflag_clear(m, PGA_WRITEABLE);
} }
} }
if (pmap == kernel_pmap) { if (pmap == kernel_pmap) {
pmap_remove_kernel_l2(pmap, l2, sva); pmap_remove_kernel_l2(pmap, l2, sva);
} else { } else {
ml3 = pmap_remove_pt_page(pmap, sva); ml3 = pmap_remove_pt_page(pmap, sva);
if (ml3 != NULL) { if (ml3 != NULL) {
KASSERT(ml3->valid == VM_PAGE_BITS_ALL,
("pmap_remove_l2: l3 page not promoted"));
pmap_resident_count_dec(pmap, 1); pmap_resident_count_dec(pmap, 1);
KASSERT(ml3->wire_count == NL3PG, KASSERT(ml3->wire_count == NL3PG,
("pmap_remove_l2: l3 page wire count error")); ("pmap_remove_l2: l3 page wire count error"));
ml3->wire_count = 1; ml3->wire_count = 1;
vm_page_unwire_noq(ml3); vm_page_unwire_noq(ml3);
pmap_add_delayed_free_list(ml3, free, FALSE); pmap_add_delayed_free_list(ml3, free, FALSE);
} }
} }
▲ Show 20 Lines • Show All 340 Lines • ▼ Show 20 Linespmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
} }
/* /*
* Inserts the specified page table page into the specified pmap's collection * Inserts the specified page table page into the specified pmap's collection
* of idle page table pages. Each of a pmap's page table pages is responsible * of idle page table pages. Each of a pmap's page table pages is responsible
* for mapping a distinct range of virtual addresses. The pmap's collection is * for mapping a distinct range of virtual addresses. The pmap's collection is
* ordered by this virtual address range. * ordered by this virtual address range.
*
* If "promoted" is false, then the page table page "mpte" must be zero filled.
*/ */
static __inline int static __inline int
pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted)
{ {
PMAP_LOCK_ASSERT(pmap, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED);
mpte->valid = promoted ? VM_PAGE_BITS_ALL : 0;
return (vm_radix_insert(&pmap->pm_root, mpte)); return (vm_radix_insert(&pmap->pm_root, mpte));
} }
/* /*
* Removes the page table page mapping the specified virtual address from the * Removes the page table page mapping the specified virtual address from the
* specified pmap's collection of idle page table pages, and returns it. * specified pmap's collection of idle page table pages, and returns it.
* Otherwise, returns NULL if there is no page table page corresponding to the * Otherwise, returns NULL if there is no page table page corresponding to the
* specified virtual address. * specified virtual address.
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Linespmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va,
* destroyed by pmap_remove_l3(). * destroyed by pmap_remove_l3().
*/ */
mpte = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK); mpte = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK);
KASSERT(mpte >= vm_page_array && KASSERT(mpte >= vm_page_array &&
mpte < &vm_page_array[vm_page_array_size], mpte < &vm_page_array[vm_page_array_size],
("pmap_promote_l2: page table page is out of range")); ("pmap_promote_l2: page table page is out of range"));
KASSERT(mpte->pindex == pmap_l2_pindex(va), KASSERT(mpte->pindex == pmap_l2_pindex(va),
("pmap_promote_l2: page table page's pindex is wrong")); ("pmap_promote_l2: page table page's pindex is wrong"));
if (pmap_insert_pt_page(pmap, mpte)) { if (pmap_insert_pt_page(pmap, mpte, true)) {
atomic_add_long(&pmap_l2_p_failures, 1); atomic_add_long(&pmap_l2_p_failures, 1);
CTR2(KTR_PMAP, CTR2(KTR_PMAP,
"pmap_promote_l2: failure for va %#lx in pmap %p", va, "pmap_promote_l2: failure for va %#lx in pmap %p", va,
pmap); pmap);
return; return;
} }
if ((newl2 & ATTR_SW_MANAGED) != 0) if ((newl2 & ATTR_SW_MANAGED) != 0)
▲ Show 20 Lines • Show All 407 Lines • ▼ Show 20 Lines else
l3 = pmap_l2_to_l3(l2, sva); l3 = pmap_l2_to_l3(l2, sva);
if (pmap_l3_valid(pmap_load(l3)) && if (pmap_l3_valid(pmap_load(l3)) &&
pmap_remove_l3(pmap, l3, sva, old_l2, &free, pmap_remove_l3(pmap, l3, sva, old_l2, &free,
lockp) != 0) lockp) != 0)
break; break;
} }
vm_page_free_pages_toq(&free, true); vm_page_free_pages_toq(&free, true);
if (va >= VM_MAXUSER_ADDRESS) { if (va >= VM_MAXUSER_ADDRESS) {
mt = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK);
if (pmap_insert_pt_page(pmap, mt)) {
/* /*
* XXX Currently, this can't happen bacuse * Both pmap_remove_l2() and pmap_remove_l3() will
* we do not perform pmap_enter(psind == 1) * leave the kernel page table page zero filled.
* on the kernel pmap.
*/ */
mt = PHYS_TO_VM_PAGE(pmap_load(l2) & ~ATTR_MASK);
if (pmap_insert_pt_page(pmap, mt, false))
panic("pmap_enter_l2: trie insert failed"); panic("pmap_enter_l2: trie insert failed");
}
} else } else
KASSERT(pmap_load(l2) == 0, KASSERT(pmap_load(l2) == 0,
("pmap_enter_l2: non-zero L2 entry %p", l2)); ("pmap_enter_l2: non-zero L2 entry %p", l2));
} }
if ((new_l2 & ATTR_SW_MANAGED) != 0) { if ((new_l2 & ATTR_SW_MANAGED) != 0) {
/* /*
* Abort this mapping if its PV entry could not be created. * Abort this mapping if its PV entry could not be created.
▲ Show 20 Lines • Show All 670 Lines • ▼ Show 20 Lines */
for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++)
if ((mt->aflags & PGA_WRITEABLE) != 0 && if ((mt->aflags & PGA_WRITEABLE) != 0 &&
TAILQ_EMPTY(&mt->md.pv_list)) TAILQ_EMPTY(&mt->md.pv_list))
vm_page_aflag_clear(mt, PGA_WRITEABLE); vm_page_aflag_clear(mt, PGA_WRITEABLE);
} }
ml3 = pmap_remove_pt_page(pmap, ml3 = pmap_remove_pt_page(pmap,
pv->pv_va); pv->pv_va);
if (ml3 != NULL) { if (ml3 != NULL) {
KASSERT(ml3->valid == VM_PAGE_BITS_ALL,
("pmap_remove_pages: l3 page not promoted"));
pmap_resident_count_dec(pmap,1); pmap_resident_count_dec(pmap,1);
KASSERT(ml3->wire_count == NL3PG, KASSERT(ml3->wire_count == NL3PG,
("pmap_remove_pages: l3 page wire count error")); ("pmap_remove_pages: l3 page wire count error"));
ml3->wire_count = 1; ml3->wire_count = 1;
vm_page_unwire_noq(ml3); vm_page_unwire_noq(ml3);
pmap_add_delayed_free_list(ml3, pmap_add_delayed_free_list(ml3,
&free, FALSE); &free, FALSE);
} }
▲ Show 20 Lines • Show All 938 Lines • ▼ Show 20 Lines if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) {
ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va), ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va),
(VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) |
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); VM_ALLOC_NOOBJ | VM_ALLOC_WIRED);
if (ml3 == NULL) { if (ml3 == NULL) {
CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx"
" in pmap %p", va, pmap); " in pmap %p", va, pmap);
goto fail; goto fail;
} }
if (va < VM_MAXUSER_ADDRESS) if (va < VM_MAXUSER_ADDRESS) {
ml3->wire_count = NL3PG;
pmap_resident_count_inc(pmap, 1); pmap_resident_count_inc(pmap, 1);
} }
}
l3phys = VM_PAGE_TO_PHYS(ml3); l3phys = VM_PAGE_TO_PHYS(ml3);
l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys);
/* Address the range points at */ /* Address the range points at */
phys = oldl2 & ~ATTR_MASK; phys = oldl2 & ~ATTR_MASK;
/* The attributed from the old l2 table to be copied */ /* The attributed from the old l2 table to be copied */
newl3 = (oldl2 & (ATTR_MASK & ~ATTR_DESCR_MASK)) | L3_PAGE; newl3 = (oldl2 & (ATTR_MASK & ~ATTR_DESCR_MASK)) | L3_PAGE;
/* /*
* If the page table page is new, initialize it. * If the page table page is not leftover from an earlier promotion,
* initialize it.
*/ */
if (ml3->wire_count == 1) { if (ml3->valid == 0) {
ml3->wire_count = NL3PG;
for (i = 0; i < Ln_ENTRIES; i++) { for (i = 0; i < Ln_ENTRIES; i++) {
l3[i] = newl3 | phys; l3[i] = newl3 | phys;
phys += L3_SIZE; phys += L3_SIZE;
} }
} }
KASSERT(l3[0] == ((oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE), KASSERT(l3[0] == ((oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE),
("Invalid l3 page (%lx != %lx)", l3[0], ("Invalid l3 page (%lx != %lx)", l3[0],
(oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE)); (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE));
▲ Show 20 Lines • Show All 356 Lines • Show Last 20 Lines