Changeset View
Changeset View
Standalone View
Standalone View
sys/arm/arm/pmap-v6.c
Show First 20 Lines • Show All 2,359 Lines • ▼ Show 20 Lines | |||||
* Each L2 page table in a page has own counter which counts a number of | * Each L2 page table in a page has own counter which counts a number of | ||||
* valid mappings in a table. Global page counter counts mappings in all | * valid mappings in a table. Global page counter counts mappings in all | ||||
* tables in a page plus a single itself mapping in PT2TAB. | * tables in a page plus a single itself mapping in PT2TAB. | ||||
* | * | ||||
* During a promotion we leave the associated L2 page table counter | * During a promotion we leave the associated L2 page table counter | ||||
* untouched, so the table (strictly speaking a page which holds it) | * untouched, so the table (strictly speaking a page which holds it) | ||||
* is never freed if promoted. | * is never freed if promoted. | ||||
* | * | ||||
* If a page m->wire_count == 1 then no valid mappings exist in any L2 page | * If a page m->ref_count == 1 then no valid mappings exist in any L2 page | ||||
* table in the page and the page itself is only mapped in PT2TAB. | * table in the page and the page itself is only mapped in PT2TAB. | ||||
*/ | */ | ||||
static __inline void | static __inline void | ||||
pt2_wirecount_init(vm_page_t m) | pt2_wirecount_init(vm_page_t m) | ||||
{ | { | ||||
u_int i; | u_int i; | ||||
/* | /* | ||||
* Note: A page m is allocated with VM_ALLOC_WIRED flag and | * Note: A page m is allocated with VM_ALLOC_WIRED flag and | ||||
* m->wire_count should be already set correctly. | * m->ref_count should be already set correctly. | ||||
* So, there is no need to set it again herein. | * So, there is no need to set it again herein. | ||||
*/ | */ | ||||
for (i = 0; i < NPT2_IN_PG; i++) | for (i = 0; i < NPT2_IN_PG; i++) | ||||
m->md.pt2_wirecount[i] = 0; | m->md.pt2_wirecount[i] = 0; | ||||
} | } | ||||
static __inline void | static __inline void | ||||
pt2_wirecount_inc(vm_page_t m, uint32_t pte1_idx) | pt2_wirecount_inc(vm_page_t m, uint32_t pte1_idx) | ||||
{ | { | ||||
/* | /* | ||||
* Note: A just modificated pte2 (i.e. already allocated) | * Note: A just modificated pte2 (i.e. already allocated) | ||||
* is acquiring one extra reference which must be | * is acquiring one extra reference which must be | ||||
* explicitly cleared. It influences the KASSERTs herein. | * explicitly cleared. It influences the KASSERTs herein. | ||||
* All L2 page tables in a page always belong to the same | * All L2 page tables in a page always belong to the same | ||||
* pmap, so we allow only one extra reference for the page. | * pmap, so we allow only one extra reference for the page. | ||||
*/ | */ | ||||
KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] < (NPTE2_IN_PT2 + 1), | KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] < (NPTE2_IN_PT2 + 1), | ||||
("%s: PT2 is overflowing ...", __func__)); | ("%s: PT2 is overflowing ...", __func__)); | ||||
KASSERT(m->wire_count <= (NPTE2_IN_PG + 1), | KASSERT(m->ref_count <= (NPTE2_IN_PG + 1), | ||||
("%s: PT2PG is overflowing ...", __func__)); | ("%s: PT2PG is overflowing ...", __func__)); | ||||
m->wire_count++; | m->ref_count++; | ||||
m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]++; | m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]++; | ||||
} | } | ||||
static __inline void | static __inline void | ||||
pt2_wirecount_dec(vm_page_t m, uint32_t pte1_idx) | pt2_wirecount_dec(vm_page_t m, uint32_t pte1_idx) | ||||
{ | { | ||||
KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] != 0, | KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] != 0, | ||||
("%s: PT2 is underflowing ...", __func__)); | ("%s: PT2 is underflowing ...", __func__)); | ||||
KASSERT(m->wire_count > 1, | KASSERT(m->ref_count > 1, | ||||
("%s: PT2PG is underflowing ...", __func__)); | ("%s: PT2PG is underflowing ...", __func__)); | ||||
m->wire_count--; | m->ref_count--; | ||||
m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]--; | m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]--; | ||||
} | } | ||||
static __inline void | static __inline void | ||||
pt2_wirecount_set(vm_page_t m, uint32_t pte1_idx, uint16_t count) | pt2_wirecount_set(vm_page_t m, uint32_t pte1_idx, uint16_t count) | ||||
{ | { | ||||
KASSERT(count <= NPTE2_IN_PT2, | KASSERT(count <= NPTE2_IN_PT2, | ||||
("%s: invalid count %u", __func__, count)); | ("%s: invalid count %u", __func__, count)); | ||||
KASSERT(m->wire_count > m->md.pt2_wirecount[pte1_idx & PT2PG_MASK], | KASSERT(m->ref_count > m->md.pt2_wirecount[pte1_idx & PT2PG_MASK], | ||||
("%s: PT2PG corrupting (%u, %u) ...", __func__, m->wire_count, | ("%s: PT2PG corrupting (%u, %u) ...", __func__, m->ref_count, | ||||
m->md.pt2_wirecount[pte1_idx & PT2PG_MASK])); | m->md.pt2_wirecount[pte1_idx & PT2PG_MASK])); | ||||
m->wire_count -= m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]; | m->ref_count -= m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]; | ||||
m->wire_count += count; | m->ref_count += count; | ||||
m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] = count; | m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] = count; | ||||
KASSERT(m->wire_count <= (NPTE2_IN_PG + 1), | KASSERT(m->ref_count <= (NPTE2_IN_PG + 1), | ||||
("%s: PT2PG is overflowed (%u) ...", __func__, m->wire_count)); | ("%s: PT2PG is overflowed (%u) ...", __func__, m->ref_count)); | ||||
} | } | ||||
static __inline uint32_t | static __inline uint32_t | ||||
pt2_wirecount_get(vm_page_t m, uint32_t pte1_idx) | pt2_wirecount_get(vm_page_t m, uint32_t pte1_idx) | ||||
{ | { | ||||
return (m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]); | return (m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]); | ||||
} | } | ||||
Show All 12 Lines | pt2_is_full(vm_page_t m, vm_offset_t va) | ||||
return (m->md.pt2_wirecount[pte1_index(va) & PT2PG_MASK] == | return (m->md.pt2_wirecount[pte1_index(va) & PT2PG_MASK] == | ||||
NPTE2_IN_PT2); | NPTE2_IN_PT2); | ||||
} | } | ||||
static __inline boolean_t | static __inline boolean_t | ||||
pt2pg_is_empty(vm_page_t m) | pt2pg_is_empty(vm_page_t m) | ||||
{ | { | ||||
return (m->wire_count == 1); | return (m->ref_count == 1); | ||||
} | } | ||||
/* | /* | ||||
* This routine is called if the L2 page table | * This routine is called if the L2 page table | ||||
* is not mapped correctly. | * is not mapped correctly. | ||||
*/ | */ | ||||
static vm_page_t | static vm_page_t | ||||
_pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags) | _pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags) | ||||
▲ Show 20 Lines • Show All 157 Lines • ▼ Show 20 Lines | #endif | ||||
/* | /* | ||||
* Unmap the page from PT2TAB. | * Unmap the page from PT2TAB. | ||||
*/ | */ | ||||
pte2p = pmap_pt2tab_entry(pmap, va); | pte2p = pmap_pt2tab_entry(pmap, va); | ||||
(void)pt2tab_load_clear(pte2p); | (void)pt2tab_load_clear(pte2p); | ||||
pmap_tlb_flush(pmap, pt2map_pt2pg(va)); | pmap_tlb_flush(pmap, pt2map_pt2pg(va)); | ||||
m->wire_count = 0; | m->ref_count = 0; | ||||
pmap->pm_stats.resident_count--; | pmap->pm_stats.resident_count--; | ||||
/* | /* | ||||
* This barrier is so that the ordinary store unmapping | * This barrier is so that the ordinary store unmapping | ||||
* the L2 page table page is globally performed before TLB shoot- | * the L2 page table page is globally performed before TLB shoot- | ||||
* down is begun. | * down is begun. | ||||
*/ | */ | ||||
wmb(); | wmb(); | ||||
Show All 32 Lines | |||||
static __inline void | static __inline void | ||||
pmap_unwire_pt2_all(pmap_t pmap, vm_offset_t va, vm_page_t m, | pmap_unwire_pt2_all(pmap_t pmap, vm_offset_t va, vm_page_t m, | ||||
struct spglist *free) | struct spglist *free) | ||||
{ | { | ||||
u_int pte1_idx = pte1_index(va); | u_int pte1_idx = pte1_index(va); | ||||
KASSERT(m->pindex == (pte1_idx & ~PT2PG_MASK), | KASSERT(m->pindex == (pte1_idx & ~PT2PG_MASK), | ||||
("%s: PT2 page's pindex is wrong", __func__)); | ("%s: PT2 page's pindex is wrong", __func__)); | ||||
KASSERT(m->wire_count > pt2_wirecount_get(m, pte1_idx), | KASSERT(m->ref_count > pt2_wirecount_get(m, pte1_idx), | ||||
("%s: bad pt2 wire count %u > %u", __func__, m->wire_count, | ("%s: bad pt2 wire count %u > %u", __func__, m->ref_count, | ||||
pt2_wirecount_get(m, pte1_idx))); | pt2_wirecount_get(m, pte1_idx))); | ||||
/* | /* | ||||
* It's possible that the L2 page table was never used. | * It's possible that the L2 page table was never used. | ||||
* It happened in case that a section was created without promotion. | * It happened in case that a section was created without promotion. | ||||
*/ | */ | ||||
if (pt2_is_full(m, va)) { | if (pt2_is_full(m, va)) { | ||||
pt2_wirecount_set(m, pte1_idx, 0); | pt2_wirecount_set(m, pte1_idx, 0); | ||||
▲ Show 20 Lines • Show All 248 Lines • ▼ Show 20 Lines | out: | ||||
if (pmap != NULL) { | if (pmap != NULL) { | ||||
if (pmap != locked_pmap) | if (pmap != locked_pmap) | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
} | } | ||||
if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) { | if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) { | ||||
m_pc = SLIST_FIRST(&free); | m_pc = SLIST_FIRST(&free); | ||||
SLIST_REMOVE_HEAD(&free, plinks.s.ss); | SLIST_REMOVE_HEAD(&free, plinks.s.ss); | ||||
/* Recycle a freed page table page. */ | /* Recycle a freed page table page. */ | ||||
m_pc->wire_count = 1; | m_pc->ref_count = 1; | ||||
vm_wire_add(1); | vm_wire_add(1); | ||||
} | } | ||||
vm_page_free_pages_toq(&free, false); | vm_page_free_pages_toq(&free, false); | ||||
return (m_pc); | return (m_pc); | ||||
} | } | ||||
static void | static void | ||||
free_pv_chunk(struct pv_chunk *pc) | free_pv_chunk(struct pv_chunk *pc) | ||||
▲ Show 20 Lines • Show All 3,741 Lines • ▼ Show 20 Lines | for (i = 0; i < NPTE1_IN_PT1; i++) { | ||||
pmap_pte2_release(pte2p); | pmap_pte2_release(pte2p); | ||||
if (!pte2_is_valid(pte2)) | if (!pte2_is_valid(pte2)) | ||||
continue; | continue; | ||||
pa = pte2_pa(pte2); | pa = pte2_pa(pte2); | ||||
m = PHYS_TO_VM_PAGE(pa); | m = PHYS_TO_VM_PAGE(pa); | ||||
printf("va: 0x%x, pa: 0x%x, w: %d, " | printf("va: 0x%x, pa: 0x%x, w: %d, " | ||||
"f: 0x%x", va, pa, | "f: 0x%x", va, pa, | ||||
m->wire_count, m->flags); | m->ref_count, m->flags); | ||||
npte2++; | npte2++; | ||||
index++; | index++; | ||||
if (index >= 2) { | if (index >= 2) { | ||||
index = 0; | index = 0; | ||||
printf("\n"); | printf("\n"); | ||||
} else { | } else { | ||||
printf(" "); | printf(" "); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | if (!pte2_is_valid(pte2)) { | ||||
printf("\n"); | printf("\n"); | ||||
continue; | continue; | ||||
} | } | ||||
m = PHYS_TO_VM_PAGE(pte2_pa(pte2)); | m = PHYS_TO_VM_PAGE(pte2_pa(pte2)); | ||||
printf(" 0x%08X: 0x%08X, TEX%d, s:%d, g:%d, m:%p", va , pte2, | printf(" 0x%08X: 0x%08X, TEX%d, s:%d, g:%d, m:%p", va , pte2, | ||||
pte2_class(pte2), !!(pte2 & PTE2_S), !(pte2 & PTE2_NG), m); | pte2_class(pte2), !!(pte2 & PTE2_S), !(pte2 & PTE2_NG), m); | ||||
if (m != NULL) { | if (m != NULL) { | ||||
printf(" v:%d w:%d f:0x%04X\n", m->valid, | printf(" v:%d w:%d f:0x%04X\n", m->valid, | ||||
m->wire_count, m->flags); | m->ref_count, m->flags); | ||||
} else { | } else { | ||||
printf("\n"); | printf("\n"); | ||||
} | } | ||||
} | } | ||||
} | } | ||||
static __inline boolean_t | static __inline boolean_t | ||||
is_pv_chunk_space(vm_offset_t va) | is_pv_chunk_space(vm_offset_t va) | ||||
▲ Show 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | if (pte1_is_section(pte1)) { | ||||
if (is_pv_chunk_space(va)) { | if (is_pv_chunk_space(va)) { | ||||
printf(" - pv_chunk space"); | printf(" - pv_chunk space"); | ||||
if (dump_pv_chunk) | if (dump_pv_chunk) | ||||
invalid_ok = TRUE; | invalid_ok = TRUE; | ||||
else | else | ||||
dump_link_ok = FALSE; | dump_link_ok = FALSE; | ||||
} | } | ||||
else if (m != NULL) | else if (m != NULL) | ||||
printf(" w:%d w2:%u", m->wire_count, | printf(" w:%d w2:%u", m->ref_count, | ||||
pt2_wirecount_get(m, pte1_index(va))); | pt2_wirecount_get(m, pte1_index(va))); | ||||
if (pte2 == 0) | if (pte2 == 0) | ||||
printf(" !!! pt2tab entry is ZERO"); | printf(" !!! pt2tab entry is ZERO"); | ||||
else if (pte2_pa(pte1) != pte2_pa(pte2)) | else if (pte2_pa(pte1) != pte2_pa(pte2)) | ||||
printf(" !!! pt2tab entry is DIFFERENT - m: %p", | printf(" !!! pt2tab entry is DIFFERENT - m: %p", | ||||
PHYS_TO_VM_PAGE(pte2_pa(pte2))); | PHYS_TO_VM_PAGE(pte2_pa(pte2))); | ||||
printf("\n"); | printf("\n"); | ||||
if (dump_link_ok) | if (dump_link_ok) | ||||
Show All 19 Lines | if (!pte2_is_valid(pte2)) | ||||
continue; | continue; | ||||
va = i << PT2TAB_SHIFT; | va = i << PT2TAB_SHIFT; | ||||
pa = pte2_pa(pte2); | pa = pte2_pa(pte2); | ||||
m = PHYS_TO_VM_PAGE(pa); | m = PHYS_TO_VM_PAGE(pa); | ||||
printf(" 0x%08X: 0x%08X, TEX%d, s:%d, m:%p", va, pte2, | printf(" 0x%08X: 0x%08X, TEX%d, s:%d, m:%p", va, pte2, | ||||
pte2_class(pte2), !!(pte2 & PTE2_S), m); | pte2_class(pte2), !!(pte2 & PTE2_S), m); | ||||
if (m != NULL) | if (m != NULL) | ||||
printf(" , w: %d, f: 0x%04X pidx: %lld", | printf(" , w: %d, f: 0x%04X pidx: %lld", | ||||
m->wire_count, m->flags, m->pindex); | m->ref_count, m->flags, m->pindex); | ||||
printf("\n"); | printf("\n"); | ||||
} | } | ||||
} | } | ||||
DB_SHOW_COMMAND(pmap_pt2tab, pmap_pt2tab_print) | DB_SHOW_COMMAND(pmap_pt2tab, pmap_pt2tab_print) | ||||
{ | { | ||||
/* XXX convert args. */ | /* XXX convert args. */ | ||||
pmap_t pmap = (pmap_t)addr; | pmap_t pmap = (pmap_t)addr; | ||||
Show All 37 Lines |