Changeset View
Changeset View
Standalone View
Standalone View
sys/powerpc/aim/mmu_radix.c
Show First 20 Lines • Show All 1,208 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
if (free == 0) | if (free == 0) | ||||
break; | break; | ||||
avail += free; | avail += free; | ||||
if (avail >= needed) | if (avail >= needed) | ||||
break; | break; | ||||
} | } | ||||
for (reclaimed = false; avail < needed; avail += _NPCPV) { | for (reclaimed = false; avail < needed; avail += _NPCPV) { | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_NORMAL | VM_ALLOC_WIRED); | ||||
VM_ALLOC_WIRED); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
m = reclaim_pv_chunk(pmap, lockp); | m = reclaim_pv_chunk(pmap, lockp); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
reclaimed = true; | reclaimed = true; | ||||
} | } | ||||
PV_STAT(atomic_add_int(&pc_chunk_count, 1)); | PV_STAT(atomic_add_int(&pc_chunk_count, 1)); | ||||
PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); | PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); | ||||
▲ Show 20 Lines • Show All 405 Lines • ▼ Show 20 Lines | if (field < _NPCM) { | ||||
} | } | ||||
PV_STAT(atomic_add_long(&pv_entry_count, 1)); | PV_STAT(atomic_add_long(&pv_entry_count, 1)); | ||||
PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | ||||
MPASS(PV_PMAP(pv) != NULL); | MPASS(PV_PMAP(pv) != NULL); | ||||
return (pv); | return (pv); | ||||
} | } | ||||
} | } | ||||
/* No free items, allocate another chunk */ | /* No free items, allocate another chunk */ | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_NORMAL | VM_ALLOC_WIRED); | ||||
VM_ALLOC_WIRED); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
if (lockp == NULL) { | if (lockp == NULL) { | ||||
PV_STAT(pc_chunk_tryfail++); | PV_STAT(pc_chunk_tryfail++); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
m = reclaim_pv_chunk(pmap, lockp); | m = reclaim_pv_chunk(pmap, lockp); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 1,880 Lines • ▼ Show 20 Lines | mmu_radix_growkernel(vm_offset_t addr) | ||||
addr = roundup2(addr, L3_PAGE_SIZE); | addr = roundup2(addr, L3_PAGE_SIZE); | ||||
if (addr - 1 >= vm_map_max(kernel_map)) | if (addr - 1 >= vm_map_max(kernel_map)) | ||||
addr = vm_map_max(kernel_map); | addr = vm_map_max(kernel_map); | ||||
while (kernel_vm_end < addr) { | while (kernel_vm_end < addr) { | ||||
l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); | l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); | ||||
if ((be64toh(*l2e) & PG_V) == 0) { | if ((be64toh(*l2e) & PG_V) == 0) { | ||||
/* We need a new PDP entry */ | /* We need a new PDP entry */ | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_PAGE_SIZE_SHIFT, | nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT; | ||||
mmu_radix_zero_page(nkpg); | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pde_store(l2e, paddr); | pde_store(l2e, paddr); | ||||
continue; /* try again */ | continue; /* try again */ | ||||
} | } | ||||
l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); | l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); | ||||
if ((be64toh(*l3e) & PG_V) != 0) { | if ((be64toh(*l3e) & PG_V) != 0) { | ||||
kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; | kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
nkpg = vm_page_alloc(NULL, pmap_l3e_pindex(kernel_vm_end), | nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | nkpg->pindex = pmap_l3e_pindex(kernel_vm_end); | ||||
mmu_radix_zero_page(nkpg); | |||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pde_store(l3e, paddr); | pde_store(l3e, paddr); | ||||
kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; | kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 670 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
vm_page_t m, pdppg, pdpg; | vm_page_t m, pdppg, pdpg; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
/* | /* | ||||
* Allocate a page table page. | * Allocate a page table page. | ||||
*/ | */ | ||||
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | |||||
if (lockp != NULL) { | if (lockp != NULL) { | ||||
RELEASE_PV_LIST_LOCK(lockp); | RELEASE_PV_LIST_LOCK(lockp); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
vm_wait(NULL); | vm_wait(NULL); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
} | } | ||||
/* | /* | ||||
* Indicate the need to retry. While waiting, the page table | * Indicate the need to retry. While waiting, the page table | ||||
* page may have been allocated. | * page may have been allocated. | ||||
*/ | */ | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
if ((m->flags & PG_ZERO) == 0) | m->pindex = ptepindex; | ||||
mmu_radix_zero_page(m); | |||||
/* | /* | ||||
* Map the pagetable page into the process address space, if | * Map the pagetable page into the process address space, if | ||||
* it isn't already there. | * it isn't already there. | ||||
*/ | */ | ||||
if (ptepindex >= (NUPDE + NUPDPE)) { | if (ptepindex >= (NUPDE + NUPDPE)) { | ||||
pml1_entry_t *l1e; | pml1_entry_t *l1e; | ||||
▲ Show 20 Lines • Show All 640 Lines • ▼ Show 20 Lines | if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == | ||||
* the direct map region of the kernel's address space, then | * the direct map region of the kernel's address space, then | ||||
* the page allocation request specifies the highest possible | * the page allocation request specifies the highest possible | ||||
* priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is | * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is | ||||
* normal. Page table pages are preallocated for every other | * normal. Page table pages are preallocated for every other | ||||
* part of the kernel address space, so the direct map region | * part of the kernel address space, so the direct map region | ||||
* is the only part of the kernel address space that must be | * is the only part of the kernel address space that must be | ||||
* handled here. | * handled here. | ||||
*/ | */ | ||||
if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, | if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj( | ||||
pmap_l3e_pindex(va), (va >= DMAP_MIN_ADDRESS && va < | (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ? | ||||
DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { | VM_ALLOC_WIRED)) == NULL) { | ||||
SLIST_INIT(&free); | SLIST_INIT(&free); | ||||
sva = trunc_2mpage(va); | sva = trunc_2mpage(va); | ||||
pmap_remove_l3e(pmap, l3e, sva, &free, lockp); | pmap_remove_l3e(pmap, l3e, sva, &free, lockp); | ||||
pmap_invalidate_l3e_page(pmap, sva, oldpde); | pmap_invalidate_l3e_page(pmap, sva, oldpde); | ||||
vm_page_free_pages_toq(&free, true); | vm_page_free_pages_toq(&free, true); | ||||
CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
mpte->pindex = pmap_l3e_pindex(va); | |||||
if (va < VM_MAXUSER_ADDRESS) | if (va < VM_MAXUSER_ADDRESS) | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
mptepa = VM_PAGE_TO_PHYS(mpte); | mptepa = VM_PAGE_TO_PHYS(mpte); | ||||
firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); | ||||
KASSERT((oldpde & PG_A) != 0, | KASSERT((oldpde & PG_A) != 0, | ||||
("pmap_demote_l3e: oldpde is missing PG_A")); | ("pmap_demote_l3e: oldpde is missing PG_A")); | ||||
KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, | KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, | ||||
▲ Show 20 Lines • Show All 1,003 Lines • ▼ Show 20 Lines | pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) | ||||
pml3_entry_t *firstpde, newpde, *pde; | pml3_entry_t *firstpde, newpde, *pde; | ||||
vm_paddr_t pdpgpa; | vm_paddr_t pdpgpa; | ||||
vm_page_t pdpg; | vm_page_t pdpg; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
oldpdpe = be64toh(*l2e); | oldpdpe = be64toh(*l2e); | ||||
KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), | KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), | ||||
("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); | ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); | ||||
pdpg = vm_page_alloc(NULL, va >> L2_PAGE_SIZE_SHIFT, | pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); | |||||
if (pdpg == NULL) { | if (pdpg == NULL) { | ||||
CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
return (FALSE); | return (FALSE); | ||||
} | } | ||||
pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT; | |||||
pdpgpa = VM_PAGE_TO_PHYS(pdpg); | pdpgpa = VM_PAGE_TO_PHYS(pdpg); | ||||
firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); | firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); | ||||
KASSERT((oldpdpe & PG_A) != 0, | KASSERT((oldpdpe & PG_A) != 0, | ||||
("pmap_demote_pdpe: oldpdpe is missing PG_A")); | ("pmap_demote_pdpe: oldpdpe is missing PG_A")); | ||||
KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, | KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, | ||||
("pmap_demote_pdpe: oldpdpe is missing PG_M")); | ("pmap_demote_pdpe: oldpdpe is missing PG_M")); | ||||
newpde = oldpdpe; | newpde = oldpdpe; | ||||
▲ Show 20 Lines • Show All 506 Lines • Show Last 20 Lines |