Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 1,758 Lines • ▼ Show 20 Lines | |||||
int | int | ||||
pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels) | pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
/* | /* | ||||
* allocate the l0 page | * allocate the l0 page | ||||
*/ | */ | ||||
while ((m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | m = vm_page_alloc_noobj(VM_ALLOC_NORMAL | VM_ALLOC_WAITOK | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
vm_wait(NULL); | |||||
pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m); | pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m); | ||||
pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr); | pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr); | ||||
if ((m->flags & PG_ZERO) == 0) | |||||
pagezero(pmap->pm_l0); | |||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | ||||
pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX); | pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX); | ||||
MPASS(levels == 3 || levels == 4); | MPASS(levels == 3 || levels == 4); | ||||
pmap->pm_levels = levels; | pmap->pm_levels = levels; | ||||
pmap->pm_stage = stage; | pmap->pm_stage = stage; | ||||
switch (stage) { | switch (stage) { | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | |||||
{ | { | ||||
vm_page_t m, l1pg, l2pg; | vm_page_t m, l1pg, l2pg; | ||||
PMAP_LOCK_ASSERT(pmap, MA_OWNED); | PMAP_LOCK_ASSERT(pmap, MA_OWNED); | ||||
/* | /* | ||||
* Allocate a page table page. | * Allocate a page table page. | ||||
*/ | */ | ||||
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { | |||||
if (lockp != NULL) { | if (lockp != NULL) { | ||||
RELEASE_PV_LIST_LOCK(lockp); | RELEASE_PV_LIST_LOCK(lockp); | ||||
PMAP_UNLOCK(pmap); | PMAP_UNLOCK(pmap); | ||||
vm_wait(NULL); | vm_wait(NULL); | ||||
PMAP_LOCK(pmap); | PMAP_LOCK(pmap); | ||||
} | } | ||||
/* | /* | ||||
* Indicate the need to retry. While waiting, the page table | * Indicate the need to retry. While waiting, the page table | ||||
* page may have been allocated. | * page may have been allocated. | ||||
*/ | */ | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
if ((m->flags & PG_ZERO) == 0) | m->pindex = ptepindex; | ||||
pmap_zero_page(m); | |||||
/* | /* | ||||
* Because of AArch64's weak memory consistency model, we must have a | * Because of AArch64's weak memory consistency model, we must have a | ||||
* barrier here to ensure that the stores for zeroing "m", whether by | * barrier here to ensure that the stores for zeroing "m", whether by | ||||
* pmap_zero_page() or an earlier function, are visible before adding | * pmap_zero_page() or an earlier function, are visible before adding | ||||
* "m" to the page table. Otherwise, a page table walk by another | * "m" to the page table. Otherwise, a page table walk by another | ||||
* processor's MMU could see the mapping to "m" and a stale, non-zero | * processor's MMU could see the mapping to "m" and a stale, non-zero | ||||
* PTE within "m". | * PTE within "m". | ||||
▲ Show 20 Lines • Show All 305 Lines • ▼ Show 20 Lines | pmap_growkernel(vm_offset_t addr) | ||||
while (kernel_vm_end < addr) { | while (kernel_vm_end < addr) { | ||||
l0 = pmap_l0(kernel_pmap, kernel_vm_end); | l0 = pmap_l0(kernel_pmap, kernel_vm_end); | ||||
KASSERT(pmap_load(l0) != 0, | KASSERT(pmap_load(l0) != 0, | ||||
("pmap_growkernel: No level 0 kernel entry")); | ("pmap_growkernel: No level 0 kernel entry")); | ||||
l1 = pmap_l0_to_l1(l0, kernel_vm_end); | l1 = pmap_l0_to_l1(l0, kernel_vm_end); | ||||
if (pmap_load(l1) == 0) { | if (pmap_load(l1) == 0) { | ||||
/* We need a new PDP entry */ | /* We need a new PDP entry */ | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, | nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | |||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | nkpg->pindex = kernel_vm_end >> L1_SHIFT; | ||||
pmap_zero_page(nkpg); | |||||
/* See the dmb() in _pmap_alloc_l3(). */ | /* See the dmb() in _pmap_alloc_l3(). */ | ||||
dmb(ishst); | dmb(ishst); | ||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pmap_store(l1, paddr | L1_TABLE); | pmap_store(l1, paddr | L1_TABLE); | ||||
continue; /* try again */ | continue; /* try again */ | ||||
} | } | ||||
l2 = pmap_l1_to_l2(l1, kernel_vm_end); | l2 = pmap_l1_to_l2(l1, kernel_vm_end); | ||||
if (pmap_load(l2) != 0) { | if (pmap_load(l2) != 0) { | ||||
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, | nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | |||||
VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | nkpg->pindex = kernel_vm_end >> L2_SHIFT; | ||||
pmap_zero_page(nkpg); | |||||
/* See the dmb() in _pmap_alloc_l3(). */ | /* See the dmb() in _pmap_alloc_l3(). */ | ||||
dmb(ishst); | dmb(ishst); | ||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pmap_store(l2, paddr | L2_TABLE); | pmap_store(l2, paddr | L2_TABLE); | ||||
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
▲ Show 20 Lines • Show All 328 Lines • ▼ Show 20 Lines | if (field < _NPCM) { | ||||
pc_list); | pc_list); | ||||
} | } | ||||
PV_STAT(atomic_add_long(&pv_entry_count, 1)); | PV_STAT(atomic_add_long(&pv_entry_count, 1)); | ||||
PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); | ||||
return (pv); | return (pv); | ||||
} | } | ||||
} | } | ||||
/* No free items, allocate another chunk */ | /* No free items, allocate another chunk */ | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_NORMAL | VM_ALLOC_WIRED); | ||||
VM_ALLOC_WIRED); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
if (lockp == NULL) { | if (lockp == NULL) { | ||||
PV_STAT(pc_chunk_tryfail++); | PV_STAT(pc_chunk_tryfail++); | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
m = reclaim_pv_chunk(pmap, lockp); | m = reclaim_pv_chunk(pmap, lockp); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | bit_count((bitstr_t *)pc->pc_map, 0, | ||||
sizeof(pc->pc_map) * NBBY, &free); | sizeof(pc->pc_map) * NBBY, &free); | ||||
if (free == 0) | if (free == 0) | ||||
break; | break; | ||||
avail += free; | avail += free; | ||||
if (avail >= needed) | if (avail >= needed) | ||||
break; | break; | ||||
} | } | ||||
for (reclaimed = false; avail < needed; avail += _NPCPV) { | for (reclaimed = false; avail < needed; avail += _NPCPV) { | ||||
m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | | m = vm_page_alloc_noobj(VM_ALLOC_NORMAL | VM_ALLOC_WIRED); | ||||
VM_ALLOC_WIRED); | |||||
if (m == NULL) { | if (m == NULL) { | ||||
m = reclaim_pv_chunk(pmap, lockp); | m = reclaim_pv_chunk(pmap, lockp); | ||||
if (m == NULL) | if (m == NULL) | ||||
goto retry; | goto retry; | ||||
reclaimed = true; | reclaimed = true; | ||||
} | } | ||||
PV_STAT(atomic_add_int(&pc_chunk_count, 1)); | PV_STAT(atomic_add_int(&pc_chunk_count, 1)); | ||||
PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); | PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); | ||||
▲ Show 20 Lines • Show All 3,420 Lines • ▼ Show 20 Lines | pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va) | ||||
tmpl1 = 0; | tmpl1 = 0; | ||||
if (va <= (vm_offset_t)l1 && va + L1_SIZE > (vm_offset_t)l1) { | if (va <= (vm_offset_t)l1 && va + L1_SIZE > (vm_offset_t)l1) { | ||||
tmpl1 = kva_alloc(PAGE_SIZE); | tmpl1 = kva_alloc(PAGE_SIZE); | ||||
if (tmpl1 == 0) | if (tmpl1 == 0) | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
if ((ml2 = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT | | if ((ml2 = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED)) == | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { | NULL) { | ||||
CTR2(KTR_PMAP, "pmap_demote_l1: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_demote_l1: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
l2 = NULL; | l2 = NULL; | ||||
goto fail; | goto fail; | ||||
} | } | ||||
l2phys = VM_PAGE_TO_PHYS(ml2); | l2phys = VM_PAGE_TO_PHYS(ml2); | ||||
l2 = (pt_entry_t *)PHYS_TO_DMAP(l2phys); | l2 = (pt_entry_t *)PHYS_TO_DMAP(l2phys); | ||||
▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines | if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) { | ||||
/* | /* | ||||
* If the 2MB page mapping belongs to the direct map | * If the 2MB page mapping belongs to the direct map | ||||
* region of the kernel's address space, then the page | * region of the kernel's address space, then the page | ||||
* allocation request specifies the highest possible | * allocation request specifies the highest possible | ||||
* priority (VM_ALLOC_INTERRUPT). Otherwise, the | * priority (VM_ALLOC_INTERRUPT). Otherwise, the | ||||
* priority is normal. | * priority is normal. | ||||
*/ | */ | ||||
ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va), | ml3 = vm_page_alloc_noobj( | ||||
(VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); | VM_ALLOC_WIRED); | ||||
/* | /* | ||||
* If the allocation of the new page table page fails, | * If the allocation of the new page table page fails, | ||||
* invalidate the 2MB page mapping and return "failure". | * invalidate the 2MB page mapping and return "failure". | ||||
*/ | */ | ||||
if (ml3 == NULL) { | if (ml3 == NULL) { | ||||
pmap_demote_l2_abort(pmap, va, l2, lockp); | pmap_demote_l2_abort(pmap, va, l2, lockp); | ||||
CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" | CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" | ||||
" in pmap %p", va, pmap); | " in pmap %p", va, pmap); | ||||
goto fail; | goto fail; | ||||
} | } | ||||
ml3->pindex = pmap_l2_pindex(va); | |||||
if (!ADDR_IS_KERNEL(va)) { | if (!ADDR_IS_KERNEL(va)) { | ||||
ml3->ref_count = NL3PG; | ml3->ref_count = NL3PG; | ||||
pmap_resident_count_inc(pmap, 1); | pmap_resident_count_inc(pmap, 1); | ||||
} | } | ||||
} | } | ||||
l3phys = VM_PAGE_TO_PHYS(ml3); | l3phys = VM_PAGE_TO_PHYS(ml3); | ||||
l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); | l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); | ||||
▲ Show 20 Lines • Show All 867 Lines • Show Last 20 Lines |