Changeset View
Changeset View
Standalone View
Standalone View
sys/arm64/arm64/pmap.c
Show First 20 Lines • Show All 1,779 Lines • ▼ Show 20 Lines | pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels) | ||||
*/ | */ | ||||
while ((m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | while ((m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | | ||||
VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) | ||||
vm_wait(NULL); | vm_wait(NULL); | ||||
pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m); | pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m); | ||||
pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr); | pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->pm_l0_paddr); | ||||
if ((m->flags & PG_ZERO) == 0) | |||||
pagezero(pmap->pm_l0); | |||||
pmap->pm_root.rt_root = 0; | pmap->pm_root.rt_root = 0; | ||||
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); | ||||
pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX); | pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX); | ||||
MPASS(levels == 3 || levels == 4); | MPASS(levels == 3 || levels == 4); | ||||
pmap->pm_levels = levels; | pmap->pm_levels = levels; | ||||
pmap->pm_stage = stage; | pmap->pm_stage = stage; | ||||
switch (stage) { | switch (stage) { | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | | ||||
} | } | ||||
/* | /* | ||||
* Indicate the need to retry. While waiting, the page table | * Indicate the need to retry. While waiting, the page table | ||||
* page may have been allocated. | * page may have been allocated. | ||||
*/ | */ | ||||
return (NULL); | return (NULL); | ||||
} | } | ||||
if ((m->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(m); | |||||
/* | /* | ||||
* Because of AArch64's weak memory consistency model, we must have a | * Because of AArch64's weak memory consistency model, we must have a | ||||
* barrier here to ensure that the stores for zeroing "m", whether by | * barrier here to ensure that the stores for zeroing "m", whether by | ||||
* pmap_zero_page() or an earlier function, are visible before adding | * pmap_zero_page() or an earlier function, are visible before adding | ||||
* "m" to the page table. Otherwise, a page table walk by another | * "m" to the page table. Otherwise, a page table walk by another | ||||
* processor's MMU could see the mapping to "m" and a stale, non-zero | * processor's MMU could see the mapping to "m" and a stale, non-zero | ||||
* PTE within "m". | * PTE within "m". | ||||
▲ Show 20 Lines • Show All 301 Lines • ▼ Show 20 Lines | while (kernel_vm_end < addr) { | ||||
l1 = pmap_l0_to_l1(l0, kernel_vm_end); | l1 = pmap_l0_to_l1(l0, kernel_vm_end); | ||||
if (pmap_load(l1) == 0) { | if (pmap_load(l1) == 0) { | ||||
/* We need a new PDP entry */ | /* We need a new PDP entry */ | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, | nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | | ||||
VM_ALLOC_WIRED | VM_ALLOC_ZERO); | VM_ALLOC_WIRED | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(nkpg); | |||||
/* See the dmb() in _pmap_alloc_l3(). */ | /* See the dmb() in _pmap_alloc_l3(). */ | ||||
dmb(ishst); | dmb(ishst); | ||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pmap_store(l1, paddr | L1_TABLE); | pmap_store(l1, paddr | L1_TABLE); | ||||
continue; /* try again */ | continue; /* try again */ | ||||
} | } | ||||
l2 = pmap_l1_to_l2(l1, kernel_vm_end); | l2 = pmap_l1_to_l2(l1, kernel_vm_end); | ||||
if (pmap_load(l2) != 0) { | if (pmap_load(l2) != 0) { | ||||
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
break; | break; | ||||
} | } | ||||
continue; | continue; | ||||
} | } | ||||
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, | nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, | ||||
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | | ||||
VM_ALLOC_ZERO); | VM_ALLOC_ZERO); | ||||
if (nkpg == NULL) | if (nkpg == NULL) | ||||
panic("pmap_growkernel: no memory to grow kernel"); | panic("pmap_growkernel: no memory to grow kernel"); | ||||
if ((nkpg->flags & PG_ZERO) == 0) | |||||
pmap_zero_page(nkpg); | |||||
/* See the dmb() in _pmap_alloc_l3(). */ | /* See the dmb() in _pmap_alloc_l3(). */ | ||||
dmb(ishst); | dmb(ishst); | ||||
paddr = VM_PAGE_TO_PHYS(nkpg); | paddr = VM_PAGE_TO_PHYS(nkpg); | ||||
pmap_store(l2, paddr | L2_TABLE); | pmap_store(l2, paddr | L2_TABLE); | ||||
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; | ||||
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { | ||||
kernel_vm_end = vm_map_max(kernel_map); | kernel_vm_end = vm_map_max(kernel_map); | ||||
▲ Show 20 Lines • Show All 4,998 Lines • Show Last 20 Lines |