Page MenuHomeFreeBSD
Differential D21126 Diff 60408 head/sys/arm64/arm64/pmap.c

Changeset View

Standalone View

View Options

head/sys/arm64/arm64/pmap.c

Show First 20 Lines • Show All 1,519 Lines • ▼ Show 20 Lines if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
* page may have been allocated. * page may have been allocated.
*/ */
return (NULL); return (NULL);
} }
if ((m->flags & PG_ZERO) == 0) if ((m->flags & PG_ZERO) == 0)
pmap_zero_page(m); pmap_zero_page(m);
/* /*
* Because of AArch64's weak memory consistency model, we must have a
* barrier here to ensure that the stores for zeroing "m", whether by
* pmap_zero_page() or an earlier function, are visible before adding
* "m" to the page table. Otherwise, a page table walk by another
* processor's MMU could see the mapping to "m" and a stale, non-zero
* PTE within "m".
*/
dmb(ishst);
/*
* Map the pagetable page into the process address space, if * Map the pagetable page into the process address space, if
* it isn't already there. * it isn't already there.
*/ */
if (ptepindex >= (NUL2E + NUL1E)) { if (ptepindex >= (NUL2E + NUL1E)) {
pd_entry_t *l0; pd_entry_t *l0;
vm_pindex_t l0index; vm_pindex_t l0index;
▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines if (pmap_load(l1) == 0) {
/* We need a new PDP entry */ /* We need a new PDP entry */
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT, nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT,
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED | VM_ALLOC_ZERO); VM_ALLOC_WIRED | VM_ALLOC_ZERO);
if (nkpg == NULL) if (nkpg == NULL)
panic("pmap_growkernel: no memory to grow kernel"); panic("pmap_growkernel: no memory to grow kernel");
if ((nkpg->flags & PG_ZERO) == 0) if ((nkpg->flags & PG_ZERO) == 0)
pmap_zero_page(nkpg); pmap_zero_page(nkpg);
/* See the dmb() in _pmap_alloc_l3(). */
dmb(ishst);
paddr = VM_PAGE_TO_PHYS(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg);
pmap_store(l1, paddr | L1_TABLE); pmap_store(l1, paddr | L1_TABLE);
continue; /* try again */ continue; /* try again */
} }
l2 = pmap_l1_to_l2(l1, kernel_vm_end); l2 = pmap_l1_to_l2(l1, kernel_vm_end);
if ((pmap_load(l2) & ATTR_AF) != 0) { if (pmap_load(l2) != 0) {
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
kernel_vm_end = vm_map_max(kernel_map); kernel_vm_end = vm_map_max(kernel_map);
break; break;
} }
continue; continue;
} }
nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT, nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT,
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
VM_ALLOC_ZERO); VM_ALLOC_ZERO);
if (nkpg == NULL) if (nkpg == NULL)
panic("pmap_growkernel: no memory to grow kernel"); panic("pmap_growkernel: no memory to grow kernel");
if ((nkpg->flags & PG_ZERO) == 0) if ((nkpg->flags & PG_ZERO) == 0)
pmap_zero_page(nkpg); pmap_zero_page(nkpg);
/* See the dmb() in _pmap_alloc_l3(). */
dmb(ishst);
paddr = VM_PAGE_TO_PHYS(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg);
pmap_load_store(l2, paddr | L2_TABLE); pmap_store(l2, paddr | L2_TABLE);
pmap_invalidate_page(kernel_pmap, kernel_vm_end);
kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
kernel_vm_end = vm_map_max(kernel_map); kernel_vm_end = vm_map_max(kernel_map);
break; break;
} }
} }
} }
▲ Show 20 Lines • Show All 3,716 Lines • ▼ Show 20 Linespmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va,
newl3 = (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE; newl3 = (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE;
KASSERT((oldl2 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) != KASSERT((oldl2 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) !=
(ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM), (ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM),
("pmap_demote_l2: L2 entry is writeable but not dirty")); ("pmap_demote_l2: L2 entry is writeable but not dirty"));
/* /*
* If the page table page is not leftover from an earlier promotion, * If the page table page is not leftover from an earlier promotion,
* or the mapping attributes have changed, (re)initialize the L3 table. * or the mapping attributes have changed, (re)initialize the L3 table.
*
* When pmap_update_entry() clears the old L2 mapping, it (indirectly)
* performs a dsb(). That dsb() ensures that the stores for filling
* "l3" are visible before "l3" is added to the page table.
*/ */
if (ml3->valid == 0 || (l3[0] & ATTR_MASK) != (newl3 & ATTR_MASK)) if (ml3->valid == 0 || (l3[0] & ATTR_MASK) != (newl3 & ATTR_MASK))
pmap_fill_l3(l3, newl3); pmap_fill_l3(l3, newl3);
/* /*
* Map the temporary page so we don't lose access to the l2 table. * Map the temporary page so we don't lose access to the l2 table.
*/ */
if (tmpl2 != 0) { if (tmpl2 != 0) {
▲ Show 20 Lines • Show All 392 Lines • Show Last 20 Lines