Index: sys/arm64/arm64/locore.S =================================================================== --- sys/arm64/arm64/locore.S +++ sys/arm64/arm64/locore.S @@ -38,7 +38,6 @@ #include #define VIRT_BITS 48 -#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) .globl kernbase .set kernbase, KERNBASE @@ -378,7 +377,6 @@ * - The Kernel L0 table (TTBR1) * - The identity (PA = VA) L1 table * - The identity (PA = VA) L0 table (TTBR0) - * - The DMAP L1 tables */ LENTRY(create_pagetables) /* Save the Link register */ @@ -476,13 +474,6 @@ mov x10, #1 bl link_l0_pagetable - /* Link the DMAP tables */ - ldr x8, =DMAP_MIN_ADDRESS - adrp x9, pagetable_dmap - add x9, x9, :lo12:pagetable_dmap - mov x10, #DMAP_TABLES - bl link_l0_pagetable - /* * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. * They are only needed early on, so the VA = PA map is uncached. @@ -771,6 +762,7 @@ * L0 bootstrap for user * L0 for user */ + .globl pagetable_l0_ttbr1 pagetable: .space PAGE_SIZE pagetable_l1_ttbr1: @@ -786,9 +778,6 @@ pagetable_l0_ttbr0: .space PAGE_SIZE - .globl pagetable_dmap -pagetable_dmap: - .space PAGE_SIZE * DMAP_TABLES pagetable_end: el2_pagetable: Index: sys/arm64/arm64/pmap.c =================================================================== --- sys/arm64/arm64/pmap.c +++ sys/arm64/arm64/pmap.c @@ -290,8 +290,7 @@ CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS); CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS); -#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) -extern pt_entry_t pagetable_dmap[]; +extern pt_entry_t pagetable_l0_ttbr1[]; #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) static vm_paddr_t physmap[PHYSMAP_SIZE]; @@ -807,104 +806,139 @@ return (pa_page | (va & PAR_LOW_MASK)); } -static vm_offset_t -pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, u_int *prev_l1_slot, - pt_entry_t **l2p, int i, vm_offset_t freemempos) +/* State of the bootstrapped DMAP page tables */ +struct dmap_bootstrap_state { + vm_offset_t va; + vm_paddr_t pa; + pt_entry_t *l1; + pt_entry_t *l2; + u_int l0_slot; + u_int l1_slot; + vm_offset_t freemempos; +}; + +static void +pmap_bootstrap_dmap_l1(struct dmap_bootstrap_state *state) +{ + vm_paddr_t l1_pa; + u_int l0_slot; + + /* Link the level 0 table to a level 1 table */ + l0_slot = pmap_l0_index(state->va); + if (l0_slot != state->l0_slot) { + state->l0_slot = l0_slot; + state->l1 = (pt_entry_t *)state->freemempos; + state->l2 = NULL; + state->l1_slot = Ln_ENTRIES; + state->freemempos += PAGE_SIZE; + + l1_pa = pmap_early_vtophys((vm_offset_t)state->l1); + pmap_store(&pagetable_l0_ttbr1[l0_slot], + (l1_pa & ~Ln_TABLE_MASK) | + TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0 | L0_TABLE); + + memset(state->l1, 0, PAGE_SIZE); + } + KASSERT(state->l1 != NULL, + ("pmap_bootstrap_dmap: NULL l1 map")); +} + +static void +pmap_bootstrap_dmap_l2(struct dmap_bootstrap_state *state, int i) { - pt_entry_t *l2; vm_paddr_t l2_pa; u_int l1_slot, l2_slot; bool first; - l2 = *l2p; - l1_slot = ((*va - DMAP_MIN_ADDRESS) >> L1_SHIFT); - if (l1_slot != *prev_l1_slot) { - *prev_l1_slot = l1_slot; - l2 = (pt_entry_t *)freemempos; - l2_pa = pmap_early_vtophys((vm_offset_t)l2); - freemempos += PAGE_SIZE; + /* Make sure there is a valid L1 table */ + pmap_bootstrap_dmap_l1(state); + + l1_slot = pmap_l1_index(state->va); + if (l1_slot != state->l1_slot) { + state->l1_slot = l1_slot; + state->l2 = (pt_entry_t *)state->freemempos; + l2_pa = pmap_early_vtophys((vm_offset_t)state->l2); + state->freemempos += PAGE_SIZE; - pmap_store(&pagetable_dmap[l1_slot], + pmap_store(&state->l1[l1_slot], (l2_pa & ~Ln_TABLE_MASK) | TATTR_PXN_TABLE | L1_TABLE); - memset(l2, 0, PAGE_SIZE); + memset(state->l2, 0, PAGE_SIZE); } - KASSERT(l2 != NULL, + KASSERT(state->l2 != NULL, ("pmap_bootstrap_dmap_l2: NULL l2 map")); - for (first = true; *va < DMAP_MAX_ADDRESS && *pa < physmap[i + 1]; - *pa += L2_SIZE, *va += L2_SIZE) { + for (first = true; + state->va < DMAP_MAX_ADDRESS && state->pa < physmap[i + 1]; + state->va += L2_SIZE, state->pa += L2_SIZE) { /* * Stop if we are about to walk off the end of what the * current L1 slot can address. */ - if (!first && (*pa & L1_OFFSET) == 0) + if (!first && (state->pa & L1_OFFSET) == 0) break; first = false; - l2_slot = pmap_l2_index(*va); - pmap_store(&l2[l2_slot], - (*pa & ~L2_OFFSET) | ATTR_DEFAULT | + l2_slot = pmap_l2_index(state->va); + pmap_store(&state->l2[l2_slot], + (state->pa & ~L2_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK); } - MPASS(*va == (*pa - dmap_phys_base + DMAP_MIN_ADDRESS)); - *l2p = l2; - - return (freemempos); + MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); } static vm_offset_t pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa, - vm_offset_t freemempos) + vm_offset_t _freemempos) { - pt_entry_t *l2; - vm_offset_t va; - vm_paddr_t pa; - u_int l1_slot, prev_l1_slot; + struct dmap_bootstrap_state state; int i; dmap_phys_base = min_pa & ~L1_OFFSET; dmap_phys_max = 0; dmap_max_addr = 0; - l2 = NULL; - prev_l1_slot = -1; + + state.l1 = state.l2 = NULL; + state.l0_slot = L0_ENTRIES; + state.l1_slot = Ln_ENTRIES; + state.freemempos = _freemempos; for (i = 0; i < (physmap_idx * 2); i += 2) { - pa = physmap[i] & ~L2_OFFSET; - va = pa - dmap_phys_base + DMAP_MIN_ADDRESS; + state.pa = physmap[i] & ~L2_OFFSET; + state.va = state.pa - dmap_phys_base + DMAP_MIN_ADDRESS; /* Create L2 mappings at the start of the region */ - if ((pa & L1_OFFSET) != 0) { - freemempos = pmap_bootstrap_dmap_l2(&va, &pa, - &prev_l1_slot, &l2, i, freemempos); - } - - for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] && - (physmap[i + 1] - pa) >= L1_SIZE; - pa += L1_SIZE, va += L1_SIZE) { - l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT); - pmap_store(&pagetable_dmap[l1_slot], - (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN | - ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L1_BLOCK); + if ((state.pa & L1_OFFSET) != 0) + pmap_bootstrap_dmap_l2(&state, i); + + /* Create the main L1 block mappings */ + for (; state.va < DMAP_MAX_ADDRESS && + state.pa < physmap[i + 1] && + (physmap[i + 1] - state.pa) >= L1_SIZE; + state.va += L1_SIZE, state.pa += L1_SIZE) { + /* Make sure there is a valid L1 table */ + pmap_bootstrap_dmap_l1(&state); + pmap_store(&state.l1[pmap_l1_index(state.va)], + (state.pa & ~L1_OFFSET) | ATTR_DEFAULT | + ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | + L1_BLOCK); } /* Create L2 mappings at the end of the region */ - if (pa < physmap[i + 1]) { - freemempos = pmap_bootstrap_dmap_l2(&va, &pa, - &prev_l1_slot, &l2, i, freemempos); - } + if (state.pa < physmap[i + 1]) + pmap_bootstrap_dmap_l2(&state, i); - if (pa > dmap_phys_max) { - dmap_phys_max = pa; - dmap_max_addr = va; + if (state.pa > dmap_phys_max) { + dmap_phys_max = state.pa; + dmap_max_addr = state.va; } } cpu_tlb_flushID(); - return (freemempos); + return (state.freemempos); } static vm_offset_t