diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -38,7 +38,6 @@ #include #define VIRT_BITS 48 -#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) .globl kernbase .set kernbase, KERNBASE @@ -378,7 +377,6 @@ * - The Kernel L0 table (TTBR1) * - The identity (PA = VA) L1 table * - The identity (PA = VA) L0 table (TTBR0) - * - The DMAP L1 tables */ LENTRY(create_pagetables) /* Save the Link register */ @@ -476,13 +474,6 @@ mov x10, #1 bl link_l0_pagetable - /* Link the DMAP tables */ - ldr x8, =DMAP_MIN_ADDRESS - adrp x9, pagetable_dmap - add x9, x9, :lo12:pagetable_dmap - mov x10, #DMAP_TABLES - bl link_l0_pagetable - /* * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. * They are only needed early on, so the VA = PA map is uncached. @@ -771,6 +762,7 @@ * L0 bootstrap for user * L0 for user */ + .globl pagetable_l0_ttbr1 pagetable: .space PAGE_SIZE pagetable_l1_ttbr1: @@ -786,9 +778,6 @@ pagetable_l0_ttbr0: .space PAGE_SIZE - .globl pagetable_dmap -pagetable_dmap: - .space PAGE_SIZE * DMAP_TABLES pagetable_end: el2_pagetable: diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -290,8 +290,7 @@ CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS); CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS); -#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) -extern pt_entry_t pagetable_dmap[]; +extern pt_entry_t pagetable_l0_ttbr1[]; #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) static vm_paddr_t physmap[PHYSMAP_SIZE]; @@ -808,104 +807,248 @@ return (pa_page | (va & PAR_LOW_MASK)); } -static vm_offset_t -pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, u_int *prev_l1_slot, - pt_entry_t **l2p, int i, vm_offset_t freemempos) +/* State of the bootstrapped DMAP page tables */ +struct dmap_bootstrap_state { + vm_offset_t va; + vm_paddr_t pa; + pt_entry_t *l1; + pt_entry_t *l2; + pt_entry_t *l3; + u_int l0_slot; + u_int l1_slot; + u_int l2_slot; + vm_offset_t freemempos; +}; + +static void +pmap_bootstrap_dmap_l0_table(struct dmap_bootstrap_state *state) +{ + vm_paddr_t l1_pa; + u_int l0_slot; + + /* Link the level 0 table to a level 1 table */ + l0_slot = pmap_l0_index(state->va); + if (l0_slot != state->l0_slot) { + MPASS(state->l0_slot < l0_slot || + state->l0_slot == L0_ENTRIES); + + /* Create a new L0 table entry */ + state->l0_slot = l0_slot; + state->l1 = (pt_entry_t *)state->freemempos; + memset(state->l1, 0, PAGE_SIZE); + state->freemempos += PAGE_SIZE; + + /* Reset lower levels */ + state->l2 = NULL; + state->l3 = NULL; + state->l1_slot = Ln_ENTRIES; + state->l2_slot = Ln_ENTRIES; + + l1_pa = pmap_early_vtophys((vm_offset_t)state->l1); + MPASS((l1_pa & Ln_TABLE_MASK) == 0); + MPASS(pagetable_l0_ttbr1[l0_slot] == 0); + pmap_store(&pagetable_l0_ttbr1[l0_slot], l1_pa | + TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0 | L0_TABLE); + } + KASSERT(state->l1 != NULL, ("%s: NULL l1", __func__)); +} + +static void +pmap_bootstrap_dmap_l1_table(struct dmap_bootstrap_state *state) { - pt_entry_t *l2; vm_paddr_t l2_pa; - u_int l1_slot, l2_slot; - bool first; + u_int l1_slot; + + /* Make sure there is a valid L0 -> L1 table */ + pmap_bootstrap_dmap_l0_table(state); + + /* Link the level 1 table to a level 2 table */ + l1_slot = pmap_l1_index(state->va); + if (l1_slot != state->l1_slot) { + MPASS(state->l1_slot < l1_slot || + state->l1_slot == Ln_ENTRIES); + + /* Create a new L1 table entry */ + state->l1_slot = l1_slot; + state->l2 = (pt_entry_t *)state->freemempos; + memset(state->l2, 0, PAGE_SIZE); + state->freemempos += PAGE_SIZE; + + /* Reset lower levels */ + state->l3 = NULL; + state->l2_slot = Ln_ENTRIES; + + l2_pa = pmap_early_vtophys((vm_offset_t)state->l2); + MPASS((l2_pa & Ln_TABLE_MASK) == 0); + MPASS(state->l1[l1_slot] == 0); + pmap_store(&state->l1[l1_slot], l2_pa | TATTR_PXN_TABLE | + L1_TABLE); + } + KASSERT(state->l2 != NULL, ("%s: NULL l2", __func__)); +} - l2 = *l2p; - l1_slot = ((*va - DMAP_MIN_ADDRESS) >> L1_SHIFT); - if (l1_slot != *prev_l1_slot) { - *prev_l1_slot = l1_slot; - l2 = (pt_entry_t *)freemempos; - l2_pa = pmap_early_vtophys((vm_offset_t)l2); - freemempos += PAGE_SIZE; +static void +pmap_bootstrap_dmap_l2_table(struct dmap_bootstrap_state *state) +{ + vm_paddr_t l3_pa; + u_int l2_slot; + + /* Make sure there is a valid L1 -> L2 table */ + pmap_bootstrap_dmap_l1_table(state); - pmap_store(&pagetable_dmap[l1_slot], - (l2_pa & ~Ln_TABLE_MASK) | - TATTR_PXN_TABLE | L1_TABLE); + /* Link the level 2 table to a level 3 table */ + l2_slot = pmap_l2_index(state->va); + if (l2_slot != state->l2_slot) { + MPASS(state->l2_slot < l2_slot || + state->l2_slot == Ln_ENTRIES); - memset(l2, 0, PAGE_SIZE); + /* Create a new L2 table entry */ + state->l2_slot = l2_slot; + state->l3 = (pt_entry_t *)state->freemempos; + memset(state->l3, 0, PAGE_SIZE); + state->freemempos += PAGE_SIZE; + + l3_pa = pmap_early_vtophys((vm_offset_t)state->l3); + MPASS((l3_pa & Ln_TABLE_MASK) == 0); + MPASS(state->l2[l2_slot] == 0); + pmap_store(&state->l2[l2_slot], l3_pa | TATTR_PXN_TABLE | + L2_TABLE); } - KASSERT(l2 != NULL, - ("pmap_bootstrap_dmap_l2: NULL l2 map")); - for (first = true; *va < DMAP_MAX_ADDRESS && *pa < physmap[i + 1]; - *pa += L2_SIZE, *va += L2_SIZE) { + KASSERT(state->l3 != NULL, ("%s: NULL l3", __func__)); +} + +static void +pmap_bootstrap_dmap_l2_block(struct dmap_bootstrap_state *state, int i) +{ + u_int l2_slot; + bool first; + + if ((physmap[i + 1] - state->pa) < L2_SIZE) + return; + + /* Make sure there is a valid L1 table */ + pmap_bootstrap_dmap_l1_table(state); + + MPASS((state->va & L2_OFFSET) == 0); + for (first = true; + state->va < DMAP_MAX_ADDRESS && + (physmap[i + 1] - state->pa) >= L2_SIZE; + state->va += L2_SIZE, state->pa += L2_SIZE) { /* * Stop if we are about to walk off the end of what the * current L1 slot can address. */ - if (!first && (*pa & L1_OFFSET) == 0) + if (!first && (state->pa & L1_OFFSET) == 0) break; first = false; - l2_slot = pmap_l2_index(*va); - pmap_store(&l2[l2_slot], - (*pa & ~L2_OFFSET) | ATTR_DEFAULT | - ATTR_S1_XN | - ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | + l2_slot = pmap_l2_index(state->va); + MPASS((state->pa & L2_OFFSET) == 0); + MPASS(state->l2[l2_slot] == 0); + pmap_store(&state->l2[l2_slot], state->pa | ATTR_DEFAULT | + ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK); } - MPASS(*va == (*pa - dmap_phys_base + DMAP_MIN_ADDRESS)); - *l2p = l2; + MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); +} - return (freemempos); +static void +pmap_bootstrap_dmap_l3_page(struct dmap_bootstrap_state *state, int i) +{ + u_int l3_slot; + bool first; + + if ((physmap[i + 1] - state->pa) < L3_SIZE) + return; + + /* Make sure there is a valid L2 table */ + pmap_bootstrap_dmap_l2_table(state); + + MPASS((state->va & L3_OFFSET) == 0); + for (first = true; + state->va < DMAP_MAX_ADDRESS && + (physmap[i + 1] - state->pa) >= L3_SIZE; + state->va += L3_SIZE, state->pa += L3_SIZE) { + /* + * Stop if we are about to walk off the end of what the + * current L2 slot can address. + */ + if (!first && (state->pa & L2_OFFSET) == 0) + break; + + first = false; + l3_slot = pmap_l3_index(state->va); + MPASS((state->pa & L3_OFFSET) == 0); + MPASS(state->l3[l3_slot] == 0); + pmap_store(&state->l3[l3_slot], state->pa | ATTR_DEFAULT | + ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | + L3_PAGE); + } + MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); } static vm_offset_t pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa, vm_offset_t freemempos) { - pt_entry_t *l2; - vm_offset_t va; - vm_paddr_t pa; - u_int l1_slot, prev_l1_slot; + struct dmap_bootstrap_state state; int i; dmap_phys_base = min_pa & ~L1_OFFSET; dmap_phys_max = 0; dmap_max_addr = 0; - l2 = NULL; - prev_l1_slot = -1; + + state.l1 = state.l2 = state.l3 = NULL; + state.l0_slot = L0_ENTRIES; + state.l1_slot = Ln_ENTRIES; + state.l2_slot = Ln_ENTRIES; + state.freemempos = freemempos; for (i = 0; i < (physmap_idx * 2); i += 2) { - pa = physmap[i] & ~L2_OFFSET; - va = pa - dmap_phys_base + DMAP_MIN_ADDRESS; + state.pa = physmap[i] & ~L3_OFFSET; + state.va = state.pa - dmap_phys_base + DMAP_MIN_ADDRESS; - /* Create L2 mappings at the start of the region */ - if ((pa & L1_OFFSET) != 0) { - freemempos = pmap_bootstrap_dmap_l2(&va, &pa, - &prev_l1_slot, &l2, i, freemempos); - } + /* Create L3 mappings at the start of the region */ + if ((state.pa & L2_OFFSET) != 0) + pmap_bootstrap_dmap_l3_page(&state, i); + MPASS(state.pa <= physmap[i + 1]); - for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] && - (physmap[i + 1] - pa) >= L1_SIZE; - pa += L1_SIZE, va += L1_SIZE) { - l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT); - pmap_store(&pagetable_dmap[l1_slot], - (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN | - ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L1_BLOCK); - } + /* Create L2 mappings at the start of the region */ + if ((state.pa & L1_OFFSET) != 0) + pmap_bootstrap_dmap_l2_block(&state, i); + MPASS(state.pa <= physmap[i + 1]); + + /* Create the main L1 block mappings */ + for (; state.va < DMAP_MAX_ADDRESS && + (physmap[i + 1] - state.pa) >= L1_SIZE; + state.va += L1_SIZE, state.pa += L1_SIZE) { + /* Make sure there is a valid L1 table */ + pmap_bootstrap_dmap_l0_table(&state); + MPASS((state.pa & L1_OFFSET) == 0); + pmap_store(&state.l1[pmap_l1_index(state.va)], + state.pa | ATTR_DEFAULT | ATTR_S1_XN | + ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | + L1_BLOCK); + } + MPASS(state.pa <= physmap[i + 1]); /* Create L2 mappings at the end of the region */ - if (pa < physmap[i + 1]) { - freemempos = pmap_bootstrap_dmap_l2(&va, &pa, - &prev_l1_slot, &l2, i, freemempos); - } + pmap_bootstrap_dmap_l2_block(&state, i); + MPASS(state.pa <= physmap[i + 1]); + + /* Create L3 mappings at the end of the region */ + pmap_bootstrap_dmap_l3_page(&state, i); + MPASS(state.pa == physmap[i + 1]); - if (pa > dmap_phys_max) { - dmap_phys_max = pa; - dmap_max_addr = va; + if (state.pa > dmap_phys_max) { + dmap_phys_max = state.pa; + dmap_max_addr = state.va; } } cpu_tlb_flushID(); - return (freemempos); + return (state.freemempos); } static vm_offset_t