Index: sys/arm64/arm64/locore.S =================================================================== --- sys/arm64/arm64/locore.S +++ sys/arm64/arm64/locore.S @@ -38,7 +38,6 @@ #include #define VIRT_BITS 48 -#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) .globl kernbase .set kernbase, KERNBASE @@ -378,7 +377,6 @@ * - The Kernel L0 table (TTBR1) * - The identity (PA = VA) L1 table * - The identity (PA = VA) L0 table (TTBR0) - * - The DMAP L1 tables */ LENTRY(create_pagetables) /* Save the Link register */ @@ -476,13 +474,6 @@ mov x10, #1 bl link_l0_pagetable - /* Link the DMAP tables */ - ldr x8, =DMAP_MIN_ADDRESS - adrp x9, pagetable_dmap - add x9, x9, :lo12:pagetable_dmap - mov x10, #DMAP_TABLES - bl link_l0_pagetable - /* * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. * They are only needed early on, so the VA = PA map is uncached. @@ -771,6 +762,7 @@ * L0 bootstrap for user * L0 for user */ + .globl pagetable_l0_ttbr1 pagetable: .space PAGE_SIZE pagetable_l1_ttbr1: @@ -786,9 +778,6 @@ pagetable_l0_ttbr0: .space PAGE_SIZE - .globl pagetable_dmap -pagetable_dmap: - .space PAGE_SIZE * DMAP_TABLES pagetable_end: el2_pagetable: Index: sys/arm64/arm64/pmap.c =================================================================== --- sys/arm64/arm64/pmap.c +++ sys/arm64/arm64/pmap.c @@ -290,8 +290,7 @@ CTASSERT((DMAP_MIN_ADDRESS & ~L0_OFFSET) == DMAP_MIN_ADDRESS); CTASSERT((DMAP_MAX_ADDRESS & ~L0_OFFSET) == DMAP_MAX_ADDRESS); -#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) -extern pt_entry_t pagetable_dmap[]; +extern pt_entry_t pagetable_l0_ttbr1[]; #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) static vm_paddr_t physmap[PHYSMAP_SIZE]; @@ -808,8 +807,8 @@ } static vm_offset_t -pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, u_int *prev_l1_slot, - pt_entry_t **l2p, int i, vm_offset_t freemempos) +pmap_bootstrap_dmap_l2(vm_offset_t *va, vm_paddr_t *pa, pt_entry_t *l1, + u_int *prev_l1_slot, pt_entry_t **l2p, int i, vm_offset_t freemempos) { pt_entry_t *l2; vm_paddr_t l2_pa; @@ -817,14 +816,14 @@ bool first; l2 = *l2p; - l1_slot = ((*va - DMAP_MIN_ADDRESS) >> L1_SHIFT); + l1_slot = pmap_l1_index(*va); if (l1_slot != *prev_l1_slot) { *prev_l1_slot = l1_slot; l2 = (pt_entry_t *)freemempos; l2_pa = pmap_early_vtophys((vm_offset_t)l2); freemempos += PAGE_SIZE; - pmap_store(&pagetable_dmap[l1_slot], + pmap_store(&l1[l1_slot], (l2_pa & ~Ln_TABLE_MASK) | TATTR_PXN_TABLE | L1_TABLE); @@ -859,40 +858,63 @@ pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t min_pa, vm_offset_t freemempos) { - pt_entry_t *l2; + pt_entry_t *l1, *l2; vm_offset_t va; - vm_paddr_t pa; - u_int l1_slot, prev_l1_slot; + vm_paddr_t l1_pa, pa; + u_int l0_slot, prev_l0_slot, prev_l1_slot; int i; dmap_phys_base = min_pa & ~L1_OFFSET; dmap_phys_max = 0; dmap_max_addr = 0; + l1 = NULL; l2 = NULL; + prev_l0_slot = -1; prev_l1_slot = -1; for (i = 0; i < (physmap_idx * 2); i += 2) { pa = physmap[i] & ~L2_OFFSET; va = pa - dmap_phys_base + DMAP_MIN_ADDRESS; + /* Link the level 0 table to a level 1 table */ + l0_slot = pmap_l0_index(va); + if (l0_slot != prev_l0_slot) { + prev_l0_slot = l0_slot; + l1 = (pt_entry_t *)freemempos; + freemempos += PAGE_SIZE; + + /* Reset the state so we create the l1 entry */ + l2 = NULL; + prev_l1_slot = -1; + + l1_pa = pmap_early_vtophys((vm_offset_t)l1); + pmap_store(&pagetable_l0_ttbr1[l0_slot], + (l1_pa & ~Ln_TABLE_MASK) | + TATTR_PXN_TABLE | TATTR_UXN_TABLE | + TATTR_AP_TABLE_NO_EL0 | L0_TABLE); + + memset(l1, 0, PAGE_SIZE); + } + KASSERT(l1 != NULL, + ("pmap_bootstrap_dmap: NULL l1 map")); + /* Create L2 mappings at the start of the region */ if ((pa & L1_OFFSET) != 0) { - freemempos = pmap_bootstrap_dmap_l2(&va, &pa, + freemempos = pmap_bootstrap_dmap_l2(&va, &pa, l1, &prev_l1_slot, &l2, i, freemempos); } for (; va < DMAP_MAX_ADDRESS && pa < physmap[i + 1] && (physmap[i + 1] - pa) >= L1_SIZE; pa += L1_SIZE, va += L1_SIZE) { - l1_slot = ((va - DMAP_MIN_ADDRESS) >> L1_SHIFT); - pmap_store(&pagetable_dmap[l1_slot], + pmap_store(&l1[pmap_l1_index(va)], (pa & ~L1_OFFSET) | ATTR_DEFAULT | ATTR_S1_XN | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L1_BLOCK); } /* Create L2 mappings at the end of the region */ if (pa < physmap[i + 1]) { - freemempos = pmap_bootstrap_dmap_l2(&va, &pa, + freemempos = pmap_bootstrap_dmap_l2(&va, &pa, l1, &prev_l1_slot, &l2, i, freemempos); }