Index: sys/arm64/arm64/machdep.c =================================================================== --- sys/arm64/arm64/machdep.c +++ sys/arm64/arm64/machdep.c @@ -84,6 +84,9 @@ #include #endif +#define MAX_DMAP_ENTRIES 64 + +struct direct_map_desc arm64_dmap_desc[MAX_DMAP_ENTRIES]; struct pcpu __pcpu[MAXCPU]; static struct trapframe proc0_tf; @@ -794,6 +797,8 @@ vm_offset_t lastaddr; caddr_t kmdp; vm_paddr_t mem_len; + vm_offset_t dmap_address; + int dmap_cnt; int i; /* Set the module data location */ @@ -820,12 +825,28 @@ MODINFO_METADATA | MODINFOMD_EFI_MAP); add_efi_map_entries(efihdr, physmap, &physmap_idx); - /* Print the memory map */ + /* Load the memory map and prepare DMAP descriptor */ mem_len = 0; - for (i = 0; i < physmap_idx; i += 2) { + dmap_address = DMAP_MIN_ADDRESS; + for (i = 0, dmap_cnt = 0; i < physmap_idx; i += 2, dmap_cnt++) { + + /* The last entry must be zero, so decrease the limit by 1 */ + if (dmap_cnt >= (MAX_DMAP_ENTRIES - 1)) + panic("Physmap is too fragmented to fit DMAP"); + dump_avail[i] = physmap[i]; dump_avail[i + 1] = physmap[i + 1]; mem_len += physmap[i + 1] - physmap[i]; + + arm64_dmap_desc[dmap_cnt].va_start = dmap_address; + arm64_dmap_desc[dmap_cnt].va_end = dmap_address + mem_len - 1; + arm64_dmap_desc[dmap_cnt].pa_start = physmap[i]; + arm64_dmap_desc[dmap_cnt].pa_end = physmap[i + 1] - 1; + arm64_dmap_desc[dmap_cnt].flags = DMAP_FLAG_VALID; + dmap_address += mem_len; + + if (dmap_address > DMAP_MAX_ADDRESS) + panic("Physmap is too large to fit DMAP"); } dump_avail[i] = 0; dump_avail[i + 1] = 0; Index: sys/arm64/arm64/pmap.c =================================================================== --- sys/arm64/arm64/pmap.c +++ sys/arm64/arm64/pmap.c @@ -452,12 +452,14 @@ vm_paddr_t pa; pd_entry_t *l1; u_int l1_slot; + struct direct_map_desc *desc = arm64_dmap_desc; - va = DMAP_MIN_ADDRESS; + while (desc->flags == DMAP_FLAG_VALID) { + va = desc->va_start; l1 = (pd_entry_t *)l1pt; - l1_slot = pmap_l1_index(DMAP_MIN_ADDRESS); + l1_slot = pmap_l1_index(va); - for (pa = 0; va < DMAP_MAX_ADDRESS; + for (pa = desc->pa_start; va < desc->va_end; pa += L1_SIZE, va += L1_SIZE, l1_slot++) { KASSERT(l1_slot < Ln_ENTRIES, ("Invalid L1 index")); @@ -467,6 +469,9 @@ } cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); + + desc++; + } cpu_tlb_flushID(); } @@ -856,7 +861,7 @@ pt_entry_t *l3; vm_paddr_t pa; - if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { + if (VIRT_IN_DMAP(va)) { pa = DMAP_TO_PHYS(va); } else { l2 = pmap_l2(kernel_pmap, va); @@ -3120,7 +3125,7 @@ needs_mapping = FALSE; for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); - if (__predict_false(paddr >= DMAP_MAX_PHYSADDR)) { + if (__predict_false(PHYS_IN_DMAP(paddr) == 0)) { error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, &vaddr[i]); KASSERT(error == 0, ("vmem_alloc failed: %d", error)); @@ -3138,7 +3143,7 @@ sched_pin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); - if (paddr >= DMAP_MAX_PHYSADDR) { + if (PHYS_IN_DMAP(paddr) == 0) { panic( "pmap_map_io_transient: TODO: Map out of DMAP data"); } @@ -3158,7 +3163,7 @@ sched_unpin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); - if (paddr >= DMAP_MAX_PHYSADDR) { + if (PHYS_IN_DMAP(paddr) == 0) { panic("ARM64TODO: pmap_unmap_io_transient: Unmap data"); } } Index: sys/arm64/include/vmparam.h =================================================================== --- sys/arm64/include/vmparam.h +++ sys/arm64/include/vmparam.h @@ -149,6 +149,19 @@ * VM_MIN_USER_ADDRESS and VM_MAX_USER_ADDRESS define the start and end of the * user address space. */ + +#define DMAP_FLAG_VALID (0x1) + +struct direct_map_desc { + uint64_t pa_start; + uint64_t pa_end; + uint64_t va_start; + uint64_t va_end; + uint64_t flags; +}; + +extern struct direct_map_desc arm64_dmap_desc[]; + #define VM_MIN_ADDRESS (0x0000000000000000UL) #define VM_MAX_ADDRESS (0xffffffffffffffffUL) @@ -156,34 +169,77 @@ #define VM_MIN_KERNEL_ADDRESS (0xffffff8000000000UL) #define VM_MAX_KERNEL_ADDRESS (0xffffff8800000000UL) -/* Direct Map for 128 GiB of PA: 0x0 - 0x1fffffffff */ +/* VA Addresses of Direct Map for 128 GiB */ #define DMAP_MIN_ADDRESS (0xffffffc000000000UL) #define DMAP_MAX_ADDRESS (0xffffffdfffffffffUL) -#define DMAP_MIN_PHYSADDR (0x0000000000000000UL) -#define DMAP_MAX_PHYSADDR (DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) - -/* True if pa is in the dmap range */ -#define PHYS_IN_DMAP(pa) ((pa) <= DMAP_MAX_PHYSADDR) -/* True if va is in the dmap range */ -#define VIRT_IN_DMAP(va) ((va) >= DMAP_MIN_ADDRESS && \ - (va) <= DMAP_MAX_ADDRESS) - -#define PHYS_TO_DMAP(pa) \ -({ \ - KASSERT(PHYS_IN_DMAP(pa), \ - ("%s: PA out of range, PA: 0x%lx", __func__, \ - (vm_paddr_t)(pa))); \ - (pa) | DMAP_MIN_ADDRESS; \ -}) - -#define DMAP_TO_PHYS(va) \ -({ \ - KASSERT(VIRT_IN_DMAP(va), \ - ("%s: VA out of range, VA: 0x%lx", __func__, \ - (vm_offset_t)(va))); \ - (va) & ~DMAP_MIN_ADDRESS; \ -}) +static inline int +PHYS_IN_DMAP(uint64_t pa) +{ + struct direct_map_desc *desc = arm64_dmap_desc; + + while (desc->flags == DMAP_FLAG_VALID) { + if (pa >= desc->pa_start && pa <= desc->pa_end) + return (1); + desc++; + } + + return (0); +} + +static inline int +VIRT_IN_DMAP(uint64_t va) +{ + struct direct_map_desc *desc = arm64_dmap_desc; + + while (desc->flags == DMAP_FLAG_VALID) { + if (va >= desc->va_start && va <= desc->va_end) + return (1); + desc++; + } + + return (0); +} + +static inline uint64_t +PHYS_TO_DMAP(uint64_t pa) +{ + struct direct_map_desc *desc = arm64_dmap_desc; + + KASSERT(PHYS_IN_DMAP(pa), ("%s: PA out of range, PA: 0x%lx", + __func__, (vm_paddr_t)(pa))); + + while (desc->flags == DMAP_FLAG_VALID) { + if (pa >= desc->pa_start && pa <= desc->pa_end) { + uint64_t offset; + offset = pa - desc->pa_start; + return (desc->va_start + offset); + } + desc++; + } + + return (0); +} + +static inline uint64_t +DMAP_TO_PHYS(uint64_t va) +{ + struct direct_map_desc *desc = arm64_dmap_desc; + + KASSERT(VIRT_IN_DMAP(va), ("%s: VA out of range, VA: 0x%lx", + __func__, (vm_paddr_t)(va))); + + while (desc->flags == DMAP_FLAG_VALID) { + if (va >= desc->va_start && va <= desc->va_end) { + uint64_t offset; + offset = va - desc->va_start; + return (desc->pa_start + offset); + } + desc++; + } + + return (0); +} #define VM_MIN_USER_ADDRESS (0x0000000000000000UL) #define VM_MAX_USER_ADDRESS (0x0000008000000000UL)