diff --git a/sys/arm64/arm64/efirt_machdep.c b/sys/arm64/arm64/efirt_machdep.c --- a/sys/arm64/arm64/efirt_machdep.c +++ b/sys/arm64/arm64/efirt_machdep.c @@ -214,7 +214,7 @@ p->md_phys, mode, p->md_pages); } - l3_attr = ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_IDX(mode) | + l3_attr = ATTR_AF | pmap_sh_attr | ATTR_S1_IDX(mode) | ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_nG | L3_PAGE; if (mode == VM_MEMATTR_DEVICE || p->md_attr & EFI_MD_ATTR_XP) l3_attr |= ATTR_S1_XN; diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -86,6 +86,7 @@ * x27 = TTBR0 table * x26 = Kernel L1 table * x24 = TTBR1 table + * x22 = PTE shareability attributes */ /* Enable the mmu */ @@ -135,6 +136,10 @@ str x27, [x0, #BP_KERN_TTBR0] str x23, [x0, #BP_BOOT_EL] + /* Set this before it's used in kasan_init_early */ + adrp x1, pmap_sh_attr + str x22, [x1, :lo12:pmap_sh_attr] + #ifdef KASAN /* Save bootparams */ mov x19, x0 @@ -476,6 +481,30 @@ cmp x6, x27 b.lo 1b + /* + * Find the shareability attribute we should use. If FEAT_LPA2 is + * enabled then the shareability field is moved from the page table + * to tcr_el1 and the bits in the page table are reused by the + * address field. + */ +#if PAGE_SIZE == PAGE_SIZE_4K +#define LPA2_MASK ID_AA64MMFR0_TGran4_MASK +#define LPA2_VAL ID_AA64MMFR0_TGran4_LPA2 +#elif PAGE_SIZE == PAGE_SIZE_16K +#define LPA2_MASK ID_AA64MMFR0_TGran16_MASK +#define LPA2_VAL ID_AA64MMFR0_TGran16_LPA2 +#else +#error Unsupported page size +#endif + mrs x6, id_aa64mmfr0_el1 + mov x7, LPA2_VAL + and x6, x6, LPA2_MASK + cmp x6, x7 + ldr x22, =(ATTR_SH(ATTR_SH_IS)) + csel x22, xzr, x22, eq +#undef LPA2_MASK +#undef LPA2_VAL + /* * Build the TTBR1 maps. */ @@ -747,11 +776,13 @@ /* Build the L2 block entry */ orr x12, x7, #L2_BLOCK - orr x12, x12, #(ATTR_AF | ATTR_SH(ATTR_SH_IS)) + orr x12, x12, #(ATTR_AF) orr x12, x12, #(ATTR_S1_UXN) #ifdef __ARM_FEATURE_BTI_DEFAULT orr x12, x12, #(ATTR_S1_GP) #endif + /* Set the shareability attribute */ + orr x12, x12, x22 /* Only use the output address bits */ lsr x9, x9, #L2_SHIFT @@ -823,11 +854,13 @@ /* Build the L3 page entry */ orr x12, x7, #L3_PAGE - orr x12, x12, #(ATTR_AF | ATTR_SH(ATTR_SH_IS)) + orr x12, x12, #(ATTR_AF) orr x12, x12, #(ATTR_S1_UXN) #ifdef __ARM_FEATURE_BTI_DEFAULT orr x12, x12, #(ATTR_S1_GP) #endif + /* Set the shareability attribute */ + orr x12, x12, x22 /* Only use the output address bits */ lsr x9, x9, #L3_SHIFT @@ -886,6 +919,13 @@ * to 1 only if the ASIDBits field equals 0b0010. */ ldr x2, tcr + + /* If x22 contains a non-zero value then LPA2 is not implemented */ + cbnz x22, .Lno_lpa2 + ldr x3, =(TCR_DS) + orr x2, x2, x3 +.Lno_lpa2: + mrs x3, id_aa64mmfr0_el1 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ diff --git a/sys/arm64/arm64/minidump_machdep.c b/sys/arm64/arm64/minidump_machdep.c --- a/sys/arm64/arm64/minidump_machdep.c +++ b/sys/arm64/arm64/minidump_machdep.c @@ -311,7 +311,7 @@ for (j = 0; j < Ln_ENTRIES; j++) { tmpbuffer[j] = (pa + i * L2_SIZE + j * PAGE_SIZE) | ATTR_AF | - ATTR_SH(ATTR_SH_IS) | L3_PAGE; + pmap_sh_attr | L3_PAGE; } error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); @@ -330,7 +330,7 @@ /* Generate fake l3 entries based upon the l1 entry */ for (i = 0; i < Ln_ENTRIES; i++) { tmpbuffer[i] = (pa + i * PAGE_SIZE) | - ATTR_AF | ATTR_SH(ATTR_SH_IS) | L3_PAGE; + ATTR_AF | pmap_sh_attr | L3_PAGE; } error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE); if (error) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -185,7 +185,7 @@ #else #define ATTR_KERN_GP 0 #endif -#define PMAP_SAN_PTE_BITS (ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_XN | \ +#define PMAP_SAN_PTE_BITS (ATTR_AF | ATTR_S1_XN | pmap_sh_attr | \ ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_AP(ATTR_S1_AP_RW)) struct pmap_large_md_page { @@ -355,6 +355,8 @@ static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "VM/pmap parameters"); +pt_entry_t pmap_sh_attr __read_mostly = ATTR_SH(ATTR_SH_IS); + #if PAGE_SIZE == PAGE_SIZE_4K #define L1_BLOCKS_SUPPORTED 1 #else @@ -1150,7 +1152,7 @@ MPASS((state->pa & L2_OFFSET) == 0); MPASS(state->l2[l2_slot] == 0); pmap_store(&state->l2[l2_slot], PHYS_TO_PTE(state->pa) | - ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_XN | ATTR_KERN_GP | + ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | contig | L2_BLOCK); } MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); @@ -1200,7 +1202,7 @@ MPASS((state->pa & L3_OFFSET) == 0); MPASS(state->l3[l3_slot] == 0); pmap_store(&state->l3[l3_slot], PHYS_TO_PTE(state->pa) | - ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_XN | ATTR_KERN_GP | + ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | contig | L3_PAGE); } MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); @@ -1243,7 +1245,7 @@ pmap_store( &bs_state.l1[pmap_l1_index(bs_state.va)], PHYS_TO_PTE(bs_state.pa) | ATTR_AF | - ATTR_SH(ATTR_SH_IS) | + pmap_sh_attr | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN | ATTR_KERN_GP | L1_BLOCK); } @@ -2112,7 +2114,7 @@ KASSERT((size & PAGE_MASK) == 0, ("pmap_kenter: Mapping is not page-sized")); - attr = ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_AP(ATTR_S1_AP_RW) | + attr = ATTR_AF | pmap_sh_attr | ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(mode); old_l3e = 0; va = sva; @@ -2327,7 +2329,7 @@ ("pmap_qenter: Invalid level %d", lvl)); m = ma[i]; - attr = ATTR_AF | ATTR_SH(ATTR_SH_IS) | + attr = ATTR_AF | pmap_sh_attr | ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(m->md.pv_memattr) | L3_PAGE; pte = pmap_l2_to_l3(pde, va); @@ -5124,7 +5126,7 @@ if ((m->oflags & VPO_UNMANAGED) == 0) VM_PAGE_OBJECT_BUSY_ASSERT(m); pa = VM_PAGE_TO_PHYS(m); - new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_AF | ATTR_SH(ATTR_SH_IS) | + new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr | L3_PAGE); new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr); new_l3 |= pmap_pte_prot(pmap, prot); @@ -5468,7 +5470,7 @@ KASSERT(ADDR_IS_CANONICAL(va), ("%s: Address not in canonical form: %lx", __func__, va)); - new_l2 = (pd_entry_t)(VM_PAGE_TO_PTE(m) | ATTR_SH(ATTR_SH_IS) | + new_l2 = (pd_entry_t)(VM_PAGE_TO_PTE(m) | pmap_sh_attr | ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | L2_BLOCK); if ((m->oflags & VPO_UNMANAGED) == 0) @@ -5697,7 +5699,7 @@ KASSERT(ADDR_IS_CANONICAL(va), ("%s: Address not in canonical form: %lx", __func__, va)); - l3e = VM_PAGE_TO_PTE(m) | ATTR_SH(ATTR_SH_IS) | + l3e = VM_PAGE_TO_PTE(m) | pmap_sh_attr | ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_CONTIGUOUS | L3_PAGE; if ((m->oflags & VPO_UNMANAGED) == 0) @@ -6094,7 +6096,7 @@ pmap_resident_count_inc(pmap, 1); pa = VM_PAGE_TO_PHYS(m); - l3_val = PHYS_TO_PTE(pa) | ATTR_SH(ATTR_SH_IS) | + l3_val = PHYS_TO_PTE(pa) | pmap_sh_attr | ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | L3_PAGE; l3_val |= pmap_pte_bti(pmap, va); if ((prot & VM_PROT_EXECUTE) == 0 || @@ -7744,7 +7746,7 @@ /* Insert L2_BLOCK */ l2 = pmap_l1_to_l2(pde, va); old_l2e |= pmap_load_store(l2, - PHYS_TO_PTE(pa) | ATTR_AF | ATTR_SH(ATTR_SH_IS) | + PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK); diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h --- a/sys/arm64/include/hypervisor.h +++ b/sys/arm64/include/hypervisor.h @@ -241,6 +241,8 @@ #define VTCR_EL2_PS_42BIT (0x3UL << VTCR_EL2_PS_SHIFT) #define VTCR_EL2_PS_44BIT (0x4UL << VTCR_EL2_PS_SHIFT) #define VTCR_EL2_PS_48BIT (0x5UL << VTCR_EL2_PS_SHIFT) +#define VTCR_EL2_DS_SHIFT 32 +#define VTCR_EL2_DS (0x1UL << VTCR_EL2_DS_SHIFT) /* VTTBR_EL2 - Virtualization Translation Table Base Register */ #define VTTBR_VMID_MASK 0xffff000000000000 diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h --- a/sys/arm64/include/pmap.h +++ b/sys/arm64/include/pmap.h @@ -127,6 +127,8 @@ extern vm_offset_t virtual_avail; extern vm_offset_t virtual_end; +extern pt_entry_t pmap_sh_attr; + /* * Macros to test if a mapping is mappable with an L1 Section mapping * or an L2 Large Page mapping. diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c --- a/sys/arm64/vmm/vmm_arm64.c +++ b/sys/arm64/vmm/vmm_arm64.c @@ -396,6 +396,14 @@ #ifdef SMP el2_regs.vtcr_el2 |= VTCR_EL2_SH0_IS; #endif + /* + * If FEAT_LPA2 is enabled in the host then we need to enable it here + * so the page tables created by pmap.c are correct. The meaning of + * the shareability field changes to become address bits when this + * is set. + */ + if ((READ_SPECIALREG(tcr_el1) & TCR_DS) != 0) + el2_regs.vtcr_el2 |= VTCR_EL2_DS; smp_rendezvous(NULL, arm_setup_vectors, NULL, &el2_regs);