diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -676,6 +676,9 @@ orr x12, x7, #L2_BLOCK orr x12, x12, #(ATTR_DEFAULT) orr x12, x12, #(ATTR_S1_UXN) +#ifdef __ARM_FEATURE_BTI_DEFAULT + orr x12, x12, #(ATTR_S1_GP) +#endif /* Only use the output address bits */ lsr x9, x9, #L2_SHIFT @@ -745,6 +748,9 @@ orr x12, x7, #L3_PAGE orr x12, x12, #(ATTR_DEFAULT) orr x12, x12, #(ATTR_S1_UXN) +#ifdef __ARM_FEATURE_BTI_DEFAULT + orr x12, x12, #(ATTR_S1_GP) +#endif /* Only use the output address bits */ lsr x9, x9, #L3_SHIFT diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -180,7 +180,12 @@ #define pmap_l1_pindex(v) (NUL2E + ((v) >> L1_SHIFT)) #define pmap_l2_pindex(v) ((v) >> L2_SHIFT) -#define PMAP_SAN_PTE_BITS (ATTR_DEFAULT | ATTR_S1_XN | \ +#ifdef __ARM_FEATURE_BTI_DEFAULT +#define ATTR_KERN_GP ATTR_S1_GP +#else +#define ATTR_KERN_GP 0 +#endif +#define PMAP_SAN_PTE_BITS (ATTR_DEFAULT | ATTR_S1_XN | ATTR_KERN_GP | \ ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_AP(ATTR_S1_AP_RW)) struct pmap_large_md_page { @@ -474,6 +479,8 @@ static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); +static pt_entry_t pmap_pte_bti(pmap_t pmap, vm_offset_t va); + /* * These load the old table data and store the new value. * They need to be atomic as the System MMU may write to the table at @@ -1080,7 +1087,7 @@ MPASS((state->pa & L2_OFFSET) == 0); MPASS(state->l2[l2_slot] == 0); pmap_store(&state->l2[l2_slot], PHYS_TO_PTE(state->pa) | - ATTR_DEFAULT | ATTR_S1_XN | + ATTR_DEFAULT | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK); } MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); @@ -1115,7 +1122,7 @@ MPASS((state->pa & L3_OFFSET) == 0); MPASS(state->l3[l3_slot] == 0); pmap_store(&state->l3[l3_slot], PHYS_TO_PTE(state->pa) | - ATTR_DEFAULT | ATTR_S1_XN | + ATTR_DEFAULT | ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L3_PAGE); } MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); @@ -1156,7 +1163,7 @@ &bs_state.l1[pmap_l1_index(bs_state.va)], PHYS_TO_PTE(bs_state.pa) | ATTR_DEFAULT | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | - ATTR_S1_XN | L1_BLOCK); + ATTR_S1_XN | ATTR_KERN_GP | L1_BLOCK); } MPASS(bs_state.pa <= physmap[i + 1]); @@ -1985,7 +1992,7 @@ ("pmap_kenter: Mapping is not page-sized")); attr = ATTR_DEFAULT | ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_XN | - ATTR_S1_IDX(mode) | L3_PAGE; + ATTR_KERN_GP | ATTR_S1_IDX(mode) | L3_PAGE; old_l3e = 0; va = sva; while (size != 0) { @@ -2109,7 +2116,7 @@ m = ma[i]; pa = VM_PAGE_TO_PHYS(m); attr = ATTR_DEFAULT | ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_XN | - ATTR_S1_IDX(m->md.pv_memattr) | L3_PAGE; + ATTR_KERN_GP | ATTR_S1_IDX(m->md.pv_memattr) | L3_PAGE; pte = pmap_l2_to_l3(pde, va); old_l3e |= pmap_load_store(pte, PHYS_TO_PTE(pa) | attr); @@ -4008,6 +4015,10 @@ mask |= ATTR_S1_XN; nbits |= ATTR_S1_XN; } + if (pmap == kernel_pmap) { + mask |= ATTR_KERN_GP; + nbits |= ATTR_KERN_GP; + } if (mask == 0) return; @@ -4436,7 +4447,6 @@ new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_DEFAULT | L3_PAGE); new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr); new_l3 |= pmap_pte_prot(pmap, prot); - if ((flags & PMAP_ENTER_WIRED) != 0) new_l3 |= ATTR_SW_WIRED; if (pmap->pm_stage == PM_STAGE1) { @@ -4478,6 +4488,9 @@ lock = NULL; PMAP_LOCK(pmap); + /* Wait until we lock the pmap to protect the bti rangeset */ + new_l3 |= pmap_pte_bti(pmap, va); + if ((flags & PMAP_ENTER_LARGEPAGE) != 0) { KASSERT((m->oflags & VPO_UNMANAGED) != 0, ("managed largepage va %#lx flags %#x", va, flags)); @@ -4746,6 +4759,7 @@ new_l2 = (pd_entry_t)(PHYS_TO_PTE(VM_PAGE_TO_PHYS(m)) | ATTR_DEFAULT | ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | L2_BLOCK); + new_l2 |= pmap_pte_bti(pmap, va); if ((m->oflags & VPO_UNMANAGED) == 0) { new_l2 |= ATTR_SW_MANAGED; new_l2 &= ~ATTR_AF; @@ -5117,6 +5131,7 @@ pa = VM_PAGE_TO_PHYS(m); l3_val = PHYS_TO_PTE(pa) | ATTR_DEFAULT | ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | L3_PAGE; + l3_val |= pmap_pte_bti(pmap, va); if ((prot & VM_PROT_EXECUTE) == 0 || m->md.pv_memattr == VM_MEMATTR_DEVICE) l3_val |= ATTR_S1_XN; @@ -6562,7 +6577,8 @@ l2 = pmap_l1_to_l2(pde, va); old_l2e |= pmap_load_store(l2, PHYS_TO_PTE(pa) | ATTR_DEFAULT | ATTR_S1_XN | - ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK); + ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | + L2_BLOCK); va += L2_SIZE; pa += L2_SIZE; @@ -7834,6 +7850,19 @@ return (mode >= VM_MEMATTR_DEVICE && mode <= VM_MEMATTR_WRITE_THROUGH); } +static pt_entry_t +pmap_pte_bti(pmap_t pmap, vm_offset_t va __diagused) +{ + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + MPASS(ADDR_IS_CANONICAL(va)); + + if (pmap->pm_stage != PM_STAGE1) + return (0); + if (pmap == kernel_pmap) + return (ATTR_KERN_GP); + return (0); +} + #if defined(KASAN) static vm_paddr_t pmap_san_early_kernstart; static pd_entry_t *pmap_san_early_l2; @@ -8029,12 +8058,13 @@ break; } - sbuf_printf(sb, "0x%016lx-0x%016lx r%c%c%c%c %6s %d %d %d %d\n", + sbuf_printf(sb, "0x%016lx-0x%016lx r%c%c%c%c%c %6s %d %d %d %d\n", range->sva, eva, (range->attrs & ATTR_S1_AP_RW_BIT) == ATTR_S1_AP_RW ? 'w' : '-', (range->attrs & ATTR_S1_PXN) != 0 ? '-' : 'x', (range->attrs & ATTR_S1_UXN) != 0 ? '-' : 'X', (range->attrs & ATTR_S1_AP(ATTR_S1_AP_USER)) != 0 ? 'u' : 's', + (range->attrs & ATTR_S1_GP) != 0 ? 'g' : '-', mode, range->l1blocks, range->l2blocks, range->l3contig, range->l3pages); @@ -8084,7 +8114,8 @@ static pt_entry_t sysctl_kmaps_block_attrs(pt_entry_t block) { - return (block & (ATTR_S1_AP_MASK | ATTR_S1_XN | ATTR_S1_IDX_MASK)); + return (block & (ATTR_S1_AP_MASK | ATTR_S1_XN | ATTR_S1_IDX_MASK | + ATTR_S1_GP)); } /* diff --git a/sys/arm64/include/pte.h b/sys/arm64/include/pte.h --- a/sys/arm64/include/pte.h +++ b/sys/arm64/include/pte.h @@ -80,6 +80,7 @@ #define ATTR_CONTIGUOUS (1UL << 52) #define ATTR_DBM (1UL << 51) +#define ATTR_S1_GP (1UL << 50) #define ATTR_S1_nG (1 << 11) #define ATTR_AF (1 << 10) #define ATTR_SH(x) ((x) << 8)