Page MenuHomeFreeBSD
Files F176340765

D60090.diff
No OneTemporary
Actions

D60090.diff
View Options

diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -459,6 +459,17 @@
#define TLBI_VA_MASK ((1ul << 44) - 1)
#define TLBI_VA(addr) (((addr) >> TLBI_VA_SHIFT) & TLBI_VA_MASK)
+#define TLBI_VA_TTL_SHIFT 44
+#if PAGE_SIZE == PAGE_SIZE_4K
+#define TLBI_VA_TTL(ttl) ((((ttl) != TLBI_TTL_UNKNOWN) ? \
+ (0x4ul | (ttl)) : 0) << \
+ TLBI_VA_TTL_SHIFT)
+#elif PAGE_SIZE == PAGE_SIZE_16K
+#define TLBI_VA_TTL(ttl) ((((ttl) != TLBI_TTL_UNKNOWN) ? \
+ (0x8ul | (ttl)) : 0) << \
+ TLBI_VA_TTL_SHIFT)
+#endif
+
/*
* The operand to a range-based TLBI instruction has the following fields:
*
@@ -490,6 +501,8 @@
#define TLBI_RANGE_VA_SHIFT() (pmap_lpa_enabled ? 16 : PAGE_SHIFT)
#define TLBI_RANGE_BADDR_MASK ((1ul << 37) - 1)
+#define TLBI_RANGE_TTL_SHIFT 37
+#define TLBI_RANGE_TTL(ttl) ((u_long)(ttl) << TLBI_RANGE_TTL_SHIFT)
#define TLBI_RANGE_NUM_SHIFT 39
#define TLBI_RANGE_SCALE_SHIFT 44
#define TLBI_RANGE_TG_SHIFT 46
@@ -500,6 +513,12 @@
#define TLBI_RANGE_UNIT_SHIFT(scale) (5 * (scale) + 1)
#define TLBI_RANGE_UNIT(scale) (1ul << TLBI_RANGE_UNIT_SHIFT(scale))
+/* Will need modifying when LPA2 support added (L0) */
+#define TLBI_TTL_UNKNOWN (0ul)
+#define TLBI_TTL_L1 (1ul)
+#define TLBI_TTL_L2 (2ul)
+#define TLBI_TTL_L3 (3ul)
+
/*
* The largest scale such that a unit fits within the given number of pages,
* i.e., the largest scale such that TLBI_RANGE_UNIT(scale) <= pages. The
@@ -516,9 +535,10 @@
#error Unsupported page size
#endif
-#define TLBI_RANGE_FIELDS(va, va_shift, num, scale) \
+#define TLBI_RANGE_FIELDS(va, va_shift, num, scale, ttl) \
(TLBI_RANGE_TG | ((u_long)(scale) << TLBI_RANGE_SCALE_SHIFT) | \
((u_long)(num) << TLBI_RANGE_NUM_SHIFT) | \
+ (pmap_ttl_support ? TLBI_RANGE_TTL(ttl) : 0) | \
(((va) >> (va_shift)) & TLBI_RANGE_BADDR_MASK))
static bool __read_frequently pmap_tlbi_range_support = false;
@@ -534,6 +554,8 @@
*/
__read_mostly static bool pmap_bti_support = false;
+__read_mostly static bool pmap_ttl_support = false;
+
/*
* Internal flags for pmap_enter()'s helper functions.
*/
@@ -2130,7 +2152,7 @@
* for the specified virtual address in the given virtual address space.
*/
static __inline void
-pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)
+pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only, int ttl)
{
uint64_t r;
@@ -2138,6 +2160,8 @@
dsb(ishst);
r = TLBI_VA(va);
+ if (pmap_ttl_support)
+ r |= TLBI_VA_TTL(ttl);
if (pmap == kernel_pmap) {
pmap_s1_invalidate_kernel(r, final_only);
} else {
@@ -2160,13 +2184,15 @@
* The pmap_multiple_tlbi errata only affect broadcast invalidations.
*/
static __inline void
-pmap_s1_invalidate_page_local(pmap_t pmap, vm_offset_t va)
+pmap_s1_invalidate_page_local(pmap_t pmap, vm_offset_t va, int ttl)
{
uint64_t r;
PMAP_ASSERT_STAGE1(pmap);
dsb(nshst);
r = TLBI_VA(va);
+ if (pmap_ttl_support)
+ r |= TLBI_VA_TTL(ttl);
if (pmap == kernel_pmap) {
__asm __volatile("tlbi vaale1, %0" : : "r" (r));
} else {
@@ -2187,10 +2213,10 @@
}
static __inline void
-pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only)
+pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only, int ttl)
{
if (pmap->pm_stage == PM_STAGE1)
- pmap_s1_invalidate_page(pmap, va, final_only);
+ pmap_s1_invalidate_page(pmap, va, final_only, ttl);
else
pmap_s2_invalidate_page(pmap, va, final_only);
}
@@ -2209,7 +2235,7 @@
static __always_inline void
pmap_s1_invalidate_loop(vm_offset_t sva, vm_offset_t eva, vm_offset_t stride,
int va_shift, vm_offset_t va_mask, uint64_t asid, bool kernel,
- bool final_only)
+ bool final_only, int ttl)
{
uint64_t units;
vm_size_t pages;
@@ -2226,11 +2252,11 @@
if (kernel)
pmap_s1_invalidate_range_kernel(asid |
TLBI_RANGE_FIELDS(va, va_shift,
- units - 1, scale), final_only);
+ units - 1, scale, ttl), final_only);
else
pmap_s1_invalidate_range_user(asid |
TLBI_RANGE_FIELDS(va, va_shift,
- units - 1, scale), final_only);
+ units - 1, scale, ttl), final_only);
va += ptoa(units << unit_shift);
continue;
}
@@ -2251,7 +2277,7 @@
*/
static __inline void
pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
- vm_offset_t stride, bool final_only)
+ vm_offset_t stride, bool final_only, int ttl)
{
uint64_t asid;
vm_offset_t va_mask;
@@ -2264,11 +2290,11 @@
dsb(ishst);
if (pmap == kernel_pmap) {
pmap_s1_invalidate_loop(sva, eva, stride, va_shift, va_mask,
- 0, true, final_only);
+ 0, true, final_only, ttl);
} else {
asid = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
pmap_s1_invalidate_loop(sva, eva, stride, va_shift, va_mask,
- asid, false, final_only);
+ asid, false, final_only, ttl);
}
if (pmap_multiple_tlbi) {
dsb(ish);
@@ -2286,7 +2312,7 @@
pmap_s1_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
bool final_only)
{
- pmap_s1_invalidate_strided(pmap, sva, eva, L3_SIZE, final_only);
+ pmap_s1_invalidate_strided(pmap, sva, eva, L3_SIZE, final_only, TLBI_TTL_UNKNOWN);
}
static __inline void
@@ -2739,7 +2765,7 @@
KASSERT((pmap_load(pte) & ATTR_CONTIGUOUS) == 0,
("pmap_kremove: unexpected ATTR_CONTIGUOUS"));
pmap_clear(pte);
- pmap_s1_invalidate_page(kernel_pmap, va, true);
+ pmap_s1_invalidate_page(kernel_pmap, va, true, TLBI_TTL_L3);
}
/*
@@ -2776,7 +2802,7 @@
true);
}
pmap_clear(ptep);
- pmap_s1_invalidate_page(kernel_pmap, va, true);
+ pmap_s1_invalidate_page(kernel_pmap, va, true, TLBI_TTL_L2);
PMAP_LOCK(kernel_pmap);
pmap_remove_kernel_l2(kernel_pmap, ptep, va);
PMAP_UNLOCK(kernel_pmap);
@@ -2991,7 +3017,7 @@
l1pg = PTE_TO_VM_PAGE(tl0);
pmap_unwire_l3(pmap, va, l1pg, free);
}
- pmap_invalidate_page(pmap, va, false);
+ pmap_invalidate_page(pmap, va, false, TLBI_TTL_UNKNOWN);
/*
* Put page on a list so that it is released after
@@ -3690,7 +3716,7 @@
if (pmap_pte_dirty(pmap, tpte))
vm_page_dirty(m);
if ((tpte & ATTR_AF) != 0) {
- pmap_s1_invalidate_page(pmap, va, true);
+ pmap_s1_invalidate_page(pmap, va, true, TLBI_TTL_L3);
vm_page_aflag_set(m, PGA_REFERENCED);
}
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
@@ -4254,7 +4280,7 @@
* Since a promotion must break the 4KB page mappings before making
* the 2MB page mapping, a pmap_s1_invalidate_page() suffices.
*/
- pmap_s1_invalidate_page(pmap, sva, true);
+ pmap_s1_invalidate_page(pmap, sva, true, TLBI_TTL_L2);
if (old_l2 & ATTR_SW_WIRED)
pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE;
@@ -4313,7 +4339,7 @@
if ((old_l3 & ATTR_CONTIGUOUS) != 0)
(void)pmap_demote_l3c(pmap, l3, va);
old_l3 = pmap_load_clear(l3);
- pmap_s1_invalidate_page(pmap, va, true);
+ pmap_s1_invalidate_page(pmap, va, true, TLBI_TTL_L3);
if (old_l3 & ATTR_SW_WIRED)
pmap->pm_stats.wired_count -= 1;
pmap_resident_count_dec(pmap, 1);
@@ -4580,7 +4606,7 @@
MPASS(pmap != kernel_pmap);
MPASS((pmap_load(l1) & ATTR_SW_MANAGED) == 0);
pmap_clear(l1);
- pmap_s1_invalidate_page(pmap, sva, true);
+ pmap_s1_invalidate_page(pmap, sva, true, TLBI_TTL_L1);
pmap_resident_count_dec(pmap, L1_SIZE / PAGE_SIZE);
pmap_unuse_pt(pmap, sva, pmap_load(l0), &free);
continue;
@@ -4734,7 +4760,7 @@
if (tpte & ATTR_SW_WIRED)
pmap->pm_stats.wired_count--;
if ((tpte & ATTR_AF) != 0) {
- pmap_invalidate_page(pmap, pv->pv_va, true);
+ pmap_invalidate_page(pmap, pv->pv_va, true, TLBI_TTL_L3);
vm_page_aflag_set(m, PGA_REFERENCED);
}
@@ -4799,7 +4825,7 @@
* Since a promotion must break the 4KB page mappings before making
* the 2MB page mapping, a pmap_s1_invalidate_page() suffices.
*/
- pmap_s1_invalidate_page(pmap, sva, true);
+ pmap_s1_invalidate_page(pmap, sva, true, TLBI_TTL_L2);
}
/*
@@ -4888,7 +4914,7 @@
if ((pmap_load(l1) & mask) != nbits) {
pmap_store(l1, (pmap_load(l1) & ~mask) | nbits);
if (invalidate)
- pmap_s1_invalidate_page(pmap, sva, true);
+ pmap_s1_invalidate_page(pmap, sva, true, TLBI_TTL_L1);
}
continue;
}
@@ -5149,7 +5175,8 @@
*/
static void __nosanitizecoverage
pmap_update_strided(pmap_t pmap, pd_entry_t *ptep, pd_entry_t *ptep_end,
- pd_entry_t newpte, vm_offset_t va, vm_offset_t stride, vm_size_t size)
+ pd_entry_t newpte, vm_offset_t va, vm_offset_t stride, vm_size_t size,
+ int ttl)
{
pd_entry_t *lip;
register_t intr;
@@ -5174,7 +5201,7 @@
pmap_clear_bits(lip, ATTR_DESCR_VALID);
/* Only final entries are changing. */
- pmap_s1_invalidate_strided(pmap, va, va + size, stride, true);
+ pmap_s1_invalidate_strided(pmap, va, va + size, stride, true, ttl);
/* Create the new mapping. */
for (lip = ptep; lip < ptep_end; lip++) {
@@ -5495,7 +5522,8 @@
* Remake the mappings with the contiguous bit set.
*/
pmap_update_strided(pmap, l3p, l3p + L3C_ENTRIES, firstl3c |
- ATTR_CONTIGUOUS, va & ~L3C_OFFSET, L3_SIZE, L3C_SIZE);
+ ATTR_CONTIGUOUS, va & ~L3C_OFFSET, L3_SIZE, L3C_SIZE,
+ TLBI_TTL_L3);
counter_u64_add(pmap_l3c_promotions, 1);
CTR2(KTR_PMAP, "pmap_promote_l3c: success for va %#lx in pmap %p", va,
@@ -5986,7 +6014,7 @@
if (pmap_pte_dirty(pmap, orig_l3))
vm_page_dirty(om);
if ((orig_l3 & ATTR_AF) != 0) {
- pmap_invalidate_page(pmap, va, true);
+ pmap_invalidate_page(pmap, va, true, TLBI_TTL_L3);
vm_page_aflag_set(om, PGA_REFERENCED);
}
CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, om);
@@ -6005,7 +6033,7 @@
} else {
KASSERT((orig_l3 & ATTR_AF) != 0,
("pmap_enter: unmanaged mapping lacks ATTR_AF"));
- pmap_invalidate_page(pmap, va, true);
+ pmap_invalidate_page(pmap, va, true, TLBI_TTL_L3);
}
orig_l3 = 0;
} else {
@@ -6076,7 +6104,7 @@
if ((orig_l3 & ATTR_CONTIGUOUS) != 0)
(void)pmap_demote_l3c(pmap, l3, va);
orig_l3 = pmap_load_store(l3, new_l3);
- pmap_invalidate_page(pmap, va, true);
+ pmap_invalidate_page(pmap, va, true, TLBI_TTL_L3);
if ((orig_l3 & ATTR_SW_MANAGED) != 0 &&
pmap_pte_dirty(pmap, orig_l3))
vm_page_dirty(m);
@@ -6294,7 +6322,7 @@
* L2_TABLE entry.
*/
pmap_clear(l2);
- pmap_s1_invalidate_page(pmap, va, false);
+ pmap_s1_invalidate_page(pmap, va, false, TLBI_TTL_UNKNOWN);
}
}
KASSERT(pmap_load(l2) == 0,
@@ -7977,7 +8005,7 @@
if ((oldpte & mask) == val)
vm_page_dirty(m);
- pmap_invalidate_page(pmap, pv->pv_va, true);
+ pmap_invalidate_page(pmap, pv->pv_va, true, TLBI_TTL_L3);
}
PMAP_UNLOCK(pmap);
}
@@ -8076,7 +8104,7 @@
(uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 &&
(tpte & ATTR_SW_WIRED) == 0) {
pmap_clear_bits(pte, ATTR_AF);
- pmap_invalidate_page(pmap, va, true);
+ pmap_invalidate_page(pmap, va, true, TLBI_TTL_L2);
cleared++;
} else
not_cleared++;
@@ -8121,7 +8149,7 @@
* regardless of the contiguous bit.
*/
pmap_clear_bits(pte, ATTR_AF);
- pmap_invalidate_page(pmap, pv->pv_va, true);
+ pmap_invalidate_page(pmap, pv->pv_va, true, TLBI_TTL_L3);
cleared++;
} else
not_cleared++;
@@ -8381,7 +8409,7 @@
(oldl3 & ~ATTR_SW_DBM) | ATTR_S1_AP(ATTR_S1_AP_RO)))
cpu_spinwait();
vm_page_dirty(m);
- pmap_s1_invalidate_page(pmap, va, true);
+ pmap_s1_invalidate_page(pmap, va, true, TLBI_TTL_L3);
}
PMAP_UNLOCK(pmap);
}
@@ -8410,7 +8438,7 @@
if ((oldl3 & ATTR_CONTIGUOUS) != 0)
(void)pmap_demote_l3c(pmap, l3, pv->pv_va);
pmap_set_bits(l3, ATTR_S1_AP(ATTR_S1_AP_RO));
- pmap_s1_invalidate_page(pmap, pv->pv_va, true);
+ pmap_s1_invalidate_page(pmap, pv->pv_va, true, TLBI_TTL_L3);
}
PMAP_UNLOCK(pmap);
}
@@ -8882,11 +8910,13 @@
switch (pte_size) {
case L2C_SIZE:
pmap_update_strided(kernel_pmap, ptep, ptep +
- L2C_ENTRIES, pte, tmpva, L2_SIZE, L2C_SIZE);
+ L2C_ENTRIES, pte, tmpva, L2_SIZE, L2C_SIZE,
+ TLBI_TTL_L2);
break;
case L3C_SIZE:
pmap_update_strided(kernel_pmap, ptep, ptep +
- L3C_ENTRIES, pte, tmpva, L3_SIZE, L3C_SIZE);
+ L3C_ENTRIES, pte, tmpva, L3_SIZE, L3C_SIZE,
+ TLBI_TTL_L3);
break;
default:
/*
@@ -8903,7 +8933,7 @@
if (tmpptep != 0) {
pmap_clear(cmap1_pte);
pmap_s1_invalidate_page(kernel_pmap,
- (vm_offset_t)tmpptep, true);
+ (vm_offset_t)tmpptep, true, TLBI_TTL_L3);
mtx_unlock(&cmap_lock);
}
@@ -9319,7 +9349,7 @@
}
if ((nbits & ATTR_AF) != 0) {
pmap_s1_invalidate_strided(pmap, va & ~L2C_OFFSET, (va +
- L2C_SIZE) & ~L2C_OFFSET, L2_SIZE, true);
+ L2C_SIZE) & ~L2C_OFFSET, L2_SIZE, true, TLBI_TTL_L2);
}
/*
@@ -9715,6 +9745,25 @@
}
SYSINIT(pmap_init_cnp, SI_SUB_SMP, SI_ORDER_ANY, pmap_init_cnp, NULL);
+/*
+ * Defer enabling TTL until we have read the ID registers to know if it's
+ * supported on all CPUs.
+ */
+static void
+pmap_init_ttl(void *dummy __unused)
+{
+ uint64_t reg;
+
+ get_kernel_reg(ID_AA64MMFR2_EL1, &reg);
+ if (ID_AA64MMFR2_TTL_VAL(reg) != ID_AA64MMFR2_TTL_NONE) {
+ if (bootverbose)
+ printf("Enabling TTL\n");
+ pmap_ttl_support = true;
+ }
+
+}
+SYSINIT(pmap_init_ttl, SI_SUB_SMP, SI_ORDER_ANY, pmap_init_ttl, NULL);
+
static bool
pmap_activate_int(struct thread *td, pmap_t pmap)
{
@@ -9999,7 +10048,7 @@
if ((pte & ATTR_S1_AP_RW_BIT) ==
ATTR_S1_AP(ATTR_S1_AP_RO))
pmap_clear_bits(ptep, ATTR_S1_AP_RW_BIT);
- pmap_s1_invalidate_page_local(pmap, far);
+ pmap_s1_invalidate_page_local(pmap, far, TLBI_TTL_UNKNOWN);
rv = KERN_SUCCESS;
}
PMAP_UNLOCK(pmap);

File Metadata

Mime Type
text/plain
Expires
Mon, Oct 19, 1:41 AM (3 h, 12 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
40745768
Default Alt Text
D60090.diff (13 KB)

Event Timeline