Page MenuHomeFreeBSD
Files F143135831

D21922.id62999.diff
No OneTemporary
Actions

D21922.id62999.diff
View Options

Index: arm64/arm64/cpufunc_asm.S
===================================================================
--- arm64/arm64/cpufunc_asm.S
+++ arm64/arm64/cpufunc_asm.S
@@ -93,14 +93,6 @@
* Generic functions to read/modify/write the internal coprocessor registers
*/
-ENTRY(arm64_setttb)
- dsb ish
- msr ttbr0_el1, x0
- dsb ish
- isb
- ret
-END(arm64_setttb)
-
ENTRY(arm64_tlb_flushID)
dsb ishst
#ifdef SMP
Index: arm64/arm64/efirt_machdep.c
===================================================================
--- arm64/arm64/efirt_machdep.c
+++ arm64/arm64/efirt_machdep.c
@@ -47,6 +47,7 @@
#include <sys/rwlock.h>
#include <sys/systm.h>
#include <sys/vmmeter.h>
+#include <sys/ktr.h>
#include <machine/metadata.h>
#include <machine/pcb.h>
@@ -62,9 +63,9 @@
#include <vm/vm_pager.h>
static vm_object_t obj_1t1_pt;
-static vm_page_t efi_l0_page;
+static vm_pindex_t efi_1t1_idx;
static pd_entry_t *efi_l0;
-static vm_pindex_t efi_1t1_idx;
+static uint64_t efi_ttbr0;
void
efi_destroy_1t1_map(void)
@@ -81,8 +82,9 @@
}
obj_1t1_pt = NULL;
+ efi_1t1_idx = 0;
efi_l0 = NULL;
- efi_l0_page = NULL;
+ efi_ttbr0 = 0;
}
static vm_page_t
@@ -164,6 +166,7 @@
struct efi_md *p;
pt_entry_t *l3, l3_attr;
vm_offset_t va;
+ vm_page_t efi_l0_page;
uint64_t idx;
int i, mode;
@@ -172,10 +175,11 @@
L0_ENTRIES * Ln_ENTRIES * Ln_ENTRIES * Ln_ENTRIES,
VM_PROT_ALL, 0, NULL);
VM_OBJECT_WLOCK(obj_1t1_pt);
- efi_1t1_idx = 0;
efi_l0_page = efi_1t1_page();
VM_OBJECT_WUNLOCK(obj_1t1_pt);
efi_l0 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(efi_l0_page));
+ efi_ttbr0 = ASID_TO_OPERAND(ASID_RESERVED_FOR_EFI) |
+ VM_PAGE_TO_PHYS(efi_l0_page);
for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p,
descsz)) {
@@ -213,7 +217,7 @@
printf("MAP %lx mode %x pages %lu\n", p->md_phys, mode, p->md_pages);
l3_attr = ATTR_DEFAULT | ATTR_IDX(mode) | ATTR_AP(ATTR_AP_RW) |
- L3_PAGE;
+ ATTR_nG | L3_PAGE;
if (mode == VM_MEMATTR_DEVICE || p->md_attr & EFI_MD_ATTR_XP)
l3_attr |= ATTR_UXN | ATTR_PXN;
@@ -236,15 +240,10 @@
efi_arch_enter(void)
{
- __asm __volatile(
- "msr ttbr0_el1, %0 \n"
- "isb \n"
- "dsb ishst \n"
- "tlbi vmalle1is \n"
- "dsb ish \n"
- "isb \n"
- : : "r"(VM_PAGE_TO_PHYS(efi_l0_page)));
+ CTR1(KTR_SPARE5, "%s: xxx", __func__);
+ set_ttbr0(efi_ttbr0);
+
return (0);
}
@@ -264,14 +263,11 @@
"mrs x18, tpidr_el1 \n"
);
td = curthread;
- __asm __volatile(
- "msr ttbr0_el1, %0 \n"
- "isb \n"
- "dsb ishst \n"
- "tlbi vmalle1is \n"
- "dsb ish \n"
- "isb \n"
- : : "r"(td->td_proc->p_md.md_l0addr));
+ set_ttbr0(td->td_proc->p_md.md_ttbr0);
+
+ CTR3(KTR_SPARE5, "%s: pid=%d, ttbr0=%lx", __func__,
+ td->td_proc->p_pid,
+ td->td_proc->p_md.md_ttbr0);
}
int
Index: arm64/arm64/genassym.c
===================================================================
--- arm64/arm64/genassym.c
+++ arm64/arm64/genassym.c
@@ -35,7 +35,6 @@
#include <machine/frame.h>
#include <machine/pcb.h>
-#include <machine/vmparam.h>
ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
@@ -54,9 +53,6 @@
ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault));
ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags));
-ASSYM(P_MD, offsetof(struct proc, p_md));
-ASSYM(MD_L0ADDR, offsetof(struct mdproc, md_l0addr));
-
ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
ASSYM(TD_PROC, offsetof(struct thread, td_proc));
Index: arm64/arm64/locore.S
===================================================================
--- arm64/arm64/locore.S
+++ arm64/arm64/locore.S
@@ -501,6 +501,9 @@
* x9 = PA start (trashed)
* x10 = Entry count
* x11, x12 and x13 are trashed
+ *
+ * XXX Currently, this function is only used to create mappings for the
+ * ttbr0 table, so we unconditionally set ATTR_nG.
*/
build_l1_block_pagetable:
/*
@@ -517,6 +520,7 @@
#ifdef SMP
orr x12, x12, ATTR_SH(ATTR_SH_IS)
#endif
+ orr x12, x12, #(ATTR_nG)
/* Only use the output address bits */
lsr x9, x9, #L1_SHIFT
Index: arm64/arm64/machdep.c
===================================================================
--- arm64/arm64/machdep.c
+++ arm64/arm64/machdep.c
@@ -753,7 +753,7 @@
pcpup->pc_curpcb = thread0.td_pcb;
/* Set the base address of translation table 0. */
- thread0.td_proc->p_md.md_l0addr = READ_SPECIALREG(ttbr0_el1);
+ thread0.td_proc->p_md.md_ttbr0 = READ_SPECIALREG(ttbr0_el1);
}
typedef struct {
Index: arm64/arm64/pmap.c
===================================================================
--- arm64/arm64/pmap.c
+++ arm64/arm64/pmap.c
@@ -229,6 +229,10 @@
struct pmap kernel_pmap_store;
+/* ASID allocator */
+static struct unrhdr asid_unr;
+static struct mtx asid_mtx;
+
/* Used for mapping ACPI memory before VM is initialized */
#define PMAP_PREINIT_MAPPING_COUNT 32
#define PMAP_PREINIT_MAPPING_SIZE (PMAP_PREINIT_MAPPING_COUNT * L2_SIZE)
@@ -794,6 +798,7 @@
/* Set this early so we can use the pagetable walking functions */
kernel_pmap_store.pm_l0 = (pd_entry_t *)l0pt;
+ kernel_pmap->pm_asid = -1;
PMAP_LOCK_INIT(kernel_pmap);
/* Assume the address we were loaded to is a valid physical address */
@@ -906,8 +911,18 @@
{
vm_size_t s;
int i, pv_npg;
+ uint64_t reg;
/*
+ * Check tcr_el1.a1 and tcr_el1.as. They should be 0 and 1,
+ * respectively.
+ */
+ reg = READ_SPECIALREG(tcr_el1);
+ printf("tcr_el1: %lx\n", reg);
+ printf("tcr_el1.a1: %lx\n", reg & (1UL << 22));
+ printf("tcr_el1.as: %lx\n", reg & (1UL << 36));
+
+ /*
* Are large page mappings enabled?
*/
TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
@@ -918,6 +933,12 @@
}
/*
+ * Initialize the ASID allocator, reserving ASID 0 and 1.
+ */
+ mtx_init(&asid_mtx, "asid", NULL, MTX_DEF);
+ init_unrhdr(&asid_unr, ASID_FIRST_AVAILABLE, (1 << 16) - 1, &asid_mtx);
+
+ /*
* Initialize the pv chunk list mutex.
*/
mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
@@ -971,14 +992,19 @@
static __inline void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
+ uint64_t r;
sched_pin();
- __asm __volatile(
- "dsb ishst \n"
- "tlbi vaae1is, %0 \n"
- "dsb ish \n"
- "isb \n"
- : : "r"(va >> PAGE_SHIFT));
+ dsb(ishst);
+ if (pmap == kernel_pmap) {
+ r = atop(va);
+ __asm __volatile("tlbi vaae1is, %0" : : "r" (r));
+ } else {
+ r = ASID_TO_OPERAND(pmap->pm_asid) | atop(va);
+ __asm __volatile("tlbi vae1is, %0" : : "r" (r));
+ }
+ dsb(ish);
+ isb();
sched_unpin();
}
@@ -985,16 +1011,23 @@
static __inline void
pmap_invalidate_range_nopin(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{
- vm_offset_t addr;
+ uint64_t end, r, start;
dsb(ishst);
- for (addr = sva; addr < eva; addr += PAGE_SIZE) {
- __asm __volatile(
- "tlbi vaae1is, %0" : : "r"(addr >> PAGE_SHIFT));
+ if (pmap == kernel_pmap) {
+ start = atop(sva);
+ end = atop(eva);
+ for (r = start; r < end; r++)
+ __asm __volatile("tlbi vaae1is, %0" : : "r" (r));
+ } else {
+ start = end = ASID_TO_OPERAND(pmap->pm_asid);
+ start |= atop(sva);
+ end |= atop(eva);
+ for (r = start; r < end; r++)
+ __asm __volatile("tlbi vae1is, %0" : : "r" (r));
}
- __asm __volatile(
- "dsb ish \n"
- "isb \n");
+ dsb(ish);
+ isb();
}
static __inline void
@@ -1009,13 +1042,18 @@
static __inline void
pmap_invalidate_all(pmap_t pmap)
{
+ uint64_t r;
sched_pin();
- __asm __volatile(
- "dsb ishst \n"
- "tlbi vmalle1is \n"
- "dsb ish \n"
- "isb \n");
+ dsb(ishst);
+ if (pmap == kernel_pmap) {
+ __asm __volatile("tlbi vmalle1is");
+ } else {
+ r = ASID_TO_OPERAND(pmap->pm_asid);
+ __asm __volatile("tlbi aside1is, %0" : : "r" (r));
+ }
+ dsb(ish);
+ isb();
sched_unpin();
}
@@ -1442,9 +1480,14 @@
pmap_pinit0(pmap_t pmap)
{
+ printf("pmap_kextract(kernel_pmap->pm_l0) = %lx\n",
+ pmap_kextract((vm_offset_t)kernel_pmap->pm_l0));
+ printf("ttbr0 = %lx\n", READ_SPECIALREG(ttbr0_el1));
+
PMAP_LOCK_INIT(pmap);
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
- pmap->pm_l0 = kernel_pmap->pm_l0;
+ pmap->pm_l0 = (pd_entry_t *)PHYS_TO_DMAP(READ_SPECIALREG(ttbr0_el1));
+ pmap->pm_asid = ASID_RESERVED_FOR_PID_0;
pmap->pm_root.rt_root = 0;
}
@@ -1467,6 +1510,9 @@
if ((l0pt->flags & PG_ZERO) == 0)
pagezero(pmap->pm_l0);
+ if ((pmap->pm_asid = alloc_unr(&asid_unr)) == -1)
+ panic("alloc_unr: ASID allocation failed");
+
pmap->pm_root.rt_root = 0;
bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
@@ -1717,6 +1763,9 @@
KASSERT(vm_radix_is_empty(&pmap->pm_root),
("pmap_release: pmap has reserved page table page(s)"));
+ free_unr(&asid_unr, pmap->pm_asid);
+ pmap->pm_asid = -1;
+
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_l0));
vm_page_unwire_noq(m);
@@ -3194,6 +3243,8 @@
new_l3 |= ATTR_SW_WIRED;
if (va < VM_MAXUSER_ADDRESS)
new_l3 |= ATTR_AP(ATTR_AP_USER) | ATTR_PXN;
+ if (pmap != kernel_pmap)
+ new_l3 |= ATTR_nG;
if ((m->oflags & VPO_UNMANAGED) == 0) {
new_l3 |= ATTR_SW_MANAGED;
if ((prot & VM_PROT_WRITE) != 0) {
@@ -3456,6 +3507,8 @@
new_l2 |= ATTR_XN;
if (va < VM_MAXUSER_ADDRESS)
new_l2 |= ATTR_AP(ATTR_AP_USER) | ATTR_PXN;
+ if (pmap != kernel_pmap)
+ new_l2 |= ATTR_nG;
return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP |
PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
KERN_SUCCESS);
@@ -3754,6 +3807,8 @@
l3_val |= ATTR_XN;
if (va < VM_MAXUSER_ADDRESS)
l3_val |= ATTR_AP(ATTR_AP_USER) | ATTR_PXN;
+ if (pmap != kernel_pmap)
+ l3_val |= ATTR_nG;
/*
* Now validate mapping with RO protection
@@ -5679,20 +5734,28 @@
return (val);
}
+uint64_t
+pmap_to_ttbr0(pmap_t pmap)
+{
+
+ return (ASID_TO_OPERAND(pmap->pm_asid) |
+ pmap_kextract((vm_offset_t)pmap->pm_l0));
+}
+
void
pmap_activate(struct thread *td)
{
- pmap_t pmap;
+ struct proc *p;
critical_enter();
- pmap = vmspace_pmap(td->td_proc->p_vmspace);
- td->td_proc->p_md.md_l0addr = vtophys(pmap->pm_l0);
- __asm __volatile(
- "msr ttbr0_el1, %0 \n"
- "isb \n"
- : : "r"(td->td_proc->p_md.md_l0addr));
- pmap_invalidate_all(pmap);
+ p = td->td_proc;
+ p->p_md.md_ttbr0 = pmap_to_ttbr0(vmspace_pmap(p->p_vmspace));
+ set_ttbr0(p->p_md.md_ttbr0);
critical_exit();
+
+ CTR3(KTR_SPARE5, "%s: pid=%d, ttbr0=%lx", __func__,
+ p->p_pid,
+ p->p_md.md_ttbr0);
}
struct pcb *
@@ -5714,18 +5777,12 @@
*/
if (old == NULL ||
- old->td_proc->p_md.md_l0addr != new->td_proc->p_md.md_l0addr) {
- __asm __volatile(
- /* Switch to the new pmap */
- "msr ttbr0_el1, %0 \n"
- "isb \n"
+ old->td_proc->p_md.md_ttbr0 != new->td_proc->p_md.md_ttbr0) {
+ set_ttbr0(new->td_proc->p_md.md_ttbr0);
- /* Invalidate the TLB */
- "dsb ishst \n"
- "tlbi vmalle1is \n"
- "dsb ish \n"
- "isb \n"
- : : "r"(new->td_proc->p_md.md_l0addr));
+ CTR3(KTR_SPARE5, "%s: pid=%d, ttbr0=%lx", __func__,
+ new->td_proc->p_pid,
+ new->td_proc->p_md.md_ttbr0);
/*
* Stop userspace from training the branch predictor against
Index: arm64/arm64/vm_machdep.c
===================================================================
--- arm64/arm64/vm_machdep.c
+++ arm64/arm64/vm_machdep.c
@@ -91,8 +91,8 @@
td2->td_pcb = pcb2;
bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
- td2->td_proc->p_md.md_l0addr =
- vtophys(vmspace_pmap(td2->td_proc->p_vmspace)->pm_l0);
+ td2->td_proc->p_md.md_ttbr0 =
+ pmap_to_ttbr0(vmspace_pmap(td2->td_proc->p_vmspace));
tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1);
bcopy(td1->td_frame, tf, sizeof(*tf));
Index: arm64/include/cpufunc.h
===================================================================
--- arm64/include/cpufunc.h
+++ arm64/include/cpufunc.h
@@ -178,6 +178,17 @@
__asm __volatile("clrex" : : : "memory");
}
+static __inline void
+set_ttbr0(uint64_t ttbr0)
+{
+
+ __asm __volatile(
+ "msr ttbr0_el1, %0 \n"
+ "isb \n"
+ :
+ : "r" (ttbr0));
+}
+
extern int64_t dcache_line_size;
extern int64_t icache_line_size;
extern int64_t idcache_line_size;
@@ -185,7 +196,6 @@
#define cpu_nullop() arm64_nullop()
#define cpufunc_nullop() arm64_nullop()
-#define cpu_setttb(a) arm64_setttb(a)
#define cpu_tlb_flushID() arm64_tlb_flushID()
@@ -198,7 +208,6 @@
#define cpu_icache_sync_range_checked(a, s) arm64_icache_sync_range_checked((a), (s))
void arm64_nullop(void);
-void arm64_setttb(vm_offset_t);
void arm64_tlb_flushID(void);
void arm64_icache_sync_range(vm_offset_t, vm_size_t);
int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t);
Index: arm64/include/pmap.h
===================================================================
--- arm64/include/pmap.h
+++ arm64/include/pmap.h
@@ -83,6 +83,7 @@
pd_entry_t *pm_l0;
TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */
struct vm_radix pm_root; /* spare page table pages */
+ int pm_asid;
};
typedef struct pmap *pmap_t;
@@ -132,6 +133,15 @@
#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)
#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)
+#define ASID_RESERVED_FOR_PID_0 0
+#define ASID_RESERVED_FOR_EFI 1
+#define ASID_FIRST_AVAILABLE (ASID_RESERVED_FOR_EFI + 1)
+#define ASID_TO_OPERAND_SHIFT 48
+#define ASID_TO_OPERAND(asid) ({ \
+ KASSERT((asid) != -1, ("invalid ASID")); \
+ (uint64_t)(asid) << ASID_TO_OPERAND_SHIFT; \
+})
+
extern vm_offset_t virtual_avail;
extern vm_offset_t virtual_end;
@@ -144,6 +154,7 @@
void pmap_bootstrap(vm_offset_t, vm_offset_t, vm_paddr_t, vm_size_t);
int pmap_change_attr(vm_offset_t va, vm_size_t size, int mode);
+uint64_t pmap_to_ttbr0(pmap_t pmap);
void pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode);
void pmap_kenter_device(vm_offset_t, vm_size_t, vm_paddr_t);
vm_paddr_t pmap_kextract(vm_offset_t va);
Index: arm64/include/proc.h
===================================================================
--- arm64/include/proc.h
+++ arm64/include/proc.h
@@ -40,7 +40,7 @@
};
struct mdproc {
- vm_offset_t md_l0addr;
+ uint64_t md_ttbr0;
};
#define KINFO_PROC_SIZE 1088

File Metadata

Mime Type
text/plain
Expires
Tue, Jan 27, 10:42 AM (12 h, 48 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
28054566
Default Alt Text
D21922.id62999.diff (13 KB)

Event Timeline