Differential D22502 Diff 64974 head/sys/arm64/arm64/pmap.c

Changeset View

Standalone View

head/sys/arm64/arm64/pmap.c

Show First 20 Lines • Show All 1,037 Lines • ▼ Show 20 Lines
/*		/*
* Invalidate a single TLB entry.		* Invalidate a single TLB entry.
*/		*/
static __inline void		static __inline void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)		pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{		{
uint64_t r;		uint64_t r;

sched_pin();
dsb(ishst);		dsb(ishst);
if (pmap == kernel_pmap) {		if (pmap == kernel_pmap) {
r = atop(va);		r = atop(va);
__asm __volatile("tlbi vaae1is, %0" : : "r" (r));		__asm __volatile("tlbi vaae1is, %0" : : "r" (r));
} else {		} else {
r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)) \| atop(va);		r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)) \| atop(va);
__asm __volatile("tlbi vae1is, %0" : : "r" (r));		__asm __volatile("tlbi vae1is, %0" : : "r" (r));
}		}
dsb(ish);		dsb(ish);
isb();		isb();
sched_unpin();
}		}

static __inline void		static __inline void
pmap_invalidate_range_nopin(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)		pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{		{
uint64_t end, r, start;		uint64_t end, r, start;

dsb(ishst);		dsb(ishst);
if (pmap == kernel_pmap) {		if (pmap == kernel_pmap) {
start = atop(sva);		start = atop(sva);
end = atop(eva);		end = atop(eva);
for (r = start; r < end; r++)		for (r = start; r < end; r++)
__asm __volatile("tlbi vaae1is, %0" : : "r" (r));		__asm __volatile("tlbi vaae1is, %0" : : "r" (r));
} else {		} else {
start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));		start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
start \|= atop(sva);		start \|= atop(sva);
end \|= atop(eva);		end \|= atop(eva);
for (r = start; r < end; r++)		for (r = start; r < end; r++)
__asm __volatile("tlbi vae1is, %0" : : "r" (r));		__asm __volatile("tlbi vae1is, %0" : : "r" (r));
}		}
dsb(ish);		dsb(ish);
isb();		isb();
}		}

static __inline void		static __inline void
pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
{

sched_pin();
pmap_invalidate_range_nopin(pmap, sva, eva);
sched_unpin();
}

static __inline void
pmap_invalidate_all(pmap_t pmap)		pmap_invalidate_all(pmap_t pmap)
{		{
uint64_t r;		uint64_t r;

sched_pin();
dsb(ishst);		dsb(ishst);
if (pmap == kernel_pmap) {		if (pmap == kernel_pmap) {
__asm __volatile("tlbi vmalle1is");		__asm __volatile("tlbi vmalle1is");
} else {		} else {
r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));		r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
__asm __volatile("tlbi aside1is, %0" : : "r" (r));		__asm __volatile("tlbi aside1is, %0" : : "r" (r));
}		}
dsb(ish);		dsb(ish);
isb();		isb();
sched_unpin();
}		}

/*		/*
* Routine: pmap_extract		* Routine: pmap_extract
* Function:		* Function:
* Extract the physical page address associated		* Extract the physical page address associated
* with the given map/virtual_address pair.		* with the given map/virtual_address pair.
*/		*/
▲ Show 20 Lines • Show All 1,994 Lines • ▼ Show 20 Lines	pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte,
intr = intr_disable();		intr = intr_disable();

/*		/*
* Clear the old mapping's valid bit, but leave the rest of the entry		* Clear the old mapping's valid bit, but leave the rest of the entry
* unchanged, so that a lockless, concurrent pmap_kextract() can still		* unchanged, so that a lockless, concurrent pmap_kextract() can still
* lookup the physical address.		* lookup the physical address.
*/		*/
pmap_clear_bits(pte, ATTR_DESCR_VALID);		pmap_clear_bits(pte, ATTR_DESCR_VALID);
pmap_invalidate_range_nopin(pmap, va, va + size);		pmap_invalidate_range(pmap, va, va + size);

/* Create the new mapping */		/* Create the new mapping */
pmap_store(pte, newpte);		pmap_store(pte, newpte);
dsb(ishst);		dsb(ishst);

intr_restore(intr);		intr_restore(intr);
}		}

▲ Show 20 Lines • Show All 3,246 Lines • Show Last 20 Lines