Differential D30071 Diff 88721 sys/powerpc/aim/mmu_oea64.c

Changeset View

Standalone View

sys/powerpc/aim/mmu_oea64.c

Show First 20 Lines • Show All 380 Lines • ▼ Show 20 Lines
static __inline bool moea64_sp_pvo_in_range(struct pvo_entry *pvo,		static __inline bool moea64_sp_pvo_in_range(struct pvo_entry *pvo,
vm_offset_t sva, vm_offset_t eva);		vm_offset_t sva, vm_offset_t eva);

/*		/*
* Kernel MMU interface		* Kernel MMU interface
*/		*/
void moea64_clear_modify(vm_page_t);		void moea64_clear_modify(vm_page_t);
void moea64_copy_page(vm_page_t, vm_page_t);		void moea64_copy_page(vm_page_t, vm_page_t);
		void moea64_copy_page_dmap(vm_page_t, vm_page_t);
void moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,		void moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
vm_page_t *mb, vm_offset_t b_offset, int xfersize);		vm_page_t *mb, vm_offset_t b_offset, int xfersize);
		void moea64_copy_pages_dmap(vm_page_t *ma, vm_offset_t a_offset,
		vm_page_t *mb, vm_offset_t b_offset, int xfersize);
int moea64_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t,		int moea64_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t,
u_int flags, int8_t psind);		u_int flags, int8_t psind);
void moea64_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,		void moea64_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
vm_prot_t);		vm_prot_t);
void moea64_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);		void moea64_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
vm_paddr_t moea64_extract(pmap_t, vm_offset_t);		vm_paddr_t moea64_extract(pmap_t, vm_offset_t);
vm_page_t moea64_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);		vm_page_t moea64_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);
void moea64_init(void);		void moea64_init(void);
Show All 12 Lines
void moea64_qremove(vm_offset_t, int);		void moea64_qremove(vm_offset_t, int);
void moea64_release(pmap_t);		void moea64_release(pmap_t);
void moea64_remove(pmap_t, vm_offset_t, vm_offset_t);		void moea64_remove(pmap_t, vm_offset_t, vm_offset_t);
void moea64_remove_pages(pmap_t);		void moea64_remove_pages(pmap_t);
void moea64_remove_all(vm_page_t);		void moea64_remove_all(vm_page_t);
void moea64_remove_write(vm_page_t);		void moea64_remove_write(vm_page_t);
void moea64_unwire(pmap_t, vm_offset_t, vm_offset_t);		void moea64_unwire(pmap_t, vm_offset_t, vm_offset_t);
void moea64_zero_page(vm_page_t);		void moea64_zero_page(vm_page_t);
		void moea64_zero_page_dmap(vm_page_t);
void moea64_zero_page_area(vm_page_t, int, int);		void moea64_zero_page_area(vm_page_t, int, int);
void moea64_activate(struct thread *);		void moea64_activate(struct thread *);
void moea64_deactivate(struct thread *);		void moea64_deactivate(struct thread *);
void *moea64_mapdev(vm_paddr_t, vm_size_t);		void *moea64_mapdev(vm_paddr_t, vm_size_t);
void *moea64_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);		void *moea64_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
void moea64_unmapdev(vm_offset_t, vm_size_t);		void moea64_unmapdev(vm_offset_t, vm_size_t);
vm_paddr_t moea64_kextract(vm_offset_t);		vm_paddr_t moea64_kextract(vm_offset_t);
void moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma);		void moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma);
void moea64_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);		void moea64_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);
void moea64_kenter(vm_offset_t, vm_paddr_t);		void moea64_kenter(vm_offset_t, vm_paddr_t);
boolean_t moea64_dev_direct_mapped(vm_paddr_t, vm_size_t);		boolean_t moea64_dev_direct_mapped(vm_paddr_t, vm_size_t);
static void moea64_sync_icache(pmap_t, vm_offset_t, vm_size_t);		static void moea64_sync_icache(pmap_t, vm_offset_t, vm_size_t);
void moea64_dumpsys_map(vm_paddr_t pa, size_t sz,		void moea64_dumpsys_map(vm_paddr_t pa, size_t sz,
void **va);		void **va);
void moea64_scan_init(void);		void moea64_scan_init(void);
vm_offset_t moea64_quick_enter_page(vm_page_t m);		vm_offset_t moea64_quick_enter_page(vm_page_t m);
		vm_offset_t moea64_quick_enter_page_dmap(vm_page_t m);
void moea64_quick_remove_page(vm_offset_t addr);		void moea64_quick_remove_page(vm_offset_t addr);
boolean_t moea64_page_is_mapped(vm_page_t m);		boolean_t moea64_page_is_mapped(vm_page_t m);
static int moea64_map_user_ptr(pmap_t pm,		static int moea64_map_user_ptr(pmap_t pm,
volatile const void uaddr, void kaddr, size_t ulen, size_t klen);		volatile const void uaddr, void kaddr, size_t ulen, size_t klen);
static int moea64_decode_kernel_ptr(vm_offset_t addr,		static int moea64_decode_kernel_ptr(vm_offset_t addr,
int is_user, vm_offset_t decoded_addr);		int is_user, vm_offset_t decoded_addr);
static size_t moea64_scan_pmap(void);		static size_t moea64_scan_pmap(void);
static void *moea64_dump_pmap_init(unsigned blkpgs);		static void *moea64_dump_pmap_init(unsigned blkpgs);
▲ Show 20 Lines • Show All 487 Lines • ▼ Show 20 Lines	#ifndef __powerpc64__
hw_direct_map = 0;		hw_direct_map = 0;

/* Make sure battable is zero, since we have no BAT */		/* Make sure battable is zero, since we have no BAT */
for (i = 0; i < 16; i++) {		for (i = 0; i < 16; i++) {
battable[i].batu = 0;		battable[i].batu = 0;
battable[i].batl = 0;		battable[i].batl = 0;
}		}
#else		#else
moea64_probe_large_page();

/* Use a direct map if we have large page support */
if (moea64_large_page_size > 0)
hw_direct_map = 1;
else
hw_direct_map = 0;

/* Install trap handlers for SLBs */		/* Install trap handlers for SLBs */
bcopy(&slbtrap, (void *)EXC_DSE,(size_t)&slbtrapend - (size_t)&slbtrap);		bcopy(&slbtrap, (void *)EXC_DSE,(size_t)&slbtrapend - (size_t)&slbtrap);
bcopy(&slbtrap, (void *)EXC_ISE,(size_t)&slbtrapend - (size_t)&slbtrap);		bcopy(&slbtrap, (void *)EXC_ISE,(size_t)&slbtrapend - (size_t)&slbtrap);
__syncicache((void *)EXC_DSE, 0x80);		__syncicache((void *)EXC_DSE, 0x80);
__syncicache((void *)EXC_ISE, 0x80);		__syncicache((void *)EXC_ISE, 0x80);
#endif		#endif

kernelphysstart = kernelstart & ~DMAP_BASE_ADDRESS;		kernelphysstart = kernelstart & ~DMAP_BASE_ADDRESS;
▲ Show 20 Lines • Show All 504 Lines • ▼ Show 20 Lines	void moea64_set_scratchpage_pa(int which, vm_paddr_t pa)
moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);		moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
PMAP_UNLOCK(pvo->pvo_pmap);		PMAP_UNLOCK(pvo->pvo_pmap);
isync();		isync();
}		}

void		void
moea64_copy_page(vm_page_t msrc, vm_page_t mdst)		moea64_copy_page(vm_page_t msrc, vm_page_t mdst)
{		{
		mtx_lock(&moea64_scratchpage_mtx);

		moea64_set_scratchpage_pa(0, VM_PAGE_TO_PHYS(msrc));
		moea64_set_scratchpage_pa(1, VM_PAGE_TO_PHYS(mdst));

		bcopy((void *)moea64_scratchpage_va[0],
		(void *)moea64_scratchpage_va[1], PAGE_SIZE);

		mtx_unlock(&moea64_scratchpage_mtx);
		}

		void
		moea64_copy_page_dmap(vm_page_t msrc, vm_page_t mdst)
		{
vm_offset_t dst;		vm_offset_t dst;
vm_offset_t src;		vm_offset_t src;

dst = VM_PAGE_TO_PHYS(mdst);		dst = VM_PAGE_TO_PHYS(mdst);
src = VM_PAGE_TO_PHYS(msrc);		src = VM_PAGE_TO_PHYS(msrc);

if (hw_direct_map) {
bcopy((void )PHYS_TO_DMAP(src), (void )PHYS_TO_DMAP(dst),		bcopy((void )PHYS_TO_DMAP(src), (void )PHYS_TO_DMAP(dst),
PAGE_SIZE);		PAGE_SIZE);
} else {
mtx_lock(&moea64_scratchpage_mtx);

moea64_set_scratchpage_pa(0, src);
moea64_set_scratchpage_pa(1, dst);

bcopy((void *)moea64_scratchpage_va[0],
(void *)moea64_scratchpage_va[1], PAGE_SIZE);

mtx_unlock(&moea64_scratchpage_mtx);
}		}
}

static inline void		inline void
moea64_copy_pages_dmap(vm_page_t *ma, vm_offset_t a_offset,		moea64_copy_pages_dmap(vm_page_t *ma, vm_offset_t a_offset,
vm_page_t *mb, vm_offset_t b_offset, int xfersize)		vm_page_t *mb, vm_offset_t b_offset, int xfersize)
{		{
void a_cp, b_cp;		void a_cp, b_cp;
vm_offset_t a_pg_offset, b_pg_offset;		vm_offset_t a_pg_offset, b_pg_offset;
int cnt;		int cnt;

while (xfersize > 0) {		while (xfersize > 0) {
Show All 9 Lines	b_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
b_pg_offset;		b_pg_offset;
bcopy(a_cp, b_cp, cnt);		bcopy(a_cp, b_cp, cnt);
a_offset += cnt;		a_offset += cnt;
b_offset += cnt;		b_offset += cnt;
xfersize -= cnt;		xfersize -= cnt;
}		}
}		}

static inline void		void
moea64_copy_pages_nodmap(vm_page_t *ma, vm_offset_t a_offset,		moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
vm_page_t *mb, vm_offset_t b_offset, int xfersize)		vm_page_t *mb, vm_offset_t b_offset, int xfersize)
{		{
void a_cp, b_cp;		void a_cp, b_cp;
vm_offset_t a_pg_offset, b_pg_offset;		vm_offset_t a_pg_offset, b_pg_offset;
int cnt;		int cnt;

mtx_lock(&moea64_scratchpage_mtx);		mtx_lock(&moea64_scratchpage_mtx);
while (xfersize > 0) {		while (xfersize > 0) {
Show All 11 Lines	while (xfersize > 0) {
a_offset += cnt;		a_offset += cnt;
b_offset += cnt;		b_offset += cnt;
xfersize -= cnt;		xfersize -= cnt;
}		}
mtx_unlock(&moea64_scratchpage_mtx);		mtx_unlock(&moea64_scratchpage_mtx);
}		}

void		void
moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
vm_page_t *mb, vm_offset_t b_offset, int xfersize)
{

if (hw_direct_map) {
moea64_copy_pages_dmap(ma, a_offset, mb, b_offset,
xfersize);
} else {
moea64_copy_pages_nodmap(ma, a_offset, mb, b_offset,
xfersize);
}
}

void
moea64_zero_page_area(vm_page_t m, int off, int size)		moea64_zero_page_area(vm_page_t m, int off, int size)
{		{
vm_paddr_t pa = VM_PAGE_TO_PHYS(m);		vm_paddr_t pa = VM_PAGE_TO_PHYS(m);

if (size + off > PAGE_SIZE)		if (size + off > PAGE_SIZE)
panic("moea64_zero_page: size + off > PAGE_SIZE");		panic("moea64_zero_page: size + off > PAGE_SIZE");

if (hw_direct_map) {		if (hw_direct_map) {
Show All 10 Lines
* Zero a page of physical memory by temporarily mapping it		* Zero a page of physical memory by temporarily mapping it
*/		*/
void		void
moea64_zero_page(vm_page_t m)		moea64_zero_page(vm_page_t m)
{		{
vm_paddr_t pa = VM_PAGE_TO_PHYS(m);		vm_paddr_t pa = VM_PAGE_TO_PHYS(m);
vm_offset_t va, off;		vm_offset_t va, off;

if (!hw_direct_map) {
mtx_lock(&moea64_scratchpage_mtx);		mtx_lock(&moea64_scratchpage_mtx);

moea64_set_scratchpage_pa(0, pa);		moea64_set_scratchpage_pa(0, pa);
va = moea64_scratchpage_va[0];		va = moea64_scratchpage_va[0];
} else {
va = PHYS_TO_DMAP(pa);
}

for (off = 0; off < PAGE_SIZE; off += cacheline_size)		for (off = 0; off < PAGE_SIZE; off += cacheline_size)
__asm __volatile("dcbz 0,%0" :: "r"(va + off));		__asm __volatile("dcbz 0,%0" :: "r"(va + off));

if (!hw_direct_map)
mtx_unlock(&moea64_scratchpage_mtx);		mtx_unlock(&moea64_scratchpage_mtx);
}		}

		void
		moea64_zero_page_dmap(vm_page_t m)
		{
		vm_paddr_t pa = VM_PAGE_TO_PHYS(m);
		vm_offset_t va, off;

		va = PHYS_TO_DMAP(pa);
		for (off = 0; off < PAGE_SIZE; off += cacheline_size)
		__asm __volatile("dcbz 0,%0" :: "r"(va + off));
		}

vm_offset_t		vm_offset_t
moea64_quick_enter_page(vm_page_t m)		moea64_quick_enter_page(vm_page_t m)
{		{
struct pvo_entry *pvo;		struct pvo_entry *pvo;
vm_paddr_t pa = VM_PAGE_TO_PHYS(m);		vm_paddr_t pa = VM_PAGE_TO_PHYS(m);

if (hw_direct_map)
return (PHYS_TO_DMAP(pa));

/*		/*
* MOEA64_PTE_REPLACE does some locking, so we can't just grab		* MOEA64_PTE_REPLACE does some locking, so we can't just grab
* a critical section and access the PCPU data like on i386.		* a critical section and access the PCPU data like on i386.
* Instead, pin the thread and grab the PCPU lock to prevent		* Instead, pin the thread and grab the PCPU lock to prevent
* a preempting thread from using the same PCPU data.		* a preempting thread from using the same PCPU data.
*/		*/
sched_pin();		sched_pin();

mtx_assert(PCPU_PTR(aim.qmap_lock), MA_NOTOWNED);		mtx_assert(PCPU_PTR(aim.qmap_lock), MA_NOTOWNED);
pvo = PCPU_GET(aim.qmap_pvo);		pvo = PCPU_GET(aim.qmap_pvo);

mtx_lock(PCPU_PTR(aim.qmap_lock));		mtx_lock(PCPU_PTR(aim.qmap_lock));
pvo->pvo_pte.pa = moea64_calc_wimg(pa, pmap_page_get_memattr(m)) \|		pvo->pvo_pte.pa = moea64_calc_wimg(pa, pmap_page_get_memattr(m)) \|
(uint64_t)pa;		(uint64_t)pa;
moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);		moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
isync();		isync();

return (PCPU_GET(qmap_addr));		return (PCPU_GET(qmap_addr));
}		}

		vm_offset_t
		moea64_quick_enter_page_dmap(vm_page_t m)
		{

		return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
		}

void		void
moea64_quick_remove_page(vm_offset_t addr)		moea64_quick_remove_page(vm_offset_t addr)
{		{
if (hw_direct_map)
return;

mtx_assert(PCPU_PTR(aim.qmap_lock), MA_OWNED);		mtx_assert(PCPU_PTR(aim.qmap_lock), MA_OWNED);
KASSERT(PCPU_GET(qmap_addr) == addr,		KASSERT(PCPU_GET(qmap_addr) == addr,
("moea64_quick_remove_page: invalid address"));		("moea64_quick_remove_page: invalid address"));
mtx_unlock(PCPU_PTR(aim.qmap_lock));		mtx_unlock(PCPU_PTR(aim.qmap_lock));
sched_unpin();		sched_unpin();
}		}

▲ Show 20 Lines • Show All 1,884 Lines • ▼ Show 20 Lines

#define DEFINE_OEA64_IFUNC(ret, func, args, def) \		#define DEFINE_OEA64_IFUNC(ret, func, args, def) \
DEFINE_IFUNC(, ret, moea64_##func, args) { \		DEFINE_IFUNC(, ret, moea64_##func, args) { \
moea64_##func##_t f; \		moea64_##func##_t f; \
if (moea64_ops == NULL) \		if (moea64_ops == NULL) \
return ((moea64_##func##_t)def); \		return ((moea64_##func##_t)def); \
f = moea64_ops->func; \		f = moea64_ops->func; \
return (f != NULL ? f : (moea64_##func##_t)def);\		return (f != NULL ? f : (moea64_##func##_t)def);\
		}

		void
		moea64_install(void)
		{
		#ifdef __powerpc64__
		if (hw_direct_map == -1) {
		moea64_probe_large_page();

		/* Use a direct map if we have large page support */
		if (moea64_large_page_size > 0)
		hw_direct_map = 1;
		else
		hw_direct_map = 0;
		}
		#endif

		/*
		* Default to non-DMAP, and switch over to DMAP functions once we know
		* we have DMAP.
		*/
		if (hw_direct_map) {
		moea64_methods.quick_enter_page = moea64_quick_enter_page_dmap;
		moea64_methods.quick_remove_page = NULL;
		moea64_methods.copy_page = moea64_copy_page_dmap;
		moea64_methods.zero_page = moea64_zero_page_dmap;
		moea64_methods.copy_pages = moea64_copy_pages_dmap;
		}
}		}

DEFINE_OEA64_IFUNC(int64_t, pte_replace, (struct pvo_entry *, int),		DEFINE_OEA64_IFUNC(int64_t, pte_replace, (struct pvo_entry *, int),
moea64_pte_replace_default)		moea64_pte_replace_default)
DEFINE_OEA64_IFUNC(int64_t, pte_insert, (struct pvo_entry *), moea64_null_method)		DEFINE_OEA64_IFUNC(int64_t, pte_insert, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_unset, (struct pvo_entry *), moea64_null_method)		DEFINE_OEA64_IFUNC(int64_t, pte_unset, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_clear, (struct pvo_entry *, uint64_t),		DEFINE_OEA64_IFUNC(int64_t, pte_clear, (struct pvo_entry *, uint64_t),
moea64_null_method)		moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_synch, (struct pvo_entry *), moea64_null_method)		DEFINE_OEA64_IFUNC(int64_t, pte_synch, (struct pvo_entry *), moea64_null_method)
▲ Show 20 Lines • Show All 767 Lines • Show Last 20 Lines