Differential D25237 Diff 76890 sys/powerpc/aim/moea64_native.c

Changeset View

Standalone View

sys/powerpc/aim/moea64_native.c

Show First 20 Lines • Show All 126 Lines • ▼ Show 20 Lines
#define SYNC() __asm __volatile("sync");		#define SYNC() __asm __volatile("sync");
#define EIEIO() __asm __volatile("eieio");		#define EIEIO() __asm __volatile("eieio");

#define VSID_HASH_MASK 0x0000007fffffffffULL		#define VSID_HASH_MASK 0x0000007fffffffffULL

/* POWER9 only permits a 64k partition table size. */		/* POWER9 only permits a 64k partition table size. */
#define PART_SIZE 0x10000		#define PART_SIZE 0x10000

		/* Actual page sizes (to be used with tlbie, when L=0) */
		#define AP_4K 0x00
		#define AP_16M 0x80

		#define LPTE_KERNEL_VSID_BIT (KERNEL_VSID_BIT << \
		(16 - (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)))

		/* Abbreviated Virtual Address Page - high bits */
		#define LPTE_AVA_PGNHI_MASK 0x0000000000000F80ULL
		#define LPTE_AVA_PGNHI_SHIFT 7

		jhibbitsUnsubmitted Done Inline Actions This may need to change in the future, to support DRI. We may need to demote to change cache characteristics for an individual DMAP page. jhibbits: This may need to change in the future, to support DRI. We may need to demote to change cache…
		luporlAuthorUnsubmitted Done Inline Actions Ok, but this would not be a trivial change. AFAIK, there is no easy way to tell if a PTE belongs to kernel or userspace. This information could be obtained from the pvo (although currently there is none in moea64_insert_to_pteg_native() , that calls TLBIE), but there are some complications: TLBIE must match the features (e.g. LPTE_BIG, 16M or 4K/16M) of the old page, but in operations such as insert and replace the pvo describes the new page. luporl: Ok, but this would not be a trivial change. AFAIK, there is no easy way to tell if a PTE…
		luporlAuthorUnsubmitted Done Inline Actions Now using LPTE_KERNEL_VSID_BIT to check if page belongs to kernel or userspace. luporl: Now using LPTE_KERNEL_VSID_BIT to check if page belongs to kernel or userspace.
		/* Effective Address Page - low bits */
		#define EA_PAGELO_MASK 0x7ffULL
		#define EA_PAGELO_SHIFT 11

static bool moea64_crop_tlbie;		static bool moea64_crop_tlbie;
static bool moea64_need_lock;		static bool moea64_need_lock;

		/*
		jhibbitsUnsubmitted Done Inline Actions I think 'old' is only needed when 'crop' is needed; otherwise, it's not used. So putting the tlb_old into the 'if (moea64_crop_tlbie)' block, with a goto, and putting the tlbie_new() as the rest of the body, should work, and might knock off a part of the hit, because you'll be removing one level of indirection. jhibbits: I think 'old' is only needed when 'crop' is needed; otherwise, it's not used. So putting the…
		luporlAuthorUnsubmitted Done Inline Actions Ok. This change worked fine on Talos, but I didn't notice any change in performance. luporl: Ok. This change worked fine on Talos, but I didn't notice any change in performance.
		* The tlbie instruction has two forms: an old one used by PowerISA
		* 2.03 and prior, and a newer one used by PowerISA 2.06 and later.
		* We need to support both.
		*/
static __inline void		static __inline void
TLBIE(uint64_t vpn) {		TLBIE(uint64_t vpn, uint64_t oldptehi)
		{
#ifndef __powerpc64__		#ifndef __powerpc64__
register_t vpn_hi, vpn_lo;		register_t vpn_hi, vpn_lo;
register_t msr;		register_t msr;
register_t scratch, intr;		register_t scratch, intr;
#endif		#endif

static volatile u_int tlbie_lock = 0;		static volatile u_int tlbie_lock = 0;
bool need_lock = moea64_need_lock;		bool need_lock = moea64_need_lock;

vpn <<= ADDR_PIDX_SHFT;		vpn <<= ADDR_PIDX_SHFT;

/* Hobo spinlock: we need stronger guarantees than mutexes provide */		/* Hobo spinlock: we need stronger guarantees than mutexes provide */
if (need_lock) {		if (need_lock) {
while (!atomic_cmpset_int(&tlbie_lock, 0, 1));		while (!atomic_cmpset_int(&tlbie_lock, 0, 1));
isync(); /* Flush instruction queue once lock acquired */		isync(); /* Flush instruction queue once lock acquired */

if (moea64_crop_tlbie)		if (moea64_crop_tlbie) {
vpn &= ~(0xffffULL << 48);		vpn &= ~(0xffffULL << 48);
		#ifdef __powerpc64__
		if ((oldptehi & LPTE_BIG) != 0)
		__asm __volatile("tlbie %0, 1" :: "r"(vpn) :
		"memory");
		else
		__asm __volatile("tlbie %0, 0" :: "r"(vpn) :
		"memory");
		__asm __volatile("eieio; tlbsync; ptesync" :::
		"memory");
		goto done;
		#endif
}		}
		}

#ifdef __powerpc64__		#ifdef __powerpc64__
/*		/*
* Explicitly clobber r0. The tlbie instruction has two forms: an old		* If this page has LPTE_BIG set and is from userspace, then
jhibbitsUnsubmitted Done Inline Actions Can you keep this comment in, to note why this silly mess is here? jhibbits: Can you keep this comment in, to note why this silly mess is here?
luporlAuthorUnsubmitted Done Inline Actions Sure. luporl: Sure.
* one used by PowerISA 2.03 and prior, and a newer one used by PowerISA		* it must be a superpage with 4KB base/16MB actual page size.
* 2.06 (maybe 2.05?) and later. We need to support both, and it just
* so happens that since we use 4k pages we can simply zero out r0, and
* clobber it, and the assembler will interpret the single-operand form
* of tlbie as having RB set, and everything else as 0. The RS operand
* in the newer form is in the same position as the L(page size) bit of
* the old form, so a slong as RS is 0, we're good on both sides.
*/		*/
__asm __volatile("li 0, 0 \n tlbie %0" :: "r"(vpn) : "r0", "memory");		if ((oldptehi & LPTE_BIG) != 0 &&
		(oldptehi & LPTE_KERNEL_VSID_BIT) == 0)
		vpn \|= AP_16M;

		__asm __volatile("li 0, 0 \n tlbie %0, 0" :: "r"(vpn) : "r0", "memory");
		jhibbitsUnsubmitted Done Inline Actions The PPC970 supports superpages, as does the POWER4. Now, we probably don't care much about the POWER4, but the PPC970 can benefit from superpages, but as mentioned in the comment right above, it uses a different tlbie instruction format. Can that be worked into this? jhibbits: The PPC970 supports superpages, as does the POWER4. Now, we probably don't care much about the…
		jhibbitsUnsubmitted Done Inline Actions Will this also work on ISA 2.03 and prior? We may want to just do a 'if running on old, use old tlbie, otherwise use new' (or use asm routines to do it all). jhibbits: Will this also work on ISA 2.03 and prior? We may want to just do a 'if running on old, use…
		luporlAuthorUnsubmitted Done Inline Actions By taking a look at PowerISA 2.03 spec, it initially looks like this wouldn't work on it, but in the end it's ok. On 2.03, AP is a 1-bit flag telling if the page to be invalidated is 4K or 64K. But this and section 5.7.5 Virtual Address Generation, when describing Mixed Page Size (MPS), suggest that only a 4K/64K page size mix is supported in a given segment, which would prevent the superpage mechanism from being used on these architectures (unless we add support for 64K superpages). This way, with superpages not supported on ISA 2.03 and earlier, the device tree check for 4K/16M MPS would set it as not available, disabling superpages. Then ap would always be 0 and the old tlbie format would continue to work. luporl: By taking a look at PowerISA 2.03 spec, it initially looks like this wouldn't work on it, but…
		luporlAuthorUnsubmitted Done Inline Actions Old tlbie instruction format is now on `__tlbie_old` and new one on `__tlbie_new`. Both support invalidating regular and large pages. (although the code to invalidate a 16M base/16M actual page should be added, if/when needed) luporl: Old tlbie instruction format is now on `__tlbie_old` and new one on `__tlbie_new`. Both support…
__asm __volatile("eieio; tlbsync; ptesync" ::: "memory");		__asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
#else		#else
vpn_hi = (uint32_t)(vpn >> 32);		vpn_hi = (uint32_t)(vpn >> 32);
vpn_lo = (uint32_t)vpn;		vpn_lo = (uint32_t)vpn;

intr = intr_disable();		intr = intr_disable();
__asm __volatile("\		__asm __volatile("\
mfmsr %0; \		mfmsr %0; \
Show All 9 Lines	__asm __volatile("\
eieio; \		eieio; \
tlbsync; \		tlbsync; \
ptesync;"		ptesync;"
: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)		: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
: "memory");		: "memory");
intr_restore(intr);		intr_restore(intr);
#endif		#endif

		done:
/* No barriers or special ops -- taken care of by ptesync above */		/* No barriers or special ops -- taken care of by ptesync above */
if (need_lock)		if (need_lock)
tlbie_lock = 0;		tlbie_lock = 0;
}		}

#define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR)		#define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR)
#define ENABLE_TRANS(msr) mtmsr(msr)		#define ENABLE_TRANS(msr) mtmsr(msr)

Show All 14 Lines
/*		/*
* PTE calls.		* PTE calls.
*/		*/
static int64_t moea64_pte_insert_native(struct pvo_entry *);		static int64_t moea64_pte_insert_native(struct pvo_entry *);
static int64_t moea64_pte_synch_native(struct pvo_entry *);		static int64_t moea64_pte_synch_native(struct pvo_entry *);
static int64_t moea64_pte_clear_native(struct pvo_entry *, uint64_t);		static int64_t moea64_pte_clear_native(struct pvo_entry *, uint64_t);
static int64_t moea64_pte_replace_native(struct pvo_entry *, int);		static int64_t moea64_pte_replace_native(struct pvo_entry *, int);
static int64_t moea64_pte_unset_native(struct pvo_entry *);		static int64_t moea64_pte_unset_native(struct pvo_entry *);
		static int64_t moea64_pte_insert_sp_native(struct pvo_entry *);
		static int64_t moea64_pte_unset_sp_native(struct pvo_entry *);
		static int64_t moea64_pte_replace_sp_native(struct pvo_entry *);

/*		/*
* Utility routines.		* Utility routines.
*/		*/
static void moea64_bootstrap_native(		static void moea64_bootstrap_native(
vm_offset_t kernelstart, vm_offset_t kernelend);		vm_offset_t kernelstart, vm_offset_t kernelend);
static void moea64_cpu_bootstrap_native(int ap);		static void moea64_cpu_bootstrap_native(int ap);
static void tlbia(void);		static void tlbia(void);
static void moea64_install_native(void);		static void moea64_install_native(void);

static struct pmap_funcs moea64_native_methods = {		static struct pmap_funcs moea64_native_methods = {
.install = moea64_install_native,		.install = moea64_install_native,

/* Internal interfaces */		/* Internal interfaces */
.bootstrap = moea64_bootstrap_native,		.bootstrap = moea64_bootstrap_native,
.cpu_bootstrap = moea64_cpu_bootstrap_native,		.cpu_bootstrap = moea64_cpu_bootstrap_native,
.dumpsys_dump_pmap = moea64_dump_pmap_native,		.dumpsys_dump_pmap = moea64_dump_pmap_native,
};		};

static struct moea64_funcs moea64_native_funcs = {		static struct moea64_funcs moea64_native_funcs = {
.pte_synch = moea64_pte_synch_native,		.pte_synch = moea64_pte_synch_native,
.pte_clear = moea64_pte_clear_native,		.pte_clear = moea64_pte_clear_native,
.pte_unset = moea64_pte_unset_native,		.pte_unset = moea64_pte_unset_native,
.pte_replace = moea64_pte_replace_native,		.pte_replace = moea64_pte_replace_native,
.pte_insert = moea64_pte_insert_native,		.pte_insert = moea64_pte_insert_native,
		.pte_insert_sp = moea64_pte_insert_sp_native,
		.pte_unset_sp = moea64_pte_unset_sp_native,
		.pte_replace_sp = moea64_pte_replace_sp_native,
};		};

MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods, oea64_mmu);		MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods, oea64_mmu);

static void		static void
moea64_install_native()		moea64_install_native()
{		{

▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines
#if BYTE_ORDER == BIG_ENDIAN		#if BYTE_ORDER == BIG_ENDIAN
((uint8_t *)(&properpt.pte_lo))[6];		((uint8_t *)(&properpt.pte_lo))[6];
#else		#else
((uint8_t *)(&properpt.pte_lo))[1];		((uint8_t *)(&properpt.pte_lo))[1];
#endif		#endif
rw_runlock(&moea64_eviction_lock);		rw_runlock(&moea64_eviction_lock);

critical_enter();		critical_enter();
TLBIE(pvo->pvo_vpn);		TLBIE(pvo->pvo_vpn, properpt.pte_hi);
critical_exit();		critical_exit();
} else {		} else {
rw_runlock(&moea64_eviction_lock);		rw_runlock(&moea64_eviction_lock);
ptelo = moea64_pte_unset_native(pvo);		ptelo = moea64_pte_unset_native(pvo);
moea64_pte_insert_native(pvo);		moea64_pte_insert_native(pvo);
}		}

return (ptelo & (LPTE_REF \| LPTE_CHG));		return (ptelo & (LPTE_REF \| LPTE_CHG));
}		}

static int64_t		static __inline int64_t
moea64_pte_unset_native(struct pvo_entry *pvo)		moea64_pte_unset_locked(volatile struct lpte *pt, uint64_t vpn)
{		{
volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot;		uint64_t ptelo;
uint64_t ptelo, pvo_ptevpn;

pvo_ptevpn = moea64_pte_vpn_from_pvo_vpn(pvo);

rw_rlock(&moea64_eviction_lock);
if ((be64toh(pt->pte_hi & LPTE_AVPN_MASK)) != pvo_ptevpn) {
/* Evicted */
STAT_MOEA64(moea64_pte_overflow--);
rw_runlock(&moea64_eviction_lock);
return (-1);
}

/*		/*
* Invalidate the pte, briefly locking it to collect RC bits. No		* Invalidate the pte, briefly locking it to collect RC bits. No
* atomics needed since this is protected against eviction by the lock.		* atomics needed since this is protected against eviction by the lock.
*/		*/
isync();		isync();
critical_enter();		critical_enter();
pt->pte_hi = be64toh((pt->pte_hi & ~LPTE_VALID) \| LPTE_LOCKED);		pt->pte_hi = be64toh((pt->pte_hi & ~LPTE_VALID) \| LPTE_LOCKED);
PTESYNC();		PTESYNC();
TLBIE(pvo->pvo_vpn);		TLBIE(vpn, pt->pte_hi);
ptelo = be64toh(pt->pte_lo);		ptelo = be64toh(pt->pte_lo);
((volatile int32_t )(&pt->pte_hi) + 1) = 0; /* Release lock */		((volatile int32_t )(&pt->pte_hi) + 1) = 0; /* Release lock */
critical_exit();		critical_exit();
rw_runlock(&moea64_eviction_lock);

/* Keep statistics */		/* Keep statistics */
STAT_MOEA64(moea64_pte_valid--);		STAT_MOEA64(moea64_pte_valid--);

return (ptelo & (LPTE_CHG \| LPTE_REF));		return (ptelo & (LPTE_CHG \| LPTE_REF));
}		}

static int64_t		static int64_t
		moea64_pte_unset_native(struct pvo_entry *pvo)
		{
		volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot;
		int64_t ret;
		uint64_t pvo_ptevpn;

		pvo_ptevpn = moea64_pte_vpn_from_pvo_vpn(pvo);

		rw_rlock(&moea64_eviction_lock);

		if ((be64toh(pt->pte_hi & LPTE_AVPN_MASK)) != pvo_ptevpn) {
		/* Evicted */
		STAT_MOEA64(moea64_pte_overflow--);
		ret = -1;
		} else
		ret = moea64_pte_unset_locked(pt, pvo->pvo_vpn);

		rw_runlock(&moea64_eviction_lock);

		return (ret);
		}

		static int64_t
moea64_pte_replace_inval_native(struct pvo_entry *pvo,		moea64_pte_replace_inval_native(struct pvo_entry *pvo,
volatile struct lpte *pt)		volatile struct lpte *pt)
{		{
struct lpte properpt;		struct lpte properpt;
uint64_t ptelo;		uint64_t ptelo;

moea64_pte_from_pvo(pvo, &properpt);		moea64_pte_from_pvo(pvo, &properpt);

Show All 9 Lines	moea64_pte_replace_inval_native(struct pvo_entry *pvo,
/*		/*
* Replace the pte, briefly locking it to collect RC bits. No		* Replace the pte, briefly locking it to collect RC bits. No
* atomics needed since this is protected against eviction by the lock.		* atomics needed since this is protected against eviction by the lock.
*/		*/
isync();		isync();
critical_enter();		critical_enter();
pt->pte_hi = be64toh((pt->pte_hi & ~LPTE_VALID) \| LPTE_LOCKED);		pt->pte_hi = be64toh((pt->pte_hi & ~LPTE_VALID) \| LPTE_LOCKED);
PTESYNC();		PTESYNC();
TLBIE(pvo->pvo_vpn);		TLBIE(pvo->pvo_vpn, pt->pte_hi);
ptelo = be64toh(pt->pte_lo);		ptelo = be64toh(pt->pte_lo);
EIEIO();		EIEIO();
pt->pte_lo = htobe64(properpt.pte_lo);		pt->pte_lo = htobe64(properpt.pte_lo);
EIEIO();		EIEIO();
pt->pte_hi = htobe64(properpt.pte_hi); /* Release lock */		pt->pte_hi = htobe64(properpt.pte_hi); /* Release lock */
PTESYNC();		PTESYNC();
critical_exit();		critical_exit();
rw_runlock(&moea64_eviction_lock);		rw_runlock(&moea64_eviction_lock);
▲ Show 20 Lines • Show All 291 Lines • ▼ Show 20 Lines	if (oldptehi & LPTE_VALID) {
if (oldptehi & LPTE_HID)		if (oldptehi & LPTE_HID)
va = (((k >> 3) ^ moea64_pteg_mask) ^ va) &		va = (((k >> 3) ^ moea64_pteg_mask) ^ va) &
(ADDR_PIDX >> ADDR_PIDX_SHFT);		(ADDR_PIDX >> ADDR_PIDX_SHFT);
else		else
va = ((k >> 3) ^ va) & (ADDR_PIDX >> ADDR_PIDX_SHFT);		va = ((k >> 3) ^ va) & (ADDR_PIDX >> ADDR_PIDX_SHFT);
va \|= (oldptehi & LPTE_AVPN_MASK) <<		va \|= (oldptehi & LPTE_AVPN_MASK) <<
(ADDR_API_SHFT64 - ADDR_PIDX_SHFT);		(ADDR_API_SHFT64 - ADDR_PIDX_SHFT);
PTESYNC();		PTESYNC();
TLBIE(va);		TLBIE(va, oldptehi);
STAT_MOEA64(moea64_pte_valid--);		STAT_MOEA64(moea64_pte_valid--);
STAT_MOEA64(moea64_pte_overflow++);		STAT_MOEA64(moea64_pte_overflow++);
}		}

/*		/*
* Update the PTE as per "Adding a Page Table Entry". Lock is released		* Update the PTE as per "Adding a Page Table Entry". Lock is released
* by setting the high doubleworld.		* by setting the high doubleworld.
*/		*/
pt->pte_lo = htobe64(pvo_pt->pte_lo);		pt->pte_lo = htobe64(pvo_pt->pte_lo);
EIEIO();		EIEIO();
pt->pte_hi = htobe64(pvo_pt->pte_hi);		pt->pte_hi = htobe64(pvo_pt->pte_hi);
PTESYNC();		PTESYNC();

/* Keep statistics */		/* Keep statistics */
STAT_MOEA64(moea64_pte_valid++);		STAT_MOEA64(moea64_pte_valid++);

return (k);		return (k);
}		}

static int64_t		static __inline int64_t
moea64_pte_insert_native(struct pvo_entry *pvo)		moea64_pte_insert_locked(struct pvo_entry pvo, struct lpte insertpt,
		uint64_t mask)
{		{
struct lpte insertpt;
uintptr_t slot;		uintptr_t slot;

/* Initialize PTE */
moea64_pte_from_pvo(pvo, &insertpt);

/* Make sure further insertion is locked out during evictions */
rw_rlock(&moea64_eviction_lock);

/*		/*
* First try primary hash.		* First try primary hash.
*/		*/
pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */		slot = moea64_insert_to_pteg_native(insertpt, pvo->pvo_pte.slot,
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot,		mask \| LPTE_WIRED \| LPTE_LOCKED);
LPTE_VALID \| LPTE_WIRED \| LPTE_LOCKED);
if (slot != -1) {		if (slot != -1) {
rw_runlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot;		pvo->pvo_pte.slot = slot;
return (0);		return (0);
}		}

/*		/*
* Now try secondary hash.		* Now try secondary hash.
*/		*/
pvo->pvo_vaddr ^= PVO_HID;		pvo->pvo_vaddr ^= PVO_HID;
insertpt.pte_hi ^= LPTE_HID;		insertpt->pte_hi ^= LPTE_HID;
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);		pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot,		slot = moea64_insert_to_pteg_native(insertpt, pvo->pvo_pte.slot,
LPTE_VALID \| LPTE_WIRED \| LPTE_LOCKED);		mask \| LPTE_WIRED \| LPTE_LOCKED);
if (slot != -1) {		if (slot != -1) {
rw_runlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot;		pvo->pvo_pte.slot = slot;
return (0);		return (0);
}		}

		return (-1);
		}

		static int64_t
		moea64_pte_insert_native(struct pvo_entry *pvo)
		{
		struct lpte insertpt;
		int64_t ret;

		/* Initialize PTE */
		moea64_pte_from_pvo(pvo, &insertpt);

		/* Make sure further insertion is locked out during evictions */
		rw_rlock(&moea64_eviction_lock);

		pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */
		ret = moea64_pte_insert_locked(pvo, &insertpt, LPTE_VALID);
		if (ret == -1) {
/*		/*
* Out of luck. Find a PTE to sacrifice.		* Out of luck. Find a PTE to sacrifice.
*/		*/

/* Lock out all insertions for a bit */		/* Lock out all insertions for a bit */
if (!rw_try_upgrade(&moea64_eviction_lock)) {		if (!rw_try_upgrade(&moea64_eviction_lock)) {
rw_runlock(&moea64_eviction_lock);		rw_runlock(&moea64_eviction_lock);
rw_wlock(&moea64_eviction_lock);		rw_wlock(&moea64_eviction_lock);
}		}
		/* Don't evict large pages */
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot,		ret = moea64_pte_insert_locked(pvo, &insertpt, LPTE_BIG);
LPTE_WIRED \| LPTE_LOCKED);
if (slot != -1) {
rw_wunlock(&moea64_eviction_lock);		rw_wunlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot;		/* No freeable slots in either PTEG? We're hosed. */
return (0);		if (ret == -1)
}		panic("moea64_pte_insert: overflow");
		} else
		rw_runlock(&moea64_eviction_lock);

/* Try other hash table. Now we're getting desperate... */
pvo->pvo_vaddr ^= PVO_HID;
insertpt.pte_hi ^= LPTE_HID;
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot,
LPTE_WIRED \| LPTE_LOCKED);
if (slot != -1) {
rw_wunlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot;
return (0);		return (0);
}		}

/* No freeable slots in either PTEG? We're hosed. */
rw_wunlock(&moea64_eviction_lock);
panic("moea64_pte_insert: overflow");
return (-1);
}

static void *		static void *
moea64_dump_pmap_native(void ctx, void buf, u_long *nbytes)		moea64_dump_pmap_native(void ctx, void buf, u_long *nbytes)
{		{
struct dump_context *dctx;		struct dump_context *dctx;
u_long ptex, ptex_end;		u_long ptex, ptex_end;

dctx = (struct dump_context *)ctx;		dctx = (struct dump_context *)ctx;
ptex = dctx->ptex;		ptex = dctx->ptex;
ptex_end = ptex + dctx->blksz / sizeof(struct lpte);		ptex_end = ptex + dctx->blksz / sizeof(struct lpte);
ptex_end = MIN(ptex_end, dctx->ptex_end);		ptex_end = MIN(ptex_end, dctx->ptex_end);
nbytes = (ptex_end - ptex) sizeof(struct lpte);		nbytes = (ptex_end - ptex) sizeof(struct lpte);

if (*nbytes == 0)		if (*nbytes == 0)
return (NULL);		return (NULL);

dctx->ptex = ptex_end;		dctx->ptex = ptex_end;
return (__DEVOLATILE(struct lpte *, moea64_pteg_table) + ptex);		return (__DEVOLATILE(struct lpte *, moea64_pteg_table) + ptex);
		}

		static __inline uint64_t
		moea64_vpn_from_pte(uint64_t ptehi, uintptr_t slot)
		{
		uint64_t pgn, pgnlo, vsid;

		vsid = (ptehi & LPTE_AVA_MASK) >> LPTE_VSID_SHIFT;
		if ((ptehi & LPTE_HID) != 0)
		slot ^= (moea64_pteg_mask << 3);
		pgnlo = ((vsid & VSID_HASH_MASK) ^ (slot >> 3)) & EA_PAGELO_MASK;
		pgn = ((ptehi & LPTE_AVA_PGNHI_MASK) << (EA_PAGELO_SHIFT -
		LPTE_AVA_PGNHI_SHIFT)) \| pgnlo;
		return ((vsid << 16) \| pgn);
		}

		static __inline int64_t
		moea64_pte_unset_sp_locked(struct pvo_entry *pvo)
		{
		volatile struct lpte *pt;
		uint64_t ptehi, refchg, vpn;
		vm_offset_t eva;
		pmap_t pm;

		pm = pvo->pvo_pmap;
		refchg = 0;
		eva = PVO_VADDR(pvo) + SP_SIZE;

		for (; pvo != NULL && PVO_VADDR(pvo) < eva;
		pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
		pt = moea64_pteg_table + pvo->pvo_pte.slot;
		ptehi = be64toh(pt->pte_hi);
		if ((ptehi & LPTE_AVPN_MASK) !=
		moea64_pte_vpn_from_pvo_vpn(pvo)) {
		/* Evicted: invalidate new entry */
		STAT_MOEA64(moea64_pte_overflow--);
		vpn = moea64_vpn_from_pte(ptehi, pvo->pvo_pte.slot);
		CTR1(KTR_PMAP, "Evicted page in pte_unset_sp: vpn=%jx",
		(uintmax_t)vpn);
		/* Assume evicted page was modified */
		refchg \|= LPTE_CHG;
		} else
		vpn = pvo->pvo_vpn;

		refchg \|= moea64_pte_unset_locked(pt, vpn);
		}

		return (refchg);
		}

		static int64_t
		moea64_pte_unset_sp_native(struct pvo_entry *pvo)
		{
		uint64_t refchg;

		PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
		KASSERT((PVO_VADDR(pvo) & SP_MASK) == 0,
		("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));

		rw_rlock(&moea64_eviction_lock);
		refchg = moea64_pte_unset_sp_locked(pvo);
		rw_runlock(&moea64_eviction_lock);

		return (refchg);
		}

		static __inline int64_t
		moea64_pte_insert_sp_locked(struct pvo_entry *pvo)
		{
		struct lpte insertpt;
		int64_t ret;
		vm_offset_t eva;
		pmap_t pm;

		pm = pvo->pvo_pmap;
		eva = PVO_VADDR(pvo) + SP_SIZE;

		for (; pvo != NULL && PVO_VADDR(pvo) < eva;
		pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
		moea64_pte_from_pvo(pvo, &insertpt);
		pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */

		ret = moea64_pte_insert_locked(pvo, &insertpt, LPTE_VALID);
		if (ret == -1) {
		/* Lock out all insertions for a bit */
		if (!rw_try_upgrade(&moea64_eviction_lock)) {
		rw_runlock(&moea64_eviction_lock);
		rw_wlock(&moea64_eviction_lock);
		}
		/* Don't evict large pages */
		ret = moea64_pte_insert_locked(pvo, &insertpt,
		LPTE_BIG);
		rw_downgrade(&moea64_eviction_lock);
		/* No freeable slots in either PTEG? We're hosed. */
		if (ret == -1)
		panic("moea64_pte_insert_sp: overflow");
		}
		}

		return (0);
		}

		static int64_t
		moea64_pte_insert_sp_native(struct pvo_entry *pvo)
		{
		PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
		KASSERT((PVO_VADDR(pvo) & SP_MASK) == 0,
		("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));

		rw_rlock(&moea64_eviction_lock);
		moea64_pte_insert_sp_locked(pvo);
		rw_runlock(&moea64_eviction_lock);

		return (0);
		}

		static int64_t
		moea64_pte_replace_sp_native(struct pvo_entry *pvo)
		{
		uint64_t refchg;

		PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
		KASSERT((PVO_VADDR(pvo) & SP_MASK) == 0,
		("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));

		rw_rlock(&moea64_eviction_lock);
		refchg = moea64_pte_unset_sp_locked(pvo);
		moea64_pte_insert_sp_locked(pvo);
		rw_runlock(&moea64_eviction_lock);

		return (refchg);
}		}