Page MenuHomeFreeBSD
Differential D25237 Diff 79262 head/sys/powerpc/aim/moea64_native.c

Changeset View

Standalone View

View Options

head/sys/powerpc/aim/moea64_native.c

Show First 20 Lines • Show All 126 Lines • ▼ Show 20 Lines
#define SYNC() __asm __volatile("sync"); #define SYNC() __asm __volatile("sync");
#define EIEIO() __asm __volatile("eieio"); #define EIEIO() __asm __volatile("eieio");
#define VSID_HASH_MASK 0x0000007fffffffffULL #define VSID_HASH_MASK 0x0000007fffffffffULL
/* POWER9 only permits a 64k partition table size. */ /* POWER9 only permits a 64k partition table size. */
#define PART_SIZE 0x10000 #define PART_SIZE 0x10000
/* Actual page sizes (to be used with tlbie, when L=0) */
#define AP_4K 0x00
#define AP_16M 0x80
#define LPTE_KERNEL_VSID_BIT (KERNEL_VSID_BIT << \
(16 - (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)))
/* Abbreviated Virtual Address Page - high bits */
#define LPTE_AVA_PGNHI_MASK 0x0000000000000F80ULL
#define LPTE_AVA_PGNHI_SHIFT 7
/* Effective Address Page - low bits */
#define EA_PAGELO_MASK 0x7ffULL
#define EA_PAGELO_SHIFT 11
static bool moea64_crop_tlbie; static bool moea64_crop_tlbie;
static bool moea64_need_lock; static bool moea64_need_lock;
/*
* The tlbie instruction has two forms: an old one used by PowerISA
* 2.03 and prior, and a newer one used by PowerISA 2.06 and later.
* We need to support both.
*/
static __inline void static __inline void
TLBIE(uint64_t vpn) { TLBIE(uint64_t vpn, uint64_t oldptehi)
{
#ifndef __powerpc64__ #ifndef __powerpc64__
register_t vpn_hi, vpn_lo; register_t vpn_hi, vpn_lo;
register_t msr; register_t msr;
register_t scratch, intr; register_t scratch, intr;
#endif #endif
static volatile u_int tlbie_lock = 0; static volatile u_int tlbie_lock = 0;
bool need_lock = moea64_need_lock; bool need_lock = moea64_need_lock;
vpn <<= ADDR_PIDX_SHFT; vpn <<= ADDR_PIDX_SHFT;
/* Hobo spinlock: we need stronger guarantees than mutexes provide */ /* Hobo spinlock: we need stronger guarantees than mutexes provide */
if (need_lock) { if (need_lock) {
while (!atomic_cmpset_int(&tlbie_lock, 0, 1)); while (!atomic_cmpset_int(&tlbie_lock, 0, 1));
isync(); /* Flush instruction queue once lock acquired */ isync(); /* Flush instruction queue once lock acquired */
if (moea64_crop_tlbie) if (moea64_crop_tlbie) {
vpn &= ~(0xffffULL << 48); vpn &= ~(0xffffULL << 48);
#ifdef __powerpc64__
if ((oldptehi & LPTE_BIG) != 0)
__asm __volatile("tlbie %0, 1" :: "r"(vpn) :
"memory");
else
__asm __volatile("tlbie %0, 0" :: "r"(vpn) :
"memory");
__asm __volatile("eieio; tlbsync; ptesync" :::
"memory");
goto done;
#endif
} }
}
#ifdef __powerpc64__ #ifdef __powerpc64__
/* /*
* If this page has LPTE_BIG set and is from userspace, then
* it must be a superpage with 4KB base/16MB actual page size.
*/
if ((oldptehi & LPTE_BIG) != 0 &&
(oldptehi & LPTE_KERNEL_VSID_BIT) == 0)
vpn |= AP_16M;
/*
* Explicitly clobber r0. The tlbie instruction has two forms: an old * Explicitly clobber r0. The tlbie instruction has two forms: an old
* one used by PowerISA 2.03 and prior, and a newer one used by PowerISA * one used by PowerISA 2.03 and prior, and a newer one used by PowerISA
* 2.06 (maybe 2.05?) and later. We need to support both, and it just * 2.06 (maybe 2.05?) and later. We need to support both, and it just
* so happens that since we use 4k pages we can simply zero out r0, and * so happens that since we use 4k pages we can simply zero out r0, and
* clobber it, and the assembler will interpret the single-operand form * clobber it, and the assembler will interpret the single-operand form
* of tlbie as having RB set, and everything else as 0. The RS operand * of tlbie as having RB set, and everything else as 0. The RS operand
* in the newer form is in the same position as the L(page size) bit of * in the newer form is in the same position as the L(page size) bit of
* the old form, so a slong as RS is 0, we're good on both sides. * the old form, so a slong as RS is 0, we're good on both sides.
*/ */
__asm __volatile("li 0, 0 \n tlbie %0" :: "r"(vpn) : "r0", "memory"); __asm __volatile("li 0, 0 \n tlbie %0, 0" :: "r"(vpn) : "r0", "memory");
__asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); __asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
#else #else
vpn_hi = (uint32_t)(vpn >> 32); vpn_hi = (uint32_t)(vpn >> 32);
vpn_lo = (uint32_t)vpn; vpn_lo = (uint32_t)vpn;
intr = intr_disable(); intr = intr_disable();
__asm __volatile("\ __asm __volatile("\
mfmsr %0; \ mfmsr %0; \
Show All 9 Lines __asm __volatile("\
eieio; \ eieio; \
tlbsync; \ tlbsync; \
ptesync;" ptesync;"
: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1) : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
: "memory"); : "memory");
intr_restore(intr); intr_restore(intr);
#endif #endif
done:
/* No barriers or special ops -- taken care of by ptesync above */ /* No barriers or special ops -- taken care of by ptesync above */
if (need_lock) if (need_lock)
tlbie_lock = 0; tlbie_lock = 0;
} }
#define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR) #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR)
#define ENABLE_TRANS(msr) mtmsr(msr) #define ENABLE_TRANS(msr) mtmsr(msr)
Show All 14 Lines
/* /*
* PTE calls. * PTE calls.
*/ */
static int64_t moea64_pte_insert_native(struct pvo_entry *); static int64_t moea64_pte_insert_native(struct pvo_entry *);
static int64_t moea64_pte_synch_native(struct pvo_entry *); static int64_t moea64_pte_synch_native(struct pvo_entry *);
static int64_t moea64_pte_clear_native(struct pvo_entry *, uint64_t); static int64_t moea64_pte_clear_native(struct pvo_entry *, uint64_t);
static int64_t moea64_pte_replace_native(struct pvo_entry *, int); static int64_t moea64_pte_replace_native(struct pvo_entry *, int);
static int64_t moea64_pte_unset_native(struct pvo_entry *); static int64_t moea64_pte_unset_native(struct pvo_entry *);
static int64_t moea64_pte_insert_sp_native(struct pvo_entry *);
static int64_t moea64_pte_unset_sp_native(struct pvo_entry *);
static int64_t moea64_pte_replace_sp_native(struct pvo_entry *);
/* /*
* Utility routines. * Utility routines.
*/ */
static void moea64_bootstrap_native( static void moea64_bootstrap_native(
vm_offset_t kernelstart, vm_offset_t kernelend); vm_offset_t kernelstart, vm_offset_t kernelend);
static void moea64_cpu_bootstrap_native(int ap); static void moea64_cpu_bootstrap_native(int ap);
static void tlbia(void); static void tlbia(void);
static void moea64_install_native(void); static void moea64_install_native(void);
static struct pmap_funcs moea64_native_methods = { static struct pmap_funcs moea64_native_methods = {
.install = moea64_install_native, .install = moea64_install_native,
/* Internal interfaces */ /* Internal interfaces */
.bootstrap = moea64_bootstrap_native, .bootstrap = moea64_bootstrap_native,
.cpu_bootstrap = moea64_cpu_bootstrap_native, .cpu_bootstrap = moea64_cpu_bootstrap_native,
.dumpsys_dump_pmap = moea64_dump_pmap_native, .dumpsys_dump_pmap = moea64_dump_pmap_native,
}; };
static struct moea64_funcs moea64_native_funcs = { static struct moea64_funcs moea64_native_funcs = {
.pte_synch = moea64_pte_synch_native, .pte_synch = moea64_pte_synch_native,
.pte_clear = moea64_pte_clear_native, .pte_clear = moea64_pte_clear_native,
.pte_unset = moea64_pte_unset_native, .pte_unset = moea64_pte_unset_native,
.pte_replace = moea64_pte_replace_native, .pte_replace = moea64_pte_replace_native,
.pte_insert = moea64_pte_insert_native, .pte_insert = moea64_pte_insert_native,
.pte_insert_sp = moea64_pte_insert_sp_native,
.pte_unset_sp = moea64_pte_unset_sp_native,
.pte_replace_sp = moea64_pte_replace_sp_native,
}; };
MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods, oea64_mmu); MMU_DEF_INHERIT(oea64_mmu_native, MMU_TYPE_G5, moea64_native_methods, oea64_mmu);
static void static void
moea64_install_native() moea64_install_native()
{ {
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines
#if BYTE_ORDER == BIG_ENDIAN #if BYTE_ORDER == BIG_ENDIAN
((uint8_t *)(&properpt.pte_lo))[6]; ((uint8_t *)(&properpt.pte_lo))[6];
#else #else
((uint8_t *)(&properpt.pte_lo))[1]; ((uint8_t *)(&properpt.pte_lo))[1];
#endif #endif
rw_runlock(&moea64_eviction_lock); rw_runlock(&moea64_eviction_lock);
critical_enter(); critical_enter();
TLBIE(pvo->pvo_vpn); TLBIE(pvo->pvo_vpn, properpt.pte_hi);
critical_exit(); critical_exit();
} else { } else {
rw_runlock(&moea64_eviction_lock); rw_runlock(&moea64_eviction_lock);
ptelo = moea64_pte_unset_native(pvo); ptelo = moea64_pte_unset_native(pvo);
moea64_pte_insert_native(pvo); moea64_pte_insert_native(pvo);
} }
return (ptelo & (LPTE_REF | LPTE_CHG)); return (ptelo & (LPTE_REF | LPTE_CHG));
} }
static int64_t static __always_inline int64_t
moea64_pte_unset_native(struct pvo_entry *pvo) moea64_pte_unset_locked(volatile struct lpte *pt, uint64_t vpn)
{ {
volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot; uint64_t ptelo;
uint64_t ptelo, pvo_ptevpn;
pvo_ptevpn = moea64_pte_vpn_from_pvo_vpn(pvo);
rw_rlock(&moea64_eviction_lock);
if ((be64toh(pt->pte_hi) & LPTE_AVPN_MASK) != pvo_ptevpn) {
/* Evicted */
STAT_MOEA64(moea64_pte_overflow--);
rw_runlock(&moea64_eviction_lock);
return (-1);
}
/* /*
* Invalidate the pte, briefly locking it to collect RC bits. No * Invalidate the pte, briefly locking it to collect RC bits. No
* atomics needed since this is protected against eviction by the lock. * atomics needed since this is protected against eviction by the lock.
*/ */
isync(); isync();
critical_enter(); critical_enter();
pt->pte_hi = htobe64((be64toh(pt->pte_hi) & ~LPTE_VALID) | LPTE_LOCKED); pt->pte_hi = htobe64((be64toh(pt->pte_hi) & ~LPTE_VALID) | LPTE_LOCKED);
PTESYNC(); PTESYNC();
TLBIE(pvo->pvo_vpn); TLBIE(vpn, pt->pte_hi);
ptelo = be64toh(pt->pte_lo); ptelo = be64toh(pt->pte_lo);
*((volatile int32_t *)(&pt->pte_hi) + 1) = 0; /* Release lock */ *((volatile int32_t *)(&pt->pte_hi) + 1) = 0; /* Release lock */
critical_exit(); critical_exit();
rw_runlock(&moea64_eviction_lock);
/* Keep statistics */ /* Keep statistics */
STAT_MOEA64(moea64_pte_valid--); STAT_MOEA64(moea64_pte_valid--);
return (ptelo & (LPTE_CHG | LPTE_REF)); return (ptelo & (LPTE_CHG | LPTE_REF));
} }
static int64_t static int64_t
moea64_pte_unset_native(struct pvo_entry *pvo)
{
volatile struct lpte *pt = moea64_pteg_table + pvo->pvo_pte.slot;
int64_t ret;
uint64_t pvo_ptevpn;
pvo_ptevpn = moea64_pte_vpn_from_pvo_vpn(pvo);
rw_rlock(&moea64_eviction_lock);
if ((be64toh(pt->pte_hi & LPTE_AVPN_MASK)) != pvo_ptevpn) {
/* Evicted */
STAT_MOEA64(moea64_pte_overflow--);
ret = -1;
} else
ret = moea64_pte_unset_locked(pt, pvo->pvo_vpn);
rw_runlock(&moea64_eviction_lock);
return (ret);
}
static int64_t
moea64_pte_replace_inval_native(struct pvo_entry *pvo, moea64_pte_replace_inval_native(struct pvo_entry *pvo,
volatile struct lpte *pt) volatile struct lpte *pt)
{ {
struct lpte properpt; struct lpte properpt;
uint64_t ptelo; uint64_t ptelo;
moea64_pte_from_pvo(pvo, &properpt); moea64_pte_from_pvo(pvo, &properpt);
Show All 9 Linesmoea64_pte_replace_inval_native(struct pvo_entry *pvo,
/* /*
* Replace the pte, briefly locking it to collect RC bits. No * Replace the pte, briefly locking it to collect RC bits. No
* atomics needed since this is protected against eviction by the lock. * atomics needed since this is protected against eviction by the lock.
*/ */
isync(); isync();
critical_enter(); critical_enter();
pt->pte_hi = htobe64((be64toh(pt->pte_hi) & ~LPTE_VALID) | LPTE_LOCKED); pt->pte_hi = htobe64((be64toh(pt->pte_hi) & ~LPTE_VALID) | LPTE_LOCKED);
PTESYNC(); PTESYNC();
TLBIE(pvo->pvo_vpn); TLBIE(pvo->pvo_vpn, pt->pte_hi);
ptelo = be64toh(pt->pte_lo); ptelo = be64toh(pt->pte_lo);
EIEIO(); EIEIO();
pt->pte_lo = htobe64(properpt.pte_lo); pt->pte_lo = htobe64(properpt.pte_lo);
EIEIO(); EIEIO();
pt->pte_hi = htobe64(properpt.pte_hi); /* Release lock */ pt->pte_hi = htobe64(properpt.pte_hi); /* Release lock */
PTESYNC(); PTESYNC();
critical_exit(); critical_exit();
rw_runlock(&moea64_eviction_lock); rw_runlock(&moea64_eviction_lock);
▲ Show 20 Lines • Show All 323 Lines • ▼ Show 20 Lines if (oldptehi & LPTE_VALID) {
if (oldptehi & LPTE_HID) if (oldptehi & LPTE_HID)
va = (((k >> 3) ^ moea64_pteg_mask) ^ va) & va = (((k >> 3) ^ moea64_pteg_mask) ^ va) &
(ADDR_PIDX >> ADDR_PIDX_SHFT); (ADDR_PIDX >> ADDR_PIDX_SHFT);
else else
va = ((k >> 3) ^ va) & (ADDR_PIDX >> ADDR_PIDX_SHFT); va = ((k >> 3) ^ va) & (ADDR_PIDX >> ADDR_PIDX_SHFT);
va |= (oldptehi & LPTE_AVPN_MASK) << va |= (oldptehi & LPTE_AVPN_MASK) <<
(ADDR_API_SHFT64 - ADDR_PIDX_SHFT); (ADDR_API_SHFT64 - ADDR_PIDX_SHFT);
PTESYNC(); PTESYNC();
TLBIE(va); TLBIE(va, oldptehi);
STAT_MOEA64(moea64_pte_valid--); STAT_MOEA64(moea64_pte_valid--);
STAT_MOEA64(moea64_pte_overflow++); STAT_MOEA64(moea64_pte_overflow++);
} }
/* /*
* Update the PTE as per "Adding a Page Table Entry". Lock is released * Update the PTE as per "Adding a Page Table Entry". Lock is released
* by setting the high doubleworld. * by setting the high doubleworld.
*/ */
pt->pte_lo = htobe64(pvo_pt->pte_lo); pt->pte_lo = htobe64(pvo_pt->pte_lo);
EIEIO(); EIEIO();
pt->pte_hi = htobe64(pvo_pt->pte_hi); pt->pte_hi = htobe64(pvo_pt->pte_hi);
PTESYNC(); PTESYNC();
/* Keep statistics */ /* Keep statistics */
STAT_MOEA64(moea64_pte_valid++); STAT_MOEA64(moea64_pte_valid++);
return (k); return (k);
} }
static int64_t static __always_inline int64_t
moea64_pte_insert_native(struct pvo_entry *pvo) moea64_pte_insert_locked(struct pvo_entry *pvo, struct lpte *insertpt,
uint64_t mask)
{ {
struct lpte insertpt;
uintptr_t slot; uintptr_t slot;
/* Initialize PTE */
moea64_pte_from_pvo(pvo, &insertpt);
/* Make sure further insertion is locked out during evictions */
rw_rlock(&moea64_eviction_lock);
/* /*
* First try primary hash. * First try primary hash.
*/ */
pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */ slot = moea64_insert_to_pteg_native(insertpt, pvo->pvo_pte.slot,
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, mask | LPTE_WIRED | LPTE_LOCKED);
LPTE_VALID | LPTE_WIRED | LPTE_LOCKED);
if (slot != -1) { if (slot != -1) {
rw_runlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot; pvo->pvo_pte.slot = slot;
return (0); return (0);
} }
/* /*
* Now try secondary hash. * Now try secondary hash.
*/ */
pvo->pvo_vaddr ^= PVO_HID; pvo->pvo_vaddr ^= PVO_HID;
insertpt.pte_hi ^= LPTE_HID; insertpt->pte_hi ^= LPTE_HID;
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, slot = moea64_insert_to_pteg_native(insertpt, pvo->pvo_pte.slot,
LPTE_VALID | LPTE_WIRED | LPTE_LOCKED); mask | LPTE_WIRED | LPTE_LOCKED);
if (slot != -1) { if (slot != -1) {
rw_runlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot; pvo->pvo_pte.slot = slot;
return (0); return (0);
} }
return (-1);
}
static int64_t
moea64_pte_insert_native(struct pvo_entry *pvo)
{
struct lpte insertpt;
int64_t ret;
/* Initialize PTE */
moea64_pte_from_pvo(pvo, &insertpt);
/* Make sure further insertion is locked out during evictions */
rw_rlock(&moea64_eviction_lock);
pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */
ret = moea64_pte_insert_locked(pvo, &insertpt, LPTE_VALID);
if (ret == -1) {
/* /*
* Out of luck. Find a PTE to sacrifice. * Out of luck. Find a PTE to sacrifice.
*/ */
/* Lock out all insertions for a bit */ /* Lock out all insertions for a bit */
if (!rw_try_upgrade(&moea64_eviction_lock)) { if (!rw_try_upgrade(&moea64_eviction_lock)) {
rw_runlock(&moea64_eviction_lock); rw_runlock(&moea64_eviction_lock);
rw_wlock(&moea64_eviction_lock); rw_wlock(&moea64_eviction_lock);
} }
/* Don't evict large pages */
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot, ret = moea64_pte_insert_locked(pvo, &insertpt, LPTE_BIG);
LPTE_WIRED | LPTE_LOCKED);
if (slot != -1) {
rw_wunlock(&moea64_eviction_lock); rw_wunlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot; /* No freeable slots in either PTEG? We're hosed. */
return (0); if (ret == -1)
} panic("moea64_pte_insert: overflow");
} else
rw_runlock(&moea64_eviction_lock);
/* Try other hash table. Now we're getting desperate... */
pvo->pvo_vaddr ^= PVO_HID;
insertpt.pte_hi ^= LPTE_HID;
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
slot = moea64_insert_to_pteg_native(&insertpt, pvo->pvo_pte.slot,
LPTE_WIRED | LPTE_LOCKED);
if (slot != -1) {
rw_wunlock(&moea64_eviction_lock);
pvo->pvo_pte.slot = slot;
return (0); return (0);
} }
/* No freeable slots in either PTEG? We're hosed. */
rw_wunlock(&moea64_eviction_lock);
panic("moea64_pte_insert: overflow");
return (-1);
}
static void * static void *
moea64_dump_pmap_native(void *ctx, void *buf, u_long *nbytes) moea64_dump_pmap_native(void *ctx, void *buf, u_long *nbytes)
{ {
struct dump_context *dctx; struct dump_context *dctx;
u_long ptex, ptex_end; u_long ptex, ptex_end;
dctx = (struct dump_context *)ctx; dctx = (struct dump_context *)ctx;
ptex = dctx->ptex; ptex = dctx->ptex;
ptex_end = ptex + dctx->blksz / sizeof(struct lpte); ptex_end = ptex + dctx->blksz / sizeof(struct lpte);
ptex_end = MIN(ptex_end, dctx->ptex_end); ptex_end = MIN(ptex_end, dctx->ptex_end);
*nbytes = (ptex_end - ptex) * sizeof(struct lpte); *nbytes = (ptex_end - ptex) * sizeof(struct lpte);
if (*nbytes == 0) if (*nbytes == 0)
return (NULL); return (NULL);
dctx->ptex = ptex_end; dctx->ptex = ptex_end;
return (__DEVOLATILE(struct lpte *, moea64_pteg_table) + ptex); return (__DEVOLATILE(struct lpte *, moea64_pteg_table) + ptex);
}
static __always_inline uint64_t
moea64_vpn_from_pte(uint64_t ptehi, uintptr_t slot)
{
uint64_t pgn, pgnlo, vsid;
vsid = (ptehi & LPTE_AVA_MASK) >> LPTE_VSID_SHIFT;
if ((ptehi & LPTE_HID) != 0)
slot ^= (moea64_pteg_mask << 3);
pgnlo = ((vsid & VSID_HASH_MASK) ^ (slot >> 3)) & EA_PAGELO_MASK;
pgn = ((ptehi & LPTE_AVA_PGNHI_MASK) << (EA_PAGELO_SHIFT -
LPTE_AVA_PGNHI_SHIFT)) | pgnlo;
return ((vsid << 16) | pgn);
}
static __always_inline int64_t
moea64_pte_unset_sp_locked(struct pvo_entry *pvo)
{
volatile struct lpte *pt;
uint64_t ptehi, refchg, vpn;
vm_offset_t eva;
pmap_t pm;
pm = pvo->pvo_pmap;
refchg = 0;
eva = PVO_VADDR(pvo) + HPT_SP_SIZE;
for (; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
pt = moea64_pteg_table + pvo->pvo_pte.slot;
ptehi = be64toh(pt->pte_hi);
if ((ptehi & LPTE_AVPN_MASK) !=
moea64_pte_vpn_from_pvo_vpn(pvo)) {
/* Evicted: invalidate new entry */
STAT_MOEA64(moea64_pte_overflow--);
vpn = moea64_vpn_from_pte(ptehi, pvo->pvo_pte.slot);
CTR1(KTR_PMAP, "Evicted page in pte_unset_sp: vpn=%jx",
(uintmax_t)vpn);
/* Assume evicted page was modified */
refchg |= LPTE_CHG;
} else
vpn = pvo->pvo_vpn;
refchg |= moea64_pte_unset_locked(pt, vpn);
}
return (refchg);
}
static int64_t
moea64_pte_unset_sp_native(struct pvo_entry *pvo)
{
uint64_t refchg;
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0,
("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));
rw_rlock(&moea64_eviction_lock);
refchg = moea64_pte_unset_sp_locked(pvo);
rw_runlock(&moea64_eviction_lock);
return (refchg);
}
static __always_inline int64_t
moea64_pte_insert_sp_locked(struct pvo_entry *pvo)
{
struct lpte insertpt;
int64_t ret;
vm_offset_t eva;
pmap_t pm;
pm = pvo->pvo_pmap;
eva = PVO_VADDR(pvo) + HPT_SP_SIZE;
for (; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
moea64_pte_from_pvo(pvo, &insertpt);
pvo->pvo_pte.slot &= ~7ULL; /* Base slot address */
ret = moea64_pte_insert_locked(pvo, &insertpt, LPTE_VALID);
if (ret == -1) {
/* Lock out all insertions for a bit */
if (!rw_try_upgrade(&moea64_eviction_lock)) {
rw_runlock(&moea64_eviction_lock);
rw_wlock(&moea64_eviction_lock);
}
/* Don't evict large pages */
ret = moea64_pte_insert_locked(pvo, &insertpt,
LPTE_BIG);
rw_downgrade(&moea64_eviction_lock);
/* No freeable slots in either PTEG? We're hosed. */
if (ret == -1)
panic("moea64_pte_insert_sp: overflow");
}
}
return (0);
}
static int64_t
moea64_pte_insert_sp_native(struct pvo_entry *pvo)
{
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0,
("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));
rw_rlock(&moea64_eviction_lock);
moea64_pte_insert_sp_locked(pvo);
rw_runlock(&moea64_eviction_lock);
return (0);
}
static int64_t
moea64_pte_replace_sp_native(struct pvo_entry *pvo)
{
uint64_t refchg;
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
KASSERT((PVO_VADDR(pvo) & HPT_SP_MASK) == 0,
("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));
rw_rlock(&moea64_eviction_lock);
refchg = moea64_pte_unset_sp_locked(pvo);
moea64_pte_insert_sp_locked(pvo);
rw_runlock(&moea64_eviction_lock);
return (refchg);
} }