Page MenuHomeFreeBSD
Differential D25237 Diff 72993 sys/powerpc/aim/mmu_oea64.c

Changeset View

Standalone View

View Options

sys/powerpc/aim/mmu_oea64.c

Show First 20 Lines • Show All 74 Lines • ▼ Show 20 Lines
#include <vm/vm_param.h> #include <vm/vm_param.h>
#include <vm/vm_kern.h> #include <vm/vm_kern.h>
#include <vm/vm_page.h> #include <vm/vm_page.h>
#include <vm/vm_phys.h> #include <vm/vm_phys.h>
#include <vm/vm_map.h> #include <vm/vm_map.h>
#include <vm/vm_object.h> #include <vm/vm_object.h>
#include <vm/vm_extern.h> #include <vm/vm_extern.h>
#include <vm/vm_pageout.h> #include <vm/vm_pageout.h>
#include <vm/vm_reserv.h>
#include <vm/uma.h> #include <vm/uma.h>
#include <machine/_inttypes.h> #include <machine/_inttypes.h>
#include <machine/cpu.h> #include <machine/cpu.h>
#include <machine/ifunc.h> #include <machine/ifunc.h>
#include <machine/platform.h> #include <machine/platform.h>
#include <machine/frame.h> #include <machine/frame.h>
#include <machine/md_var.h> #include <machine/md_var.h>
▲ Show 20 Lines • Show All 132 Lines • ▼ Show 20 Lines
vm_offset_t moea64_scratchpage_va[2]; vm_offset_t moea64_scratchpage_va[2];
struct pvo_entry *moea64_scratchpage_pvo[2]; struct pvo_entry *moea64_scratchpage_pvo[2];
struct mtx moea64_scratchpage_mtx; struct mtx moea64_scratchpage_mtx;
uint64_t moea64_large_page_mask = 0; uint64_t moea64_large_page_mask = 0;
uint64_t moea64_large_page_size = 0; uint64_t moea64_large_page_size = 0;
int moea64_large_page_shift = 0; int moea64_large_page_shift = 0;
bool moea64_has_lp_4k_16m = false;
markjUnsubmitted
Not Done
Inline Actions

There seems to be a mix of "lp" and "sp" to denote "superpage". I think we should pick one and stick with it.

markj: There seems to be a mix of "lp" and "sp" to denote "superpage". I think we should pick one and…
luporlAuthorUnsubmitted
Done
Inline Actions

Yeah, I've used "sp" (superpage) during most of the changes. The idea was to call the parts that deal with the superpage feature (like promotion and demotion) "sp", to match how other arch's call them, but most places in PPC code use "lp" (large page) when referring to a page whose size is of several 4k pages, although currently PPC64 uses large pages for DMAP only.

I've used "lp" here because this is related to a PPC-specific hardware feature and closely related to others already named as large page.

For the rest of the code, changing "lp" and "large_page" to "sp" and "superpage" doesn't seem a good idea to me. I would also prefer to leave the parts that deal with promotion, demotion, etc. as superpages, but I guess it would be ok to rename them to "large_pages" too.
What do you think?

luporl: Yeah, I've used "sp" (superpage) during most of the changes. The idea was to call the parts…
/* /*
* PVO calls. * PVO calls.
*/ */
static int moea64_pvo_enter(struct pvo_entry *pvo, static int moea64_pvo_enter(struct pvo_entry *pvo,
struct pvo_head *pvo_head, struct pvo_entry **oldpvo); struct pvo_head *pvo_head, struct pvo_entry **oldpvo);
static void moea64_pvo_remove_from_pmap(struct pvo_entry *pvo); static void moea64_pvo_remove_from_pmap(struct pvo_entry *pvo);
static void moea64_pvo_remove_from_page(struct pvo_entry *pvo); static void moea64_pvo_remove_from_page(struct pvo_entry *pvo);
static void moea64_pvo_remove_from_page_locked( static void moea64_pvo_remove_from_page_locked(
struct pvo_entry *pvo, vm_page_t m); struct pvo_entry *pvo, vm_page_t m);
static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t); static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t);
/* /*
* Utility routines. * Utility routines.
*/ */
static boolean_t moea64_query_bit(vm_page_t, uint64_t); static boolean_t moea64_query_bit(vm_page_t, uint64_t);
static u_int moea64_clear_bit(vm_page_t, uint64_t); static u_int moea64_clear_bit(vm_page_t, uint64_t);
static void moea64_kremove(vm_offset_t); static void moea64_kremove(vm_offset_t);
static void moea64_syncicache(pmap_t pmap, vm_offset_t va, static void moea64_syncicache(pmap_t pmap, vm_offset_t va,
vm_paddr_t pa, vm_size_t sz); vm_paddr_t pa, vm_size_t sz);
static void moea64_pmap_init_qpages(void); static void moea64_pmap_init_qpages(void);
static void moea64_remove_locked(pmap_t, vm_offset_t,
vm_offset_t, struct pvo_dlist *);
/* /*
* Superpages data and routines.
*/
#ifdef PPC_SUPERPAGES
#define SP_SHIFT (VM_LEVEL_0_ORDER + PAGE_SHIFT)
#define SP_SIZE (1 << SP_SHIFT)
#define SP_MASK (SP_SIZE - 1)
#define SP_PAGES (1 << VM_LEVEL_0_ORDER)
markjUnsubmitted
Done
Inline Actions

What about protection bits? Oh I see, those are checked separately. It might be worth adding a sentence explaining that.

Why is PVO_LARGE included?

markj: What about protection bits? Oh I see, those are checked separately. It might be worth adding a…
luporlAuthorUnsubmitted
Done
Inline Actions

Right, protection bits are in another PVO field and are checked separately. I'll add a comment about it, and maybe rename PVO_PROMOTE to PVO_FLAGS_PROMOTE, to make it clear.

PVO_LARGE was added to make sure all pages are of the same size (4K), but I guess it is not necessary to check it during promotion, because if any non-4K page is found during promotion, it means the logic is really broken.

luporl: Right, protection bits are in another PVO field and are checked separately. I'll add a comment…
/* PVO (vaddr) bits that must match for promotion to succeed. */
#define PVO_PROMOTE (PVO_WIRED | PVO_MANAGED | PVO_LARGE | \
PVO_PTEGIDX_VALID)
#define PVO_IS_SP(pvo) (((pvo)->pvo_vaddr & PVO_LARGE) && \
(pvo)->pvo_pmap != kernel_pmap)
/* Get physical address from PVO. */
#define PVO_PADDR(pvo) moea64_pvo_paddr(pvo)
/* MD page flag indicating that the page is a superpage. */
#define MDPG_ATTR_SP 0x40000000
/* data */
markjUnsubmitted
Done
Inline Actions

pg_ps_enabled is kind of a misnomer: "ps" refers to PG_PS, which is the name of the bit in x86 page tables that indicates that a PDE or PDPE maps a superpage. Other arches just call it superpages_enabled.

markj: pg_ps_enabled is kind of a misnomer: "ps" refers to PG_PS, which is the name of the bit in x86…
luporlAuthorUnsubmitted
Done
Inline Actions

Ah, ok, thanks for the explanation, I'll change it.

luporl: Ah, ok, thanks for the explanation, I'll change it.
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0,
"VM/pmap parameters");
static int sp_enabled = 0;
SYSCTL_INT(_vm_pmap, OID_AUTO, sp_enabled, CTLFLAG_RDTUN,
jhibbitsUnsubmitted
Done
Inline Actions

Radix (and amd64 at least) already uses vm.pmap.pg_ps_enabled, can you use that instead, for consistency?

jhibbits: Radix (and amd64 at least) already uses vm.pmap.pg_ps_enabled, can you use that instead, for…
luporlAuthorUnsubmitted
Done
Inline Actions

Sure.

luporl: Sure.
&sp_enabled, 0, "Enable support for transparent superpages");
static SYSCTL_NODE(_vm_pmap, OID_AUTO, sp, CTLFLAG_RD, 0,
"SP page mapping counters");
static u_long sp_demotions;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, demotions, CTLFLAG_RD,
&sp_demotions, 0,
"SP page demotions");
static u_long sp_mappings;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, mappings, CTLFLAG_RD,
&sp_mappings, 0,
"SP page mappings");
static u_long sp_p_failures;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_failures, CTLFLAG_RD,
&sp_p_failures, 0,
"SP page promotion failures");
static u_long sp_promotions;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, promotions, CTLFLAG_RD,
&sp_promotions, 0,
"SP page promotions");
/* protos */
static bool moea64_ps_enabled(pmap_t);
static void moea64_align_superpage(vm_object_t, vm_ooffset_t,
vm_offset_t *, vm_size_t);
static int moea64_sp_enter(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind);
static struct pvo_entry *moea64_sp_remove(struct pvo_entry *sp,
struct pvo_dlist *tofree);
static int moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m);
static void moea64_sp_demote_aligned(struct pvo_entry *sp);
static void moea64_sp_demote(struct pvo_entry *pvo);
static struct pvo_entry *moea64_sp_unwire(struct pvo_entry *sp);
static struct pvo_entry *moea64_sp_protect(struct pvo_entry *sp,
vm_prot_t prot);
static int64_t moea64_sp_query(struct pvo_entry *pvo, uint64_t ptebit);
static int64_t moea64_sp_clear(struct pvo_entry *pvo, vm_page_t m,
uint64_t ptebit);
static __inline bool moea64_sp_pvo_in_range(struct pvo_entry *pvo,
vm_offset_t sva, vm_offset_t eva);
/*
* Get physical address from PVO.
*
* For superpages, the lower bits are not stored on pvo_pte.pa and must be
* obtained from VA.
*/
static __inline vm_paddr_t
moea64_pvo_paddr(struct pvo_entry *pvo)
{
vm_paddr_t pa;
pa = (pvo)->pvo_pte.pa & LPTE_RPGN;
if (PVO_IS_SP(pvo)) {
pa &= ~SP_MASK; /* This is needed to clear LPTE_LP bits. */
pa |= PVO_VADDR(pvo) & SP_MASK;
}
return (pa);
}
#else /* !defined(PPC_SUPERPAGES) */
#define PVO_PADDR(pvo) ((pvo)->pvo_pte.pa & LPTE_RPGN)
#endif
/*
* Kernel MMU interface * Kernel MMU interface
*/ */
void moea64_clear_modify(vm_page_t); void moea64_clear_modify(vm_page_t);
void moea64_copy_page(vm_page_t, vm_page_t); void moea64_copy_page(vm_page_t, vm_page_t);
void moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset, void moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
vm_page_t *mb, vm_offset_t b_offset, int xfersize); vm_page_t *mb, vm_offset_t b_offset, int xfersize);
int moea64_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, int moea64_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t,
u_int flags, int8_t psind); u_int flags, int8_t psind);
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Linesstatic struct pmap_funcs moea64_methods = {
.deactivate = moea64_deactivate, .deactivate = moea64_deactivate,
.page_set_memattr = moea64_page_set_memattr, .page_set_memattr = moea64_page_set_memattr,
.quick_enter_page = moea64_quick_enter_page, .quick_enter_page = moea64_quick_enter_page,
.quick_remove_page = moea64_quick_remove_page, .quick_remove_page = moea64_quick_remove_page,
.page_is_mapped = moea64_page_is_mapped, .page_is_mapped = moea64_page_is_mapped,
#ifdef __powerpc64__ #ifdef __powerpc64__
.page_array_startup = moea64_page_array_startup, .page_array_startup = moea64_page_array_startup,
#endif #endif
#ifdef PPC_SUPERPAGES
.ps_enabled = moea64_ps_enabled,
.align_superpage = moea64_align_superpage,
#endif
/* Internal interfaces */ /* Internal interfaces */
.mapdev = moea64_mapdev, .mapdev = moea64_mapdev,
.mapdev_attr = moea64_mapdev_attr, .mapdev_attr = moea64_mapdev_attr,
.unmapdev = moea64_unmapdev, .unmapdev = moea64_unmapdev,
.kextract = moea64_kextract, .kextract = moea64_kextract,
.kenter = moea64_kenter, .kenter = moea64_kenter,
.kenter_attr = moea64_kenter_attr, .kenter_attr = moea64_kenter_attr,
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Linesinit_pvo_entry(struct pvo_entry *pvo, pmap_t pmap, vm_offset_t va)
pvo->pvo_pmap = pmap; pvo->pvo_pmap = pmap;
va &= ~ADDR_POFF; va &= ~ADDR_POFF;
pvo->pvo_vaddr |= va; pvo->pvo_vaddr |= va;
vsid = va_to_vsid(pmap, va); vsid = va_to_vsid(pmap, va);
pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT) pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT)
| (vsid << 16); | (vsid << 16);
shift = (pvo->pvo_vaddr & PVO_LARGE) ? moea64_large_page_shift : if (pmap == kernel_pmap && (pvo->pvo_vaddr & PVO_LARGE) != 0)
ADDR_PIDX_SHFT; shift = moea64_large_page_shift;
else
shift = ADDR_PIDX_SHFT;
hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)va & ADDR_PIDX) >> shift); hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)va & ADDR_PIDX) >> shift);
pvo->pvo_pte.slot = (hash & moea64_pteg_mask) << 3; pvo->pvo_pte.slot = (hash & moea64_pteg_mask) << 3;
} }
static void static void
free_pvo_entry(struct pvo_entry *pvo) free_pvo_entry(struct pvo_entry *pvo)
{ {
▲ Show 20 Lines • Show All 327 Lines • ▼ Show 20 Lines
void void
moea64_early_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) moea64_early_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
{ {
int i, j; int i, j;
vm_size_t physsz, hwphyssz; vm_size_t physsz, hwphyssz;
vm_paddr_t kernelphysstart, kernelphysend; vm_paddr_t kernelphysstart, kernelphysend;
int rm_pavail; int rm_pavail;
/* Level 0 reservations consist of 4096 pages (16MB superpage). */
vm_level_0_order = 12;
#ifndef __powerpc64__ #ifndef __powerpc64__
/* We don't have a direct map since there is no BAT */ /* We don't have a direct map since there is no BAT */
hw_direct_map = 0; hw_direct_map = 0;
/* Make sure battable is zero, since we have no BAT */ /* Make sure battable is zero, since we have no BAT */
for (i = 0; i < 16; i++) { for (i = 0; i < 16; i++) {
battable[i].batu = 0; battable[i].batu = 0;
battable[i].batl = 0; battable[i].batl = 0;
▲ Show 20 Lines • Show All 415 Lines • ▼ Show 20 Linesmoea64_unwire(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
vm_page_t m; vm_page_t m;
int64_t refchg; int64_t refchg;
key.pvo_vaddr = sva; key.pvo_vaddr = sva;
PMAP_LOCK(pm); PMAP_LOCK(pm);
for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
pvo != NULL && PVO_VADDR(pvo) < eva; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
pvo = moea64_sp_unwire(pvo);
continue;
} else {
CTR1(KTR_PMAP, "%s: demote before unwire",
__func__);
moea64_sp_demote(pvo);
}
}
#endif
if ((pvo->pvo_vaddr & PVO_WIRED) == 0) if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
panic("moea64_unwire: pvo %p is missing PVO_WIRED", panic("moea64_unwire: pvo %p is missing PVO_WIRED",
pvo); pvo);
pvo->pvo_vaddr &= ~PVO_WIRED; pvo->pvo_vaddr &= ~PVO_WIRED;
refchg = moea64_pte_replace(pvo, 0 /* No invalidation */); refchg = moea64_pte_replace(pvo, 0 /* No invalidation */);
if ((pvo->pvo_vaddr & PVO_MANAGED) && if ((pvo->pvo_vaddr & PVO_MANAGED) &&
(pvo->pvo_pte.prot & VM_PROT_WRITE)) { (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
if (refchg < 0) if (refchg < 0)
refchg = LPTE_CHG; refchg = LPTE_CHG;
m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pa & LPTE_RPGN); m = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
refchg |= atomic_readandclear_32(&m->md.mdpg_attrs); refchg |= atomic_readandclear_32(&m->md.mdpg_attrs);
if (refchg & LPTE_CHG) if (refchg & LPTE_CHG)
vm_page_dirty(m); vm_page_dirty(m);
if (refchg & LPTE_REF) if (refchg & LPTE_REF)
vm_page_aflag_set(m, PGA_REFERENCED); vm_page_aflag_set(m, PGA_REFERENCED);
} }
pm->pm_stats.wired_count--; pm->pm_stats.wired_count--;
▲ Show 20 Lines • Show All 218 Lines • ▼ Show 20 Lines
int int
moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, u_int flags, int8_t psind) vm_prot_t prot, u_int flags, int8_t psind)
{ {
struct pvo_entry *pvo, *oldpvo; struct pvo_entry *pvo, *oldpvo;
struct pvo_head *pvo_head; struct pvo_head *pvo_head;
uint64_t pte_lo; uint64_t pte_lo;
int error; int error;
#ifdef PPC_SUPERPAGES
struct pvo_entry *tpvo;
#endif
if ((m->oflags & VPO_UNMANAGED) == 0) { if ((m->oflags & VPO_UNMANAGED) == 0) {
if ((flags & PMAP_ENTER_QUICK_LOCKED) == 0) if ((flags & PMAP_ENTER_QUICK_LOCKED) == 0)
VM_PAGE_OBJECT_BUSY_ASSERT(m); VM_PAGE_OBJECT_BUSY_ASSERT(m);
else else
VM_OBJECT_ASSERT_LOCKED(m->object); VM_OBJECT_ASSERT_LOCKED(m->object);
} }
#ifdef PPC_SUPERPAGES
if (psind > 0)
return (moea64_sp_enter(pmap, va, m, prot, flags, psind));
#endif
pvo = alloc_pvo_entry(0); pvo = alloc_pvo_entry(0);
if (pvo == NULL) if (pvo == NULL)
return (KERN_RESOURCE_SHORTAGE); return (KERN_RESOURCE_SHORTAGE);
pvo->pvo_pmap = NULL; /* to be filled in later */ pvo->pvo_pmap = NULL; /* to be filled in later */
pvo->pvo_pte.prot = prot; pvo->pvo_pte.prot = prot;
pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m)); pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m));
pvo->pvo_pte.pa = VM_PAGE_TO_PHYS(m) | pte_lo; pvo->pvo_pte.pa = VM_PAGE_TO_PHYS(m) | pte_lo;
if ((flags & PMAP_ENTER_WIRED) != 0) if ((flags & PMAP_ENTER_WIRED) != 0)
pvo->pvo_vaddr |= PVO_WIRED; pvo->pvo_vaddr |= PVO_WIRED;
if ((m->oflags & VPO_UNMANAGED) != 0 || !moea64_initialized) { if ((m->oflags & VPO_UNMANAGED) != 0 || !moea64_initialized) {
pvo_head = NULL; pvo_head = NULL;
} else { } else {
pvo_head = &m->md.mdpg_pvoh; pvo_head = &m->md.mdpg_pvoh;
pvo->pvo_vaddr |= PVO_MANAGED; pvo->pvo_vaddr |= PVO_MANAGED;
} }
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
if (pvo->pvo_pmap == NULL) if (pvo->pvo_pmap == NULL)
init_pvo_entry(pvo, pmap, va); init_pvo_entry(pvo, pmap, va);
#ifdef PPC_SUPERPAGES
tpvo = moea64_pvo_find_va(pmap, va & ~SP_MASK);
if (tpvo && PVO_IS_SP(tpvo)) {
/* Demote SP before entering a regular page */
CTR2(KTR_PMAP, "%s: demote before enter: va=%#jx",
__func__, (uintmax_t)va);
moea64_sp_demote_aligned(tpvo);
}
#endif
if (prot & VM_PROT_WRITE) if (prot & VM_PROT_WRITE)
if (pmap_bootstrapped && if (pmap_bootstrapped &&
(m->oflags & VPO_UNMANAGED) == 0) (m->oflags & VPO_UNMANAGED) == 0)
vm_page_aflag_set(m, PGA_WRITEABLE); vm_page_aflag_set(m, PGA_WRITEABLE);
error = moea64_pvo_enter(pvo, pvo_head, &oldpvo); error = moea64_pvo_enter(pvo, pvo_head, &oldpvo);
if (error == EEXIST) { if (error == EEXIST) {
if (oldpvo->pvo_vaddr == pvo->pvo_vaddr && if (oldpvo->pvo_vaddr == pvo->pvo_vaddr &&
oldpvo->pvo_pte.pa == pvo->pvo_pte.pa && oldpvo->pvo_pte.pa == pvo->pvo_pte.pa &&
oldpvo->pvo_pte.prot == prot) { oldpvo->pvo_pte.prot == prot) {
/* Identical mapping already exists */ /* Identical mapping already exists */
error = 0; error = 0;
/* If not in page table, reinsert it */ /* If not in page table, reinsert it */
if (moea64_pte_synch(oldpvo) < 0) { if (moea64_pte_synch(oldpvo) < 0) {
STAT_MOEA64(moea64_pte_overflow--); STAT_MOEA64(moea64_pte_overflow--);
moea64_pte_insert(oldpvo); moea64_pte_insert(oldpvo);
} }
/* Then just clean up and go home */ /* Then just clean up and go home */
PV_PAGE_UNLOCK(m); PV_PAGE_UNLOCK(m);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
free_pvo_entry(pvo); free_pvo_entry(pvo);
pvo = NULL;
goto out; goto out;
} else { } else {
/* Otherwise, need to kill it first */ /* Otherwise, need to kill it first */
KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old " KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old "
"mapping does not match new mapping")); "mapping does not match new mapping"));
moea64_pvo_remove_from_pmap(oldpvo); moea64_pvo_remove_from_pmap(oldpvo);
moea64_pvo_enter(pvo, pvo_head, NULL); moea64_pvo_enter(pvo, pvo_head, NULL);
} }
Show All 12 Linesout:
* Flush the page from the instruction cache if this page is * Flush the page from the instruction cache if this page is
* mapped executable and cacheable. * mapped executable and cacheable.
*/ */
if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 && if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 &&
(pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { (pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
vm_page_aflag_set(m, PGA_EXECUTABLE); vm_page_aflag_set(m, PGA_EXECUTABLE);
moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE); moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE);
} }
#ifdef PPC_SUPERPAGES
/* Try to promote pages */
if (pmap != kernel_pmap && pvo != NULL &&
(pvo->pvo_vaddr & PVO_MANAGED) != 0)
moea64_sp_promote(pmap, va, m);
#endif
return (KERN_SUCCESS); return (KERN_SUCCESS);
markjUnsubmitted
Not Done
Inline Actions

Why do we not promote superpages in the kernel map? Most kernel mappings are unmanaged, so the check below will catch them, but exec_map and pipe_map create managed mappings that can in principle be promoted.

markj: Why do we not promote superpages in the kernel map? Most kernel mappings are unmanaged, so the…
luporlAuthorUnsubmitted
Done
Inline Actions

I was afraid that using superpages in kernel map could make things more complex and cause some breaks and also wasn't sure about the benefits.
But I can give it a try now and see how things go.

luporl: I was afraid that using superpages in kernel map could make things more complex and cause some…
luporlAuthorUnsubmitted
Done
Inline Actions

As this can have a big impact in performance and stability, I'll try it after improving current changes' performance.

luporl: As this can have a big impact in performance and stability, I'll try it after improving current…
} }
static void static void
moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
vm_size_t sz) vm_size_t sz)
{ {
/* /*
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines
* corresponding offset from m_start are mapped. * corresponding offset from m_start are mapped.
*/ */
void void
moea64_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, moea64_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end,
vm_page_t m_start, vm_prot_t prot) vm_page_t m_start, vm_prot_t prot)
{ {
vm_page_t m; vm_page_t m;
vm_pindex_t diff, psize; vm_pindex_t diff, psize;
vm_offset_t va;
int8_t psind;
VM_OBJECT_ASSERT_LOCKED(m_start->object); VM_OBJECT_ASSERT_LOCKED(m_start->object);
psize = atop(end - start); psize = atop(end - start);
m = m_start; m = m_start;
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
moea64_enter(pm, start + ptoa(diff), m, prot & va = start + ptoa(diff);
(VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP | #ifdef PPC_SUPERPAGES
PMAP_ENTER_QUICK_LOCKED, 0); if ((va & SP_MASK) == 0 && va + SP_SIZE <= end &&
m->psind == 1 && moea64_ps_enabled(pm))
psind = 1;
else
#endif
psind = 0;
moea64_enter(pm, va, m, prot &
(VM_PROT_READ | VM_PROT_EXECUTE),
PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, psind);
#ifdef PPC_SUPERPAGES
if (psind == 1)
m = &m[SP_SIZE / PAGE_SIZE - 1];
#endif
m = TAILQ_NEXT(m, listq); m = TAILQ_NEXT(m, listq);
} }
} }
void void
moea64_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, moea64_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m,
vm_prot_t prot) vm_prot_t prot)
{ {
moea64_enter(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), moea64_enter(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE),
PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, 0); PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, 0);
} }
vm_paddr_t vm_paddr_t
moea64_extract(pmap_t pm, vm_offset_t va) moea64_extract(pmap_t pm, vm_offset_t va)
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
vm_paddr_t pa; vm_paddr_t pa;
PMAP_LOCK(pm); PMAP_LOCK(pm);
pvo = moea64_pvo_find_va(pm, va); pvo = moea64_pvo_find_va(pm, va);
if (pvo == NULL) if (pvo == NULL)
pa = 0; pa = 0;
else else
pa = (pvo->pvo_pte.pa & LPTE_RPGN) | (va - PVO_VADDR(pvo)); pa = PVO_PADDR(pvo) | (va - PVO_VADDR(pvo));
PMAP_UNLOCK(pm); PMAP_UNLOCK(pm);
return (pa); return (pa);
} }
/* /*
* Atomically extract and hold the physical page with the given * Atomically extract and hold the physical page with the given
* pmap and virtual address pair if that mapping permits the given * pmap and virtual address pair if that mapping permits the given
* protection. * protection.
*/ */
vm_page_t vm_page_t
moea64_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) moea64_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
vm_page_t m; vm_page_t m;
m = NULL; m = NULL;
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF); pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);
if (pvo != NULL && (pvo->pvo_pte.prot & prot) == prot) { if (pvo != NULL && (pvo->pvo_pte.prot & prot) == prot) {
m = PHYS_TO_VM_PAGE(pvo->pvo_pte.pa & LPTE_RPGN); m = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
if (!vm_page_wire_mapped(m)) if (!vm_page_wire_mapped(m))
m = NULL; m = NULL;
} }
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
return (m); return (m);
} }
static void * static void *
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
bzero((void *)va, PAGE_SIZE); bzero((void *)va, PAGE_SIZE);
return (void *)va; return (void *)va;
} }
extern int elf32_nxstack; extern int elf32_nxstack;
void void
moea64_init() moea64_init(void)
{ {
CTR0(KTR_PMAP, "moea64_init"); CTR0(KTR_PMAP, "moea64_init");
moea64_pvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), moea64_pvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
UMA_ZONE_VM | UMA_ZONE_NOFREE); UMA_ZONE_VM | UMA_ZONE_NOFREE);
#ifdef PPC_SUPERPAGES
/*
* Are large page mappings enabled?
*/
TUNABLE_INT_FETCH("vm.pmap.sp_enabled", &sp_enabled);
if (sp_enabled) {
KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
("moea64_init: can't assign to pagesizes[1]"));
if (!hw_direct_map) { if (!hw_direct_map) {
printf("mmu_oea64: HW does not support direct map. "
"Disabling superpages...\n");
sp_enabled = 0;
jhibbitsUnsubmitted
Done
Inline Actions

Is a DMAP really a prerequisite for superpages? Or do they just share the same prerequisites?

jhibbits: Is a DMAP really a prerequisite for superpages? Or do they just share the same prerequisites?
luporlAuthorUnsubmitted
Done
Inline Actions

I guess they share the same prerequisite: large pages availability.
What if I change !hw_direct_map to moea64_large_page_size == 0?

luporl: I guess they share the same prerequisite: large pages availability. What if I change `!
luporlAuthorUnsubmitted
Done
Inline Actions

Changed to moea64_large_page_size == 0.

luporl: Changed to `moea64_large_page_size == 0`.
} else if (!moea64_has_lp_4k_16m) {
printf("mmu_oea64: "
"HW does not support mixed 4KB/16MB page sizes. "
"Disabling superpages...\n");
sp_enabled = 0;
} else
pagesizes[1] = SP_SIZE;
}
#endif
if (!hw_direct_map) {
uma_zone_set_allocf(moea64_pvo_zone, moea64_uma_page_alloc); uma_zone_set_allocf(moea64_pvo_zone, moea64_uma_page_alloc);
} }
#ifdef COMPAT_FREEBSD32 #ifdef COMPAT_FREEBSD32
elf32_nxstack = 1; elf32_nxstack = 1;
#endif #endif
moea64_initialized = TRUE; moea64_initialized = TRUE;
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Linesmoea64_remove_write(vm_page_t m)
int64_t refchg, ret; int64_t refchg, ret;
pmap_t pmap; pmap_t pmap;
KASSERT((m->oflags & VPO_UNMANAGED) == 0, KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("moea64_remove_write: page %p is not managed", m)); ("moea64_remove_write: page %p is not managed", m));
vm_page_assert_busied(m); vm_page_assert_busied(m);
if (!pmap_page_is_write_mapped(m)) if (!pmap_page_is_write_mapped(m))
return return;
powerpc_sync(); powerpc_sync();
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
refchg = 0; refchg = 0;
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
pmap = pvo->pvo_pmap; pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
if (!(pvo->pvo_vaddr & PVO_DEAD) && if (!(pvo->pvo_vaddr & PVO_DEAD) &&
(pvo->pvo_pte.prot & VM_PROT_WRITE)) { (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
CTR1(KTR_PMAP, "%s: demote before remwr",
__func__);
moea64_sp_demote(pvo);
}
#endif
pvo->pvo_pte.prot &= ~VM_PROT_WRITE; pvo->pvo_pte.prot &= ~VM_PROT_WRITE;
ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE); ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
if (ret < 0) if (ret < 0)
ret = LPTE_CHG; ret = LPTE_CHG;
refchg |= ret; refchg |= ret;
if (pvo->pvo_pmap == kernel_pmap) if (pvo->pvo_pmap == kernel_pmap)
isync(); isync();
} }
Show All 32 Lines
void void
moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma) moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma)
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
int64_t refchg; int64_t refchg;
pmap_t pmap; pmap_t pmap;
uint64_t lo; uint64_t lo;
CTR3(KTR_PMAP, "%s: pa=%#jx, ma=%#x",
__func__, (uintmax_t)VM_PAGE_TO_PHYS(m), ma);
if ((m->oflags & VPO_UNMANAGED) != 0) { if ((m->oflags & VPO_UNMANAGED) != 0) {
m->md.mdpg_cache_attrs = ma; m->md.mdpg_cache_attrs = ma;
return; return;
} }
lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), ma); lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), ma);
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
pmap = pvo->pvo_pmap; pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
if (!(pvo->pvo_vaddr & PVO_DEAD)) { if (!(pvo->pvo_vaddr & PVO_DEAD)) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
CTR1(KTR_PMAP,
"%s: demote before set_memattr", __func__);
moea64_sp_demote(pvo);
}
#endif
pvo->pvo_pte.pa &= ~LPTE_WIMG; pvo->pvo_pte.pa &= ~LPTE_WIMG;
pvo->pvo_pte.pa |= lo; pvo->pvo_pte.pa |= lo;
refchg = moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE); refchg = moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
if (refchg < 0) if (refchg < 0)
refchg = (pvo->pvo_pte.prot & VM_PROT_WRITE) ? refchg = (pvo->pvo_pte.prot & VM_PROT_WRITE) ?
LPTE_CHG : 0; LPTE_CHG : 0;
if ((pvo->pvo_vaddr & PVO_MANAGED) && if ((pvo->pvo_vaddr & PVO_MANAGED) &&
(pvo->pvo_pte.prot & VM_PROT_WRITE)) { (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Linesmoea64_kextract(vm_offset_t va)
*/ */
if (va < VM_MIN_KERNEL_ADDRESS) if (va < VM_MIN_KERNEL_ADDRESS)
return (va & ~DMAP_BASE_ADDRESS); return (va & ~DMAP_BASE_ADDRESS);
PMAP_LOCK(kernel_pmap); PMAP_LOCK(kernel_pmap);
pvo = moea64_pvo_find_va(kernel_pmap, va); pvo = moea64_pvo_find_va(kernel_pmap, va);
KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR, KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR,
va)); va));
pa = (pvo->pvo_pte.pa & LPTE_RPGN) | (va - PVO_VADDR(pvo)); pa = PVO_PADDR(pvo) | (va - PVO_VADDR(pvo));
PMAP_UNLOCK(kernel_pmap); PMAP_UNLOCK(kernel_pmap);
return (pa); return (pa);
} }
/* /*
* Remove a wired page from kernel virtual address space. * Remove a wired page from kernel virtual address space.
*/ */
void void
▲ Show 20 Lines • Show All 309 Lines • ▼ Show 20 Linesmoea64_pvo_protect( pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot)
PMAP_LOCK_ASSERT(pm, MA_OWNED); PMAP_LOCK_ASSERT(pm, MA_OWNED);
/* /*
* Change the protection of the page. * Change the protection of the page.
*/ */
oldprot = pvo->pvo_pte.prot; oldprot = pvo->pvo_pte.prot;
pvo->pvo_pte.prot = prot; pvo->pvo_pte.prot = prot;
pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pa & LPTE_RPGN); pg = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
/* /*
* If the PVO is in the page table, update mapping * If the PVO is in the page table, update mapping
*/ */
refchg = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE); refchg = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
if (refchg < 0) if (refchg < 0)
refchg = (oldprot & VM_PROT_WRITE) ? LPTE_CHG : 0; refchg = (oldprot & VM_PROT_WRITE) ? LPTE_CHG : 0;
if (pm != kernel_pmap && pg != NULL && if (pm != kernel_pmap && pg != NULL &&
(pg->a.flags & PGA_EXECUTABLE) == 0 && (pg->a.flags & PGA_EXECUTABLE) == 0 &&
(pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { (pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
if ((pg->oflags & VPO_UNMANAGED) == 0) if ((pg->oflags & VPO_UNMANAGED) == 0)
vm_page_aflag_set(pg, PGA_EXECUTABLE); vm_page_aflag_set(pg, PGA_EXECUTABLE);
moea64_syncicache(pm, PVO_VADDR(pvo), moea64_syncicache(pm, PVO_VADDR(pvo),
pvo->pvo_pte.pa & LPTE_RPGN, PAGE_SIZE); PVO_PADDR(pvo), PAGE_SIZE);
} }
/* /*
* Update vm about the REF/CHG bits if the page is managed and we have * Update vm about the REF/CHG bits if the page is managed and we have
* removed write access. * removed write access.
*/ */
if (pg != NULL && (pvo->pvo_vaddr & PVO_MANAGED) && if (pg != NULL && (pvo->pvo_vaddr & PVO_MANAGED) &&
(oldprot & VM_PROT_WRITE)) { (oldprot & VM_PROT_WRITE)) {
refchg |= atomic_readandclear_32(&pg->md.mdpg_attrs); refchg |= atomic_readandclear_32(&pg->md.mdpg_attrs);
if (refchg & LPTE_CHG) if (refchg & LPTE_CHG)
vm_page_dirty(pg); vm_page_dirty(pg);
if (refchg & LPTE_REF) if (refchg & LPTE_REF)
vm_page_aflag_set(pg, PGA_REFERENCED); vm_page_aflag_set(pg, PGA_REFERENCED);
} }
} }
void void
moea64_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, moea64_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva,
vm_prot_t prot) vm_prot_t prot)
{ {
struct pvo_entry *pvo, *tpvo, key; struct pvo_entry *pvo, key;
CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm,
sva, eva, prot); sva, eva, prot);
KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap,
("moea64_protect: non current pmap")); ("moea64_protect: non current pmap"));
if ((prot & VM_PROT_READ) == VM_PROT_NONE) { if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
moea64_remove(pm, sva, eva); moea64_remove(pm, sva, eva);
return; return;
} }
PMAP_LOCK(pm); PMAP_LOCK(pm);
key.pvo_vaddr = sva; key.pvo_vaddr = sva;
for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { pvo != NULL && PVO_VADDR(pvo) < eva;
tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
pvo = moea64_sp_protect(pvo, prot);
continue;
} else {
CTR1(KTR_PMAP, "%s: demote before protect",
__func__);
moea64_sp_demote(pvo);
}
}
#endif
moea64_pvo_protect(pm, pvo, prot); moea64_pvo_protect(pm, pvo, prot);
} }
PMAP_UNLOCK(pm); PMAP_UNLOCK(pm);
} }
/* /*
* Map a list of wired pages into kernel virtual address space. This is * Map a list of wired pages into kernel virtual address space. This is
* intended for temporary mappings which do not need page modification or * intended for temporary mappings which do not need page modification or
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Linesmoea64_remove_pages(pmap_t pm)
while (!SLIST_EMPTY(&tofree)) { while (!SLIST_EMPTY(&tofree)) {
pvo = SLIST_FIRST(&tofree); pvo = SLIST_FIRST(&tofree);
SLIST_REMOVE_HEAD(&tofree, pvo_dlink); SLIST_REMOVE_HEAD(&tofree, pvo_dlink);
moea64_pvo_remove_from_page(pvo); moea64_pvo_remove_from_page(pvo);
free_pvo_entry(pvo); free_pvo_entry(pvo);
} }
} }
/* static void
* Remove the given range of addresses from the specified map. moea64_remove_locked(pmap_t pm, vm_offset_t sva, vm_offset_t eva,
*/ struct pvo_dlist *tofree)
void
moea64_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{ {
struct pvo_entry *pvo, *tpvo, key; struct pvo_entry *pvo, *tpvo, key;
struct pvo_dlist tofree;
/* PMAP_LOCK_ASSERT(pm, MA_OWNED);
* Perform an unsynchronized read. This is, however, safe.
*/
if (pm->pm_stats.resident_count == 0)
return;
key.pvo_vaddr = sva; key.pvo_vaddr = sva;
SLIST_INIT(&tofree);
PMAP_LOCK(pm);
for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
tpvo = moea64_sp_remove(pvo, tofree);
continue;
} else {
CTR1(KTR_PMAP, "%s: demote before remove",
__func__);
moea64_sp_demote(pvo);
}
}
#endif
tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo);
/* /*
* For locking reasons, remove this from the page table and * For locking reasons, remove this from the page table and
* pmap, but save delinking from the vm_page for a second * pmap, but save delinking from the vm_page for a second
* pass * pass
*/ */
moea64_pvo_remove_from_pmap(pvo); moea64_pvo_remove_from_pmap(pvo);
SLIST_INSERT_HEAD(&tofree, pvo, pvo_dlink); SLIST_INSERT_HEAD(tofree, pvo, pvo_dlink);
} }
}
/*
* Remove the given range of addresses from the specified map.
*/
void
moea64_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{
struct pvo_entry *pvo;
struct pvo_dlist tofree;
/*
* Perform an unsynchronized read. This is, however, safe.
*/
if (pm->pm_stats.resident_count == 0)
return;
SLIST_INIT(&tofree);
PMAP_LOCK(pm);
moea64_remove_locked(pm, sva, eva, &tofree);
PMAP_UNLOCK(pm); PMAP_UNLOCK(pm);
while (!SLIST_EMPTY(&tofree)) { while (!SLIST_EMPTY(&tofree)) {
pvo = SLIST_FIRST(&tofree); pvo = SLIST_FIRST(&tofree);
SLIST_REMOVE_HEAD(&tofree, pvo_dlink); SLIST_REMOVE_HEAD(&tofree, pvo_dlink);
moea64_pvo_remove_from_page(pvo); moea64_pvo_remove_from_page(pvo);
free_pvo_entry(pvo); free_pvo_entry(pvo);
} }
Show All 13 Linesmoea64_remove_all(vm_page_t m)
LIST_INIT(&freequeue); LIST_INIT(&freequeue);
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) { LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) {
pmap = pvo->pvo_pmap; pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
wasdead = (pvo->pvo_vaddr & PVO_DEAD); wasdead = (pvo->pvo_vaddr & PVO_DEAD);
if (!wasdead) if (!wasdead) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
CTR1(KTR_PMAP, "%s: demote before remove_all",
__func__);
moea64_sp_demote(pvo);
}
#endif
moea64_pvo_remove_from_pmap(pvo); moea64_pvo_remove_from_pmap(pvo);
}
moea64_pvo_remove_from_page_locked(pvo, m); moea64_pvo_remove_from_page_locked(pvo, m);
if (!wasdead) if (!wasdead)
LIST_INSERT_HEAD(&freequeue, pvo, pvo_vlink); LIST_INSERT_HEAD(&freequeue, pvo, pvo_vlink);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
} }
KASSERT(!pmap_page_is_mapped(m), ("Page still has mappings")); KASSERT(!pmap_page_is_mapped(m), ("Page still has mappings"));
KASSERT((m->a.flags & PGA_WRITEABLE) == 0, ("Page still writable")); KASSERT((m->a.flags & PGA_WRITEABLE) == 0, ("Page still writable"));
▲ Show 20 Lines • Show All 146 Lines • ▼ Show 20 Linesmoea64_pvo_remove_from_pmap(struct pvo_entry *pvo)
/* /*
* Mark this for the next sweep * Mark this for the next sweep
*/ */
pvo->pvo_vaddr |= PVO_DEAD; pvo->pvo_vaddr |= PVO_DEAD;
/* Send RC bits to VM */ /* Send RC bits to VM */
if ((pvo->pvo_vaddr & PVO_MANAGED) && if ((pvo->pvo_vaddr & PVO_MANAGED) &&
(pvo->pvo_pte.prot & VM_PROT_WRITE)) { (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pa & LPTE_RPGN); pg = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
if (pg != NULL) { if (pg != NULL) {
refchg |= atomic_readandclear_32(&pg->md.mdpg_attrs); refchg |= atomic_readandclear_32(&pg->md.mdpg_attrs);
if (refchg & LPTE_CHG) if (refchg & LPTE_CHG)
vm_page_dirty(pg); vm_page_dirty(pg);
if (refchg & LPTE_REF) if (refchg & LPTE_REF)
vm_page_aflag_set(pg, PGA_REFERENCED); vm_page_aflag_set(pg, PGA_REFERENCED);
} }
} }
Show All 9 Linesmoea64_pvo_remove_from_page_locked(struct pvo_entry *pvo,
/* Use NULL pmaps as a sentinel for races in page deletion */ /* Use NULL pmaps as a sentinel for races in page deletion */
if (pvo->pvo_pmap == NULL) if (pvo->pvo_pmap == NULL)
return; return;
pvo->pvo_pmap = NULL; pvo->pvo_pmap = NULL;
/* /*
* Update vm about page writeability/executability if managed * Update vm about page writeability/executability if managed
*/ */
PV_LOCKASSERT(pvo->pvo_pte.pa & LPTE_RPGN); PV_LOCKASSERT(PVO_PADDR(pvo));
if (pvo->pvo_vaddr & PVO_MANAGED) { if (pvo->pvo_vaddr & PVO_MANAGED) {
if (m != NULL) { if (m != NULL) {
LIST_REMOVE(pvo, pvo_vlink); LIST_REMOVE(pvo, pvo_vlink);
if (LIST_EMPTY(vm_page_to_pvoh(m))) if (LIST_EMPTY(vm_page_to_pvoh(m)))
vm_page_aflag_clear(m, vm_page_aflag_clear(m,
PGA_WRITEABLE | PGA_EXECUTABLE); PGA_WRITEABLE | PGA_EXECUTABLE);
} }
} }
STAT_MOEA64(moea64_pvo_entries--); STAT_MOEA64(moea64_pvo_entries--);
STAT_MOEA64(moea64_pvo_remove_calls++); STAT_MOEA64(moea64_pvo_remove_calls++);
} }
static void static void
moea64_pvo_remove_from_page(struct pvo_entry *pvo) moea64_pvo_remove_from_page(struct pvo_entry *pvo)
{ {
vm_page_t pg = NULL; vm_page_t pg = NULL;
vm_paddr_t pa;
pa = PVO_PADDR(pvo);
if (pvo->pvo_vaddr & PVO_MANAGED) if (pvo->pvo_vaddr & PVO_MANAGED)
pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.pa & LPTE_RPGN); pg = PHYS_TO_VM_PAGE(pa);
PV_LOCK(pvo->pvo_pte.pa & LPTE_RPGN); PV_LOCK(pa);
moea64_pvo_remove_from_page_locked(pvo, pg); moea64_pvo_remove_from_page_locked(pvo, pg);
PV_UNLOCK(pvo->pvo_pte.pa & LPTE_RPGN); PV_UNLOCK(pa);
} }
static struct pvo_entry * static struct pvo_entry *
moea64_pvo_find_va(pmap_t pm, vm_offset_t va) moea64_pvo_find_va(pmap_t pm, vm_offset_t va)
{ {
struct pvo_entry key; struct pvo_entry key;
PMAP_LOCK_ASSERT(pm, MA_OWNED); PMAP_LOCK_ASSERT(pm, MA_OWNED);
key.pvo_vaddr = va & ~ADDR_POFF; key.pvo_vaddr = va & ~ADDR_POFF;
return (RB_FIND(pvo_tree, &pm->pmap_pvo, &key)); return (RB_FIND(pvo_tree, &pm->pmap_pvo, &key));
} }
static boolean_t static boolean_t
moea64_query_bit(vm_page_t m, uint64_t ptebit) moea64_query_bit(vm_page_t m, uint64_t ptebit)
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
int64_t ret; int64_t ret;
boolean_t rv; boolean_t rv;
#ifdef PPC_SUPERPAGES
vm_page_t sp;
#endif
/* /*
* See if this bit is stored in the page already. * See if this bit is stored in the page already.
*
* For superpages, the bit is stored in the first vm page.
*/ */
if (m->md.mdpg_attrs & ptebit) if ((m->md.mdpg_attrs & ptebit) != 0
#ifdef PPC_SUPERPAGES
|| ((sp = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(m) & ~SP_MASK)) != NULL &&
(sp->md.mdpg_attrs & (ptebit | MDPG_ATTR_SP)) ==
(ptebit | MDPG_ATTR_SP))
#endif
)
return (TRUE); return (TRUE);
/* /*
* Examine each PTE. Sync so that any pending REF/CHG bits are * Examine each PTE. Sync so that any pending REF/CHG bits are
* flushed to the PTEs. * flushed to the PTEs.
*/ */
rv = FALSE; rv = FALSE;
powerpc_sync(); powerpc_sync();
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
ret = moea64_sp_query(pvo, ptebit);
/*
* If SP was not demoted, check its REF/CHG bits here.
*/
if (ret != -1) {
if ((ret & ptebit) != 0) {
rv = TRUE;
break;
}
continue;
}
/* else, fallthrough */
}
#endif
ret = 0; ret = 0;
/* /*
* See if this pvo has a valid PTE. if so, fetch the * See if this pvo has a valid PTE. if so, fetch the
* REF/CHG bits from the valid PTE. If the appropriate * REF/CHG bits from the valid PTE. If the appropriate
* ptebit is set, return success. * ptebit is set, return success.
*/ */
PMAP_LOCK(pvo->pvo_pmap); PMAP_LOCK(pvo->pvo_pmap);
Show All 29 Linesmoea64_clear_bit(vm_page_t m, u_int64_t ptebit)
powerpc_sync(); powerpc_sync();
/* /*
* For each pvo entry, clear the pte's ptebit. * For each pvo entry, clear the pte's ptebit.
*/ */
count = 0; count = 0;
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
#ifdef PPC_SUPERPAGES
if (PVO_IS_SP(pvo)) {
if ((ret = moea64_sp_clear(pvo, m, ptebit)) != -1) {
count += ret;
continue;
}
}
#endif
ret = 0; ret = 0;
PMAP_LOCK(pvo->pvo_pmap); PMAP_LOCK(pvo->pvo_pmap);
if (!(pvo->pvo_vaddr & PVO_DEAD)) if (!(pvo->pvo_vaddr & PVO_DEAD))
ret = moea64_pte_clear(pvo, ptebit); ret = moea64_pte_clear(pvo, ptebit);
PMAP_UNLOCK(pvo->pvo_pmap); PMAP_UNLOCK(pvo->pvo_pmap);
if (ret > 0 && (ret & ptebit)) if (ret > 0 && (ret & ptebit))
Show All 16 Lines if (hw_direct_map && mem_valid(pa, size) == 0)
return (0); return (0);
PMAP_LOCK(kernel_pmap); PMAP_LOCK(kernel_pmap);
ppa = pa & ~ADDR_POFF; ppa = pa & ~ADDR_POFF;
key.pvo_vaddr = DMAP_BASE_ADDRESS + ppa; key.pvo_vaddr = DMAP_BASE_ADDRESS + ppa;
for (pvo = RB_FIND(pvo_tree, &kernel_pmap->pmap_pvo, &key); for (pvo = RB_FIND(pvo_tree, &kernel_pmap->pmap_pvo, &key);
ppa < pa + size; ppa += PAGE_SIZE, ppa < pa + size; ppa += PAGE_SIZE,
pvo = RB_NEXT(pvo_tree, &kernel_pmap->pmap_pvo, pvo)) { pvo = RB_NEXT(pvo_tree, &kernel_pmap->pmap_pvo, pvo)) {
if (pvo == NULL || (pvo->pvo_pte.pa & LPTE_RPGN) != ppa) { if (pvo == NULL || PVO_PADDR(pvo) != ppa) {
error = EFAULT; error = EFAULT;
break; break;
} }
} }
PMAP_UNLOCK(kernel_pmap); PMAP_UNLOCK(kernel_pmap);
return (error); return (error);
} }
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines if (__predict_false(pm == NULL))
pm = &curthread->td_proc->p_vmspace->vm_pmap; pm = &curthread->td_proc->p_vmspace->vm_pmap;
PMAP_LOCK(pm); PMAP_LOCK(pm);
while (sz > 0) { while (sz > 0) {
lim = round_page(va+1); lim = round_page(va+1);
len = MIN(lim - va, sz); len = MIN(lim - va, sz);
pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF); pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF);
if (pvo != NULL && !(pvo->pvo_pte.pa & LPTE_I)) { if (pvo != NULL && !(pvo->pvo_pte.pa & LPTE_I)) {
pa = (pvo->pvo_pte.pa & LPTE_RPGN) | (va & ADDR_POFF); pa = PVO_PADDR(pvo) | (va & ADDR_POFF);
moea64_syncicache(pm, va, pa, len); moea64_syncicache(pm, va, pa, len);
} }
va += len; va += len;
sz -= len; sz -= len;
} }
PMAP_UNLOCK(pm); PMAP_UNLOCK(pm);
} }
void void
moea64_dumpsys_map(vm_paddr_t pa, size_t sz, void **va) moea64_dumpsys_map(vm_paddr_t pa, size_t sz, void **va)
{ {
*va = (void *)(uintptr_t)pa; *va = (void *)(uintptr_t)pa;
} }
extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1]; extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1];
void void
moea64_scan_init() moea64_scan_init(void)
jhibbitsUnsubmitted
Done
Inline Actions

I think adding 'void' is only necessary for prototypes, not for definitions.

jhibbits: I think adding 'void' is only necessary for prototypes, not for definitions.
luporlAuthorUnsubmitted
Done
Inline Actions

Right, I always add 'void' to functions with no arguments, but it's not really necessary in definitions, so I'll remove it.

luporl: Right, I always add 'void' to functions with no arguments, but it's not really necessary in…
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
vm_offset_t va; vm_offset_t va;
int i; int i;
if (!do_minidump) { if (!do_minidump) {
/* Initialize phys. segments for dumpsys(). */ /* Initialize phys. segments for dumpsys(). */
memset(&dump_map, 0, sizeof(dump_map)); memset(&dump_map, 0, sizeof(dump_map));
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines if (va < virtual_end) {
} }
dump_map[2].pa_size = va - dump_map[2].pa_start; dump_map[2].pa_size = va - dump_map[2].pa_start;
} }
} }
#ifdef __powerpc64__ #ifdef __powerpc64__
static size_t static size_t
moea64_scan_pmap() moea64_scan_pmap(void)
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
vm_paddr_t pa, pa_end; vm_paddr_t pa, pa_end;
vm_offset_t va, pgva, kstart, kend, kstart_lp, kend_lp; vm_offset_t va, pgva, kstart, kend, kstart_lp, kend_lp;
uint64_t lpsize; uint64_t lpsize;
lpsize = moea64_large_page_size; lpsize = moea64_large_page_size;
kstart = trunc_page((vm_offset_t)_etext); kstart = trunc_page((vm_offset_t)_etext);
Show All 20 Lines if (va >= DMAP_BASE_ADDRESS && va <= DMAP_MAX_ADDRESS) {
continue; continue;
} else { } else {
pgva = trunc_page(va); pgva = trunc_page(va);
if (pgva < kstart || pgva >= kend) if (pgva < kstart || pgva >= kend)
continue; continue;
} }
} }
pa = pvo->pvo_pte.pa & LPTE_RPGN; pa = PVO_PADDR(pvo);
if (va & PVO_LARGE) { if (va & PVO_LARGE) {
pa_end = pa + lpsize; pa_end = pa + lpsize;
for (; pa < pa_end; pa += PAGE_SIZE) { for (; pa < pa_end; pa += PAGE_SIZE) {
if (is_dumpable(pa)) if (is_dumpable(pa))
dump_add_page(pa); dump_add_page(pa);
} }
} else { } else {
Show All 15 Linesmoea64_dump_pmap_init(unsigned blkpgs)
dump_ctx.ptex_end = moea64_pteg_count * 8; dump_ctx.ptex_end = moea64_pteg_count * 8;
dump_ctx.blksz = blkpgs * PAGE_SIZE; dump_ctx.blksz = blkpgs * PAGE_SIZE;
return (&dump_ctx); return (&dump_ctx);
} }
#else #else
static size_t static size_t
moea64_scan_pmap() moea64_scan_pmap(void)
{ {
return (0); return (0);
} }
static void * static void *
moea64_dump_pmap_init(unsigned blkpgs) moea64_dump_pmap_init(unsigned blkpgs)
{ {
return (NULL); return (NULL);
▲ Show 20 Lines • Show All 126 Lines • ▼ Show 20 Lines
DEFINE_OEA64_IFUNC(int64_t, pte_replace, (struct pvo_entry *, int), DEFINE_OEA64_IFUNC(int64_t, pte_replace, (struct pvo_entry *, int),
moea64_pte_replace_default) moea64_pte_replace_default)
DEFINE_OEA64_IFUNC(int64_t, pte_insert, (struct pvo_entry *), moea64_null_method) DEFINE_OEA64_IFUNC(int64_t, pte_insert, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_unset, (struct pvo_entry *), moea64_null_method) DEFINE_OEA64_IFUNC(int64_t, pte_unset, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_clear, (struct pvo_entry *, uint64_t), DEFINE_OEA64_IFUNC(int64_t, pte_clear, (struct pvo_entry *, uint64_t),
moea64_null_method) moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_synch, (struct pvo_entry *), moea64_null_method) DEFINE_OEA64_IFUNC(int64_t, pte_synch, (struct pvo_entry *), moea64_null_method)
#ifdef PPC_SUPERPAGES
jhibbitsUnsubmitted
Done
Inline Actions

Unless there's a statistically significant performance penalty associated with it, this can be unconditional.

jhibbits: Unless there's a statistically significant performance penalty associated with it, this can be…
luporlAuthorUnsubmitted
Done
Inline Actions

Right, buildworld times showed a 1% difference only, when sp_enabled=0, that doesn't seem statistically significant.

luporl: Right, buildworld times showed a 1% difference only, when sp_enabled=0, that doesn't seem…
/* MMU interface */
static bool
moea64_ps_enabled(pmap_t pmap)
{
return (sp_enabled);
}
static void
moea64_align_superpage(vm_object_t object, vm_ooffset_t offset,
vm_offset_t *addr, vm_size_t size)
{
vm_offset_t sp_offset;
if (size < SP_SIZE)
return;
CTR4(KTR_PMAP, "%s: offs=%#jx, addr=%p, size=%#jx",
__func__, (uintmax_t)offset, addr, (uintmax_t)size);
if (object != NULL && (object->flags & OBJ_COLORED) != 0)
offset += ptoa(object->pg_color);
sp_offset = offset & SP_MASK;
if (size - ((SP_SIZE - sp_offset) & SP_MASK) < SP_SIZE ||
(*addr & SP_MASK) == sp_offset)
return;
if ((*addr & SP_MASK) < sp_offset)
*addr = (*addr & ~SP_MASK) + sp_offset;
else
*addr = ((*addr + SP_MASK) & ~SP_MASK) + sp_offset;
}
/* helpers */
static __inline void
moea64_pvo_cleanup(struct pvo_dlist *tofree)
{
struct pvo_entry *pvo;
/* clean up */
while (!SLIST_EMPTY(tofree)) {
pvo = SLIST_FIRST(tofree);
SLIST_REMOVE_HEAD(tofree, pvo_dlink);
if (pvo->pvo_vaddr & PVO_DEAD)
moea64_pvo_remove_from_page(pvo);
free_pvo_entry(pvo);
}
}
static __inline uint16_t
pvo_to_vmpage_flags(struct pvo_entry *pvo)
{
uint16_t flags;
flags = 0;
if ((pvo->pvo_pte.prot & VM_PROT_WRITE) != 0)
flags |= PGA_WRITEABLE;
if ((pvo->pvo_pte.prot & VM_PROT_EXECUTE) != 0)
flags |= PGA_EXECUTABLE;
return (flags);
}
/*
* Check if the given pvo and its superpage are in sva-eva range.
*/
static __inline bool
moea64_sp_pvo_in_range(struct pvo_entry *pvo, vm_offset_t sva, vm_offset_t eva)
{
vm_offset_t spva;
spva = PVO_VADDR(pvo) & ~SP_MASK;
if (spva >= sva && spva + SP_SIZE <= eva) {
/*
* Because this function is intended to be called from loops
* that iterate over ordered pvo entries, if the condition
* above is true then the pvo must be the first of its
* superpage.
*/
KASSERT(PVO_VADDR(pvo) == spva,
("%s: unexpected unaligned superpage pvo", __func__));
return (true);
}
return (false);
}
/*
* Update vm about the REF/CHG bits if the superpage is managed and
* has (or had) write access.
*/
static void
moea64_sp_refchg_process(struct pvo_entry *sp, vm_page_t m,
int64_t sp_refchg, vm_prot_t prot)
{
vm_page_t m_end;
int64_t refchg;
if ((sp->pvo_vaddr & PVO_MANAGED) != 0 && (prot & VM_PROT_WRITE) != 0) {
for (m_end = &m[SP_PAGES]; m < m_end; m++) {
refchg = sp_refchg |
atomic_readandclear_32(&m->md.mdpg_attrs);
if (refchg & LPTE_CHG)
vm_page_dirty(m);
if (refchg & LPTE_REF)
vm_page_aflag_set(m, PGA_REFERENCED);
}
}
}
/* Superpage ops */
static int
moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, u_int flags, int8_t psind)
{
struct pvo_entry *pvo, **pvos;
struct pvo_head *pvo_head;
vm_offset_t sva;
vm_page_t sm;
vm_paddr_t pa;
bool sync;
struct pvo_dlist tofree;
int error, i;
uint16_t aflags;
KASSERT((va & SP_MASK) == 0, ("%s: va %#jx unaligned",
__func__, (uintmax_t)va));
KASSERT(psind == 1, ("%s: invalid psind: %d", __func__, psind));
KASSERT(m->psind == 1, ("%s: invalid m->psind: %d",
__func__, m->psind));
KASSERT(pmap != kernel_pmap,
("%s: function called with kernel pmap", __func__));
CTR5(KTR_PMAP, "%s: va=%#jx, pa=%#jx, prot=%#x, flags=%#x, psind=1",
__func__, (uintmax_t)va, (uintmax_t)VM_PAGE_TO_PHYS(m),
prot, flags);
SLIST_INIT(&tofree);
sva = va;
sm = m;
pa = VM_PAGE_TO_PHYS(sm);
/* Try to allocate all PVOs first, to make failure handling easier. */
pvos = malloc(SP_PAGES * sizeof(struct pvo_entry *), M_TEMP, M_NOWAIT);
if (pvos == NULL) {
CTR1(KTR_PMAP, "%s: failed to alloc pvo array", __func__);
return (KERN_RESOURCE_SHORTAGE);
}
for (i = 0; i < SP_PAGES; i++) {
pvos[i] = alloc_pvo_entry(0);
if (pvos[i] == NULL) {
CTR1(KTR_PMAP, "%s: failed to alloc pvo", __func__);
for (i = i - 1; i >= 0; i--)
free_pvo_entry(pvos[i]);
free(pvos, M_TEMP);
return (KERN_RESOURCE_SHORTAGE);
}
}
PV_PAGE_LOCK(sm);
PMAP_LOCK(pmap);
/* Note: moea64_remove_locked() also clears cached REF/CHG bits. */
moea64_remove_locked(pmap, va, va + SP_SIZE, &tofree);
/* Enter pages */
for (i = 0; i < SP_PAGES;
i++, va += PAGE_SIZE, pa += PAGE_SIZE, m++) {
pvo = pvos[i];
pvo->pvo_pte.prot = prot;
pvo->pvo_pte.pa = (pa & ~LPTE_LP_MASK) | LPTE_LP_4K_16M |
moea64_calc_wimg(pa, pmap_page_get_memattr(m));
if ((flags & PMAP_ENTER_WIRED) != 0)
pvo->pvo_vaddr |= PVO_WIRED;
pvo->pvo_vaddr |= PVO_LARGE;
if ((m->oflags & VPO_UNMANAGED) != 0)
pvo_head = NULL;
else {
pvo_head = &m->md.mdpg_pvoh;
pvo->pvo_vaddr |= PVO_MANAGED;
}
init_pvo_entry(pvo, pmap, va);
error = moea64_pvo_enter(pvo, pvo_head, NULL);
/*
* All superpage PVOs were previously removed, so no errors
* should occur while inserting the new ones.
*/
KASSERT(error == 0, ("%s: unexpected error "
"when inserting superpage PVO: %d",
__func__, error));
}
PMAP_UNLOCK(pmap);
PV_PAGE_UNLOCK(sm);
sync = (sm->a.flags & PGA_EXECUTABLE) == 0;
/* Note: moea64_pvo_cleanup() also clears page prot. flags. */
moea64_pvo_cleanup(&tofree);
pvo = pvos[0];
/* Set vm page flags */
aflags = pvo_to_vmpage_flags(pvo);
if (aflags != 0)
for (m = sm; m < &sm[SP_PAGES]; m++)
vm_page_aflag_set(m, aflags);
/*
* Flush the page from the instruction cache if this page is
* mapped executable and cacheable.
*/
if (sync && (pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)
moea64_syncicache(pmap, sva, VM_PAGE_TO_PHYS(sm), SP_SIZE);
atomic_add_long(&sp_mappings, 1);
CTR3(KTR_PMAP, "%s: SP success for va %#jx in pmap %p",
__func__, (uintmax_t)sva, pmap);
free(pvos, M_TEMP);
return (KERN_SUCCESS);
}
static int
moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m)
{
struct pvo_entry *first, *pvo;
vm_paddr_t pa, pa_end;
vm_offset_t sva, va_end;
int64_t sp_refchg;
/* Return if page promotion is not possible. */
if ((m->flags & PG_FICTITIOUS) != 0 ||
vm_reserv_level_iffullpop(m) != 0 || !moea64_ps_enabled(pmap))
return (1);
markjUnsubmitted
Done
Inline Actions

In other implementations, this check is in the caller. Why not here?

I would check moea64_ps_enabled() earlier, so as not to pessimize systems where superpages are disabled, since vm_reserv_level_iffullpop() is more expensive.

markj: In other implementations, this check is in the caller. Why not here? I would check…
luporlAuthorUnsubmitted
Done
Inline Actions

No special reason. I'll move the check to the caller.

Nice tip, I'll check moea64_ps_enabled() earlier.

luporl: No special reason. I'll move the check to the caller. Nice tip, I'll check moea64_ps_enabled()…
/* This CTR may generate a lot of output. */
/* CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)va); */
va &= ~SP_MASK;
sva = va;
/* Get superpage */
m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(m) & ~SP_MASK);
PV_PAGE_LOCK(m);
PMAP_LOCK(pmap);
/*
* Check if all pages meet promotion criteria.
*
* XXX In some cases the loop below may be executed for each or most
* of the entered pages of a superpage, which can be expensive
* (although it was not profiled) and need some optimization.
*
* Some cases where this seems to happen are:
* - When a superpage is first entered read-only and later becomes
* read-write.
* - When some of the superpage's virtual addresses map to previously
* wired/cached pages while others map to pages allocated from a
* different physical address range. A common scenario where this
* happens is when mmap'ing a file that is already present in FS
* block cache and doesn't fill a superpage.
*/
first = pvo = moea64_pvo_find_va(pmap, sva);
for (pa = VM_PAGE_TO_PHYS(m), pa_end = pa + SP_SIZE;
pa < pa_end; pa += PAGE_SIZE, va += PAGE_SIZE) {
if (pvo == NULL || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR3(KTR_PMAP,
"%s: null or dead pvo for va %#jx pmap %p",
__func__, (uintmax_t)va, pmap);
goto error;
}
if (PVO_PADDR(pvo) != pa) {
CTR3(KTR_PMAP, "%s: failure for va %#jx pmap %p "
"(pa doesn't match)",
__func__, (uintmax_t)va, pmap);
goto error;
}
if ((first->pvo_vaddr & PVO_PROMOTE) !=
(pvo->pvo_vaddr & PVO_PROMOTE) ||
first->pvo_pte.prot != pvo->pvo_pte.prot ||
(first->pvo_pte.pa & LPTE_WIMG) !=
(pvo->pvo_pte.pa & LPTE_WIMG)) {
/* This CTR may generate a lot of output. */
/*
CTR3(KTR_PMAP, "%s: failure for va %#jx pmap %p "
"(pvo flags don't match)",
__func__, (uintmax_t)va, pmap);
*/
goto error;
}
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);
}
/* All OK, promote. */
/*
* Handle superpage REF/CHG bits. If REF or CHG is set in
* any page, then it must be set in the superpage.
*
* Instead of querying each page, we take advantage of two facts:
* 1- If a page is being promoted, it was referenced.
* 2- If promoted pages are writable, they were modified.
*/
sp_refchg = LPTE_REF |
((first->pvo_pte.prot & VM_PROT_WRITE) != 0 ? LPTE_CHG : 0);
/* Promote pages */
pvo = first;
for (va = PVO_VADDR(pvo), va_end = va + SP_SIZE;
va < va_end; va += PAGE_SIZE) {
KASSERT(pvo && PVO_VADDR(pvo) == va,
("%s: missing PVO for va %#jx", __func__, (uintmax_t)va));
pvo->pvo_pte.pa &= ~LPTE_LP_MASK;
pvo->pvo_pte.pa |= LPTE_LP_4K_16M;
pvo->pvo_vaddr |= PVO_LARGE;
moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);
}
/* Send REF/CHG bits to VM */
moea64_sp_refchg_process(first, m, sp_refchg, first->pvo_pte.prot);
/* Use first page to cache REF/CHG bits */
atomic_set_32(&m->md.mdpg_attrs, sp_refchg | MDPG_ATTR_SP);
PMAP_UNLOCK(pmap);
PV_PAGE_UNLOCK(m);
atomic_add_long(&sp_mappings, 1);
atomic_add_long(&sp_promotions, 1);
markjUnsubmitted
Done
Inline Actions

Ouch, won't this be quite expensive? Do you actually need to invalidate the small mappings? This is one area where the various superpage implementations differ. ARM has an annoying requirement that a physical page not be aliased by both 4KB and large mappings, so promotions require a "break-before-make", in which the 4KB mappings are cleared (by clearing the L2 entry), TLBs are invalidated, and then the large mapping is created. See pmap_update_entry() on arm64 for example. amd64, i386 and riscv do not have this requirement and therefore do not perform TLB invalidation during promotion. I am not sure about the requirements on POWER, but it would be better to avoid or batch these invalidations if possible.

markj: Ouch, won't this be quite expensive? Do you actually need to invalidate the small mappings?
bdragonUnsubmitted
Done
Inline Actions

(Note: IBM differentiates between "Effective Address" and "Virtual Address" -- VA is an (up to) 78 bit address that is generally unique (although I suppose if two processes were to share the same VSID at any point, they could theoretically have 256M or 1T chunks of shared address space), EA is the normal 64 bit address that is the address that is stored in pointers and such)

There is no L2 entry, HPT is not a radix table, it's a segment + hash scheme, where the EA is converted into a VA by using the (32-entry) SLB to look up a VSID (or taking a segment fault, or using a process scoped segment table if we were to set one up), and then using that VSID and the page bits of the EA (as per the L,LP bits in the SLBE, for FreeBSD we can assume 4k base pages) to get the VPN, and then using that VPN as a hash key into the pagetable to identify the address of the correct PTEG to search. (As well as possibly doing a secondary search to come up with a second PTEG to scan)

The 8 records of the PTEG (and 8 of the secondary if enabled) are then matched against the properties of the SLBE to look for a match that satisfies the requirements.

It is vitally important that there not be overlapping matches in VA space because that can lead to multihits in the TLB if two different page sizes are observed at any point. This causes a machine check, which brings down the whole machine.

So yes, it is necessary to do a bunch of invalidation as far as I know.

See section 5.7 in Book III of the ISA. (Preferably 3.0) -- 5.7.7 is the main description of address translation.

FWIW, I wonder if we could optimize the pvo_entry structure by not tracking the entire address of the PTE slot and instead reconstructing it by running the hash function given the VSID.

bdragon: (Note: IBM differentiates between "Effective Address" and "Virtual Address" -- VA is an (up to)…
bdragonUnsubmitted
Done
Inline Actions

(that is, only tracking the offset into the PTEG and calculating the PTEG base address when we need it)

bdragon: (that is, only tracking the offset into the PTEG and calculating the PTEG base address when we…
bdragonUnsubmitted
Done
Inline Actions

Additional note: the main benefit of large pages on HPT is that the *tlb hit rate* goes up. You still need a PTE for every 4k of it if you want to avoid page faults, but a match will get cached in the TLB / ERATs as covering the entire range.

bdragon: Additional note: the main benefit of large pages on HPT is that the *tlb hit rate* goes up. You…
luporlAuthorUnsubmitted
Done
Inline Actions

As bdragon explained, we really need to invalidate all mappings.
We don't have a batched invalidation mechanism on PPC yet, but it could be a nice addition.

I've tried to optimize the PVO tree by using only one PVO to track a superpage, but I started facing some issues and gave up at that moment, to make everything work correctly first, before trying more optimizations.
From what I remember, the main issue was caused by not saving the PVO_HID bit for each 4K PTE. IIUC, we can't guess what it will be for a given 4K range within a superpage, because it is set when we fail to insert the PTE in the primary hash table and use the second one instead. But maybe we can save only this bit for each PTE, in a PVO bitmap, and make the idea of using only one PVO for superpage work.

luporl: As bdragon explained, we really need to invalidate all mappings. We don't have a batched…
luporlAuthorUnsubmitted
Done
Inline Actions

The last diff implements batch PTE unset and insert. TLB invalidations are performed during PTE unset.

It would be possible to perform all TLB invalidations as a separate step, but it would require more work and I'm don't know how much it would improve performance, compared to the current loop that removes the PTE and invalidates its TLB entry, since on POWER9 (and probably on POWER8 too) TLBIE don't need locks.

luporl: The last diff implements batch PTE unset and insert. TLB invalidations are performed during PTE…
CTR3(KTR_PMAP, "%s: success for va %#jx in pmap %p",
__func__, (uintmax_t)sva, pmap);
return (0);
error:
atomic_add_long(&sp_p_failures, 1);
PMAP_UNLOCK(pmap);
PV_PAGE_UNLOCK(m);
return (1);
}
static void
moea64_sp_demote_aligned(struct pvo_entry *sp)
{
struct pvo_entry *pvo;
vm_offset_t va, va_end;
vm_paddr_t pa;
vm_page_t m;
markjUnsubmitted
Done
Inline Actions

The return value is unused.

markj: The return value is unused.
luporlAuthorUnsubmitted
Done
Inline Actions

Right, I'll remove it.

luporl: Right, I'll remove it.
pmap_t pmap;
int64_t ret, refchg;
CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
pmap = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
pvo = sp;
pa = PVO_PADDR(pvo);
m = PHYS_TO_VM_PAGE(pa);
refchg = 0;
/* Demote pages */
for (va = PVO_VADDR(pvo), va_end = va + SP_SIZE;
va < va_end; va += PAGE_SIZE, pa += PAGE_SIZE) {
KASSERT(pvo && PVO_VADDR(pvo) == va,
("%s: missing PVO for va %#jx", __func__, (uintmax_t)va));
pvo->pvo_vaddr &= ~PVO_LARGE;
pvo->pvo_pte.pa &= ~LPTE_RPGN;
pvo->pvo_pte.pa |= pa;
ret = moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
if (ret < 0)
refchg |= LPTE_CHG;
else
refchg |= ret;
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);
}
/* Clear SP flag */
atomic_clear_32(&m->md.mdpg_attrs, MDPG_ATTR_SP);
/*
* Handle superpage REF/CHG bits. A bit set in the superpage
* means all pages should consider it set.
*/
moea64_sp_refchg_process(sp, m, refchg, sp->pvo_pte.prot);
atomic_add_long(&sp_demotions, 1);
CTR3(KTR_PMAP, "%s: success for va %#jx in pmap %p",
__func__, (uintmax_t)PVO_VADDR(sp), pmap);
}
static void
moea64_sp_demote(struct pvo_entry *pvo)
{
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
if ((PVO_VADDR(pvo) & SP_MASK) != 0) {
pvo = moea64_pvo_find_va(pvo->pvo_pmap,
PVO_VADDR(pvo) & ~SP_MASK);
KASSERT(pvo != NULL, ("%s: missing PVO for va %#jx",
__func__, (uintmax_t)(PVO_VADDR(pvo) & ~SP_MASK)));
}
moea64_sp_demote_aligned(pvo);
}
static struct pvo_entry *
moea64_sp_unwire(struct pvo_entry *sp)
{
struct pvo_entry *pvo, *prev;
vm_offset_t eva;
pmap_t pm;
int64_t ret, refchg;
CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
pm = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
prev = pvo, pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
panic("%s: pvo %p is missing PVO_WIRED",
__func__, pvo);
pvo->pvo_vaddr &= ~PVO_WIRED;
ret = moea64_pte_replace(pvo, 0 /* No invalidation */);
if (ret < 0)
refchg |= LPTE_CHG;
else
refchg |= ret;
pm->pm_stats.wired_count--;
}
/* Send REF/CHG bits to VM */
moea64_sp_refchg_process(sp, PHYS_TO_VM_PAGE(PVO_PADDR(sp)),
refchg, sp->pvo_pte.prot);
return (prev);
}
static struct pvo_entry *
moea64_sp_protect(struct pvo_entry *sp, vm_prot_t prot)
{
struct pvo_entry *pvo, *prev;
vm_offset_t eva;
pmap_t pm;
vm_page_t m, m_end;
int64_t ret, refchg;
vm_prot_t oldprot;
CTR3(KTR_PMAP, "%s: va=%#jx, prot=%x",
__func__, (uintmax_t)PVO_VADDR(sp), prot);
pm = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
oldprot = sp->pvo_pte.prot;
m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
KASSERT(m != NULL, ("%s: missing vm page for pa %#jx",
__func__, (uintmax_t)PVO_PADDR(sp)));
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
prev = pvo, pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
pvo->pvo_pte.prot = prot;
/*
* If the PVO is in the page table, update mapping
*/
ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
if (ret < 0)
refchg |= LPTE_CHG;
else
refchg |= ret;
}
/* Send REF/CHG bits to VM */
moea64_sp_refchg_process(sp, m, refchg, oldprot);
/* Handle pages that became executable */
if ((m->a.flags & PGA_EXECUTABLE) == 0 &&
(sp->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
if ((m->oflags & VPO_UNMANAGED) == 0)
for (m_end = &m[SP_PAGES]; m < m_end; m++)
vm_page_aflag_set(m, PGA_EXECUTABLE);
moea64_syncicache(pm, PVO_VADDR(sp), PVO_PADDR(sp), SP_SIZE);
}
return (prev);
}
static struct pvo_entry *
moea64_sp_remove(struct pvo_entry *sp, struct pvo_dlist *tofree)
{
struct pvo_entry *pvo, *tpvo;
vm_offset_t eva;
pmap_t pm;
CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
pm = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
eva = PVO_VADDR(sp) + SP_SIZE;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) {
tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo);
/*
* For locking reasons, remove this from the page table and
* pmap, but save delinking from the vm_page for a second
* pass
*/
moea64_pvo_remove_from_pmap(pvo);
SLIST_INSERT_HEAD(tofree, pvo, pvo_dlink);
}
/* Clear SP bit */
atomic_clear_32(&PHYS_TO_VM_PAGE(PVO_PADDR(sp))->md.mdpg_attrs,
MDPG_ATTR_SP);
return (tpvo);
}
static int64_t
moea64_sp_query_locked(struct pvo_entry *pvo, uint64_t ptebit)
{
int64_t refchg, ret;
vm_offset_t eva;
vm_page_t m;
pmap_t pmap;
struct pvo_entry *sp;
pmap = pvo->pvo_pmap;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/* Get first SP PVO */
if ((PVO_VADDR(pvo) & SP_MASK) != 0) {
sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~SP_MASK);
KASSERT(sp != NULL, ("%s: missing PVO for va %#jx",
__func__, (uintmax_t)(PVO_VADDR(pvo) & ~SP_MASK)));
} else
sp = pvo;
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
ret = moea64_pte_synch(pvo);
if (ret > 0) {
refchg |= ret & (LPTE_CHG | LPTE_REF);
if ((refchg & ptebit) != 0)
break;
}
}
/* Save results */
if (refchg != 0) {
m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
atomic_set_32(&m->md.mdpg_attrs, refchg | MDPG_ATTR_SP);
}
return (refchg);
}
static int64_t
moea64_sp_query(struct pvo_entry *pvo, uint64_t ptebit)
{
int64_t refchg;
pmap_t pmap;
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
/*
* Check if SP was demoted/removed before pmap lock was acquired.
*/
if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
__func__, (uintmax_t)PVO_PADDR(pvo));
PMAP_UNLOCK(pmap);
return (-1);
}
refchg = moea64_sp_query_locked(pvo, ptebit);
PMAP_UNLOCK(pmap);
CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",
__func__, (uintmax_t)PVO_VADDR(pvo),
(uintmax_t)PVO_PADDR(pvo), (uintmax_t)refchg);
return (refchg);
}
static int64_t
moea64_sp_pvo_clear(struct pvo_entry *pvo, uint64_t ptebit)
{
int64_t refchg, ret;
pmap_t pmap;
struct pvo_entry *sp;
vm_offset_t eva;
vm_page_t m;
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
/*
* Check if SP was demoted/removed before pmap lock was acquired.
*/
if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
__func__, (uintmax_t)PVO_PADDR(pvo));
PMAP_UNLOCK(pmap);
return (-1);
}
/* Get first SP PVO */
if ((PVO_VADDR(pvo) & SP_MASK) != 0) {
sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~SP_MASK);
KASSERT(sp != NULL, ("%s: missing PVO for va %#jx",
__func__, (uintmax_t)(PVO_VADDR(pvo) & ~SP_MASK)));
} else
sp = pvo;
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
ret = moea64_pte_clear(pvo, ptebit);
if (ret > 0)
refchg |= ret & (LPTE_CHG | LPTE_REF);
}
m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
atomic_clear_32(&m->md.mdpg_attrs, ptebit);
PMAP_UNLOCK(pmap);
CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",
__func__, (uintmax_t)PVO_VADDR(sp),
(uintmax_t)PVO_PADDR(sp), (uintmax_t)refchg);
return (refchg);
}
static int64_t
moea64_sp_clear(struct pvo_entry *pvo, vm_page_t m, uint64_t ptebit)
{
int64_t count, ret;
pmap_t pmap;
count = 0;
pmap = pvo->pvo_pmap;
/*
* Since this reference bit is shared by 4096 4KB pages, it
* should not be cleared every time it is tested. Apply a
* simple "hash" function on the physical page number, the
* virtual superpage number, and the pmap address to select
* one 4KB page out of the 4096 on which testing the
* reference bit will result in clearing that reference bit.
* This function is designed to avoid the selection of the
* same 4KB page for every 16MB page mapping.
*
* Always leave the reference bit of a wired mapping set, as
* the current state of its reference bit won't affect page
* replacement.
*/
if (ptebit == LPTE_REF && (((VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) ^
(PVO_VADDR(pvo) >> SP_SHIFT) ^ (uintptr_t)pmap) &
(SP_PAGES - 1)) == 0 && (pvo->pvo_vaddr & PVO_WIRED) == 0) {
if ((ret = moea64_sp_pvo_clear(pvo, ptebit)) == -1)
return (-1);
if ((ret & ptebit) != 0)
count++;
/*
* If this page was not selected by the hash function, then assume
* its REF bit was set.
*/
} else if (ptebit == LPTE_REF) {
count++;
/*
* To clear the CHG bit of a single SP page, first it must be demoted.
* But if no CHG bit is set, no bit clear and thus no SP demotion is
* needed.
*/
} else {
CTR4(KTR_PMAP, "%s: ptebit=%#jx, va=%#jx, pa=%#jx",
__func__, (uintmax_t)ptebit, (uintmax_t)PVO_VADDR(pvo),
(uintmax_t)PVO_PADDR(pvo));
PMAP_LOCK(pmap);
/*
* Make sure SP wasn't demoted/removed before pmap lock
* was acquired.
*/
if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
__func__, (uintmax_t)PVO_PADDR(pvo));
PMAP_UNLOCK(pmap);
return (-1);
}
ret = moea64_sp_query_locked(pvo, ptebit);
if ((ret & ptebit) != 0)
count++;
else {
PMAP_UNLOCK(pmap);
return (0);
}
moea64_sp_demote(pvo);
moea64_pte_clear(pvo, ptebit);
/*
* Write protect the mapping to a single page so that a
* subsequent write access may repromote.
*/
if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
moea64_pvo_protect(pmap, pvo,
pvo->pvo_pte.prot & ~VM_PROT_WRITE);
PMAP_UNLOCK(pmap);
}
return (count);
}
#endif