Page MenuHomeFreeBSD
Differential D25237 Diff 76890 sys/powerpc/aim/mmu_oea64.c

Changeset View

Standalone View

View Options

sys/powerpc/aim/mmu_oea64.c

Show First 20 Lines • Show All 74 Lines • ▼ Show 20 Lines
#include <vm/vm_param.h> #include <vm/vm_param.h>
#include <vm/vm_kern.h> #include <vm/vm_kern.h>
#include <vm/vm_page.h> #include <vm/vm_page.h>
#include <vm/vm_phys.h> #include <vm/vm_phys.h>
#include <vm/vm_map.h> #include <vm/vm_map.h>
#include <vm/vm_object.h> #include <vm/vm_object.h>
#include <vm/vm_extern.h> #include <vm/vm_extern.h>
#include <vm/vm_pageout.h> #include <vm/vm_pageout.h>
#include <vm/vm_reserv.h>
#include <vm/uma.h> #include <vm/uma.h>
#include <machine/_inttypes.h> #include <machine/_inttypes.h>
#include <machine/cpu.h> #include <machine/cpu.h>
#include <machine/ifunc.h> #include <machine/ifunc.h>
#include <machine/platform.h> #include <machine/platform.h>
#include <machine/frame.h> #include <machine/frame.h>
#include <machine/md_var.h> #include <machine/md_var.h>
Show All 12 Lines
#define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR) #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR)
#define ENABLE_TRANS(msr) mtmsr(msr) #define ENABLE_TRANS(msr) mtmsr(msr)
#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4)) #define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4))
#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff) #define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff)
#define VSID_HASH_MASK 0x0000007fffffffffULL #define VSID_HASH_MASK 0x0000007fffffffffULL
/* Get physical address from PVO. */
#define PVO_PADDR(pvo) ((pvo)->pvo_pte.pa & LPTE_RPGN)
/* /*
* Locking semantics: * Locking semantics:
* *
* There are two locks of interest: the page locks and the pmap locks, which * There are two locks of interest: the page locks and the pmap locks, which
* protect their individual PVO lists and are locked in that order. The contents * protect their individual PVO lists and are locked in that order. The contents
* of all PVO entries are protected by the locks of their respective pmaps. * of all PVO entries are protected by the locks of their respective pmaps.
* The pmap of any PVO is guaranteed not to change so long as the PVO is linked * The pmap of any PVO is guaranteed not to change so long as the PVO is linked
* into any list. * into any list.
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines
vm_offset_t moea64_scratchpage_va[2]; vm_offset_t moea64_scratchpage_va[2];
struct pvo_entry *moea64_scratchpage_pvo[2]; struct pvo_entry *moea64_scratchpage_pvo[2];
struct mtx moea64_scratchpage_mtx; struct mtx moea64_scratchpage_mtx;
uint64_t moea64_large_page_mask = 0; uint64_t moea64_large_page_mask = 0;
uint64_t moea64_large_page_size = 0; uint64_t moea64_large_page_size = 0;
int moea64_large_page_shift = 0; int moea64_large_page_shift = 0;
bool moea64_has_lp_4k_16m = false;
markjUnsubmitted
Not Done
Inline Actions

There seems to be a mix of "lp" and "sp" to denote "superpage". I think we should pick one and stick with it.

markj: There seems to be a mix of "lp" and "sp" to denote "superpage". I think we should pick one and…
luporlAuthorUnsubmitted
Done
Inline Actions

Yeah, I've used "sp" (superpage) during most of the changes. The idea was to call the parts that deal with the superpage feature (like promotion and demotion) "sp", to match how other arch's call them, but most places in PPC code use "lp" (large page) when referring to a page whose size is of several 4k pages, although currently PPC64 uses large pages for DMAP only.

I've used "lp" here because this is related to a PPC-specific hardware feature and closely related to others already named as large page.

For the rest of the code, changing "lp" and "large_page" to "sp" and "superpage" doesn't seem a good idea to me. I would also prefer to leave the parts that deal with promotion, demotion, etc. as superpages, but I guess it would be ok to rename them to "large_pages" too.
What do you think?

luporl: Yeah, I've used "sp" (superpage) during most of the changes. The idea was to call the parts…
/* /*
* PVO calls. * PVO calls.
*/ */
static int moea64_pvo_enter(struct pvo_entry *pvo, static int moea64_pvo_enter(struct pvo_entry *pvo,
struct pvo_head *pvo_head, struct pvo_entry **oldpvo); struct pvo_head *pvo_head, struct pvo_entry **oldpvo);
static void moea64_pvo_remove_from_pmap(struct pvo_entry *pvo); static void moea64_pvo_remove_from_pmap(struct pvo_entry *pvo);
static void moea64_pvo_remove_from_page(struct pvo_entry *pvo); static void moea64_pvo_remove_from_page(struct pvo_entry *pvo);
static void moea64_pvo_remove_from_page_locked( static void moea64_pvo_remove_from_page_locked(
struct pvo_entry *pvo, vm_page_t m); struct pvo_entry *pvo, vm_page_t m);
static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t); static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t);
/* /*
* Utility routines. * Utility routines.
*/ */
static boolean_t moea64_query_bit(vm_page_t, uint64_t); static boolean_t moea64_query_bit(vm_page_t, uint64_t);
static u_int moea64_clear_bit(vm_page_t, uint64_t); static u_int moea64_clear_bit(vm_page_t, uint64_t);
static void moea64_kremove(vm_offset_t); static void moea64_kremove(vm_offset_t);
static void moea64_syncicache(pmap_t pmap, vm_offset_t va, static void moea64_syncicache(pmap_t pmap, vm_offset_t va,
vm_paddr_t pa, vm_size_t sz); vm_paddr_t pa, vm_size_t sz);
static void moea64_pmap_init_qpages(void); static void moea64_pmap_init_qpages(void);
static void moea64_remove_locked(pmap_t, vm_offset_t,
vm_offset_t, struct pvo_dlist *);
/* /*
* Superpages data and routines.
*/
/*
* PVO flags (in vaddr) that must match for promotion to succeed.
* Note that protection bits are checked separately, as they reside in
* another field.
*/
#define PVO_FLAGS_PROMOTE (PVO_WIRED | PVO_MANAGED | PVO_PTEGIDX_VALID)
markjUnsubmitted
Done
Inline Actions

What about protection bits? Oh I see, those are checked separately. It might be worth adding a sentence explaining that.

Why is PVO_LARGE included?

markj: What about protection bits? Oh I see, those are checked separately. It might be worth adding a…
luporlAuthorUnsubmitted
Done
Inline Actions

Right, protection bits are in another PVO field and are checked separately. I'll add a comment about it, and maybe rename PVO_PROMOTE to PVO_FLAGS_PROMOTE, to make it clear.

PVO_LARGE was added to make sure all pages are of the same size (4K), but I guess it is not necessary to check it during promotion, because if any non-4K page is found during promotion, it means the logic is really broken.

luporl: Right, protection bits are in another PVO field and are checked separately. I'll add a comment…
#define PVO_IS_SP(pvo) (((pvo)->pvo_vaddr & PVO_LARGE) && \
(pvo)->pvo_pmap != kernel_pmap)
/* Get physical address from PVO. */
#define PVO_PADDR(pvo) moea64_pvo_paddr(pvo)
/* MD page flag indicating that the page is a superpage. */
#define MDPG_ATTR_SP 0x40000000
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0,
"VM/pmap parameters");
static int superpages_enabled = 0;
SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, CTLFLAG_RDTUN,
markjUnsubmitted
Done
Inline Actions

pg_ps_enabled is kind of a misnomer: "ps" refers to PG_PS, which is the name of the bit in x86 page tables that indicates that a PDE or PDPE maps a superpage. Other arches just call it superpages_enabled.

markj: pg_ps_enabled is kind of a misnomer: "ps" refers to PG_PS, which is the name of the bit in x86…
luporlAuthorUnsubmitted
Done
Inline Actions

Ah, ok, thanks for the explanation, I'll change it.

luporl: Ah, ok, thanks for the explanation, I'll change it.
&superpages_enabled, 0, "Enable support for transparent superpages");
static SYSCTL_NODE(_vm_pmap, OID_AUTO, sp, CTLFLAG_RD, 0,
"SP page mapping counters");
static u_long sp_demotions;
jhibbitsUnsubmitted
Done
Inline Actions

Radix (and amd64 at least) already uses vm.pmap.pg_ps_enabled, can you use that instead, for consistency?

jhibbits: Radix (and amd64 at least) already uses vm.pmap.pg_ps_enabled, can you use that instead, for…
luporlAuthorUnsubmitted
Done
Inline Actions

Sure.

luporl: Sure.
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, demotions, CTLFLAG_RD,
&sp_demotions, 0, "SP page demotions");
static u_long sp_mappings;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, mappings, CTLFLAG_RD,
&sp_mappings, 0, "SP page mappings");
static u_long sp_p_failures;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_failures, CTLFLAG_RD,
&sp_p_failures, 0, "SP page promotion failures");
static u_long sp_p_fail_pa;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_pa, CTLFLAG_RD,
&sp_p_fail_pa, 0, "SP page promotion failure: PAs don't match");
static u_long sp_p_fail_flags;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_flags, CTLFLAG_RD,
&sp_p_fail_flags, 0, "SP page promotion failure: page flags don't match");
static u_long sp_p_fail_prot;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_prot, CTLFLAG_RD,
&sp_p_fail_prot, 0,
"SP page promotion failure: page protections don't match");
static u_long sp_p_fail_wimg;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_wimg, CTLFLAG_RD,
&sp_p_fail_wimg, 0, "SP page promotion failure: WIMG bits don't match");
static u_long sp_promotions;
SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, promotions, CTLFLAG_RD,
&sp_promotions, 0, "SP page promotions");
static bool moea64_ps_enabled(pmap_t);
static void moea64_align_superpage(vm_object_t, vm_ooffset_t,
vm_offset_t *, vm_size_t);
static int moea64_sp_enter(pmap_t pmap, vm_offset_t va,
vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind);
static struct pvo_entry *moea64_sp_remove(struct pvo_entry *sp,
struct pvo_dlist *tofree);
static void moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m);
static void moea64_sp_demote_aligned(struct pvo_entry *sp);
static void moea64_sp_demote(struct pvo_entry *pvo);
static struct pvo_entry *moea64_sp_unwire(struct pvo_entry *sp);
static struct pvo_entry *moea64_sp_protect(struct pvo_entry *sp,
vm_prot_t prot);
static int64_t moea64_sp_query(struct pvo_entry *pvo, uint64_t ptebit);
static int64_t moea64_sp_clear(struct pvo_entry *pvo, vm_page_t m,
uint64_t ptebit);
static __inline bool moea64_sp_pvo_in_range(struct pvo_entry *pvo,
vm_offset_t sva, vm_offset_t eva);
/*
* Kernel MMU interface * Kernel MMU interface
*/ */
void moea64_clear_modify(vm_page_t); void moea64_clear_modify(vm_page_t);
void moea64_copy_page(vm_page_t, vm_page_t); void moea64_copy_page(vm_page_t, vm_page_t);
void moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset, void moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
vm_page_t *mb, vm_offset_t b_offset, int xfersize); vm_page_t *mb, vm_offset_t b_offset, int xfersize);
int moea64_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, int moea64_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t,
u_int flags, int8_t psind); u_int flags, int8_t psind);
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Linesstatic struct pmap_funcs moea64_methods = {
.deactivate = moea64_deactivate, .deactivate = moea64_deactivate,
.page_set_memattr = moea64_page_set_memattr, .page_set_memattr = moea64_page_set_memattr,
.quick_enter_page = moea64_quick_enter_page, .quick_enter_page = moea64_quick_enter_page,
.quick_remove_page = moea64_quick_remove_page, .quick_remove_page = moea64_quick_remove_page,
.page_is_mapped = moea64_page_is_mapped, .page_is_mapped = moea64_page_is_mapped,
#ifdef __powerpc64__ #ifdef __powerpc64__
.page_array_startup = moea64_page_array_startup, .page_array_startup = moea64_page_array_startup,
#endif #endif
.ps_enabled = moea64_ps_enabled,
.align_superpage = moea64_align_superpage,
/* Internal interfaces */ /* Internal interfaces */
.mapdev = moea64_mapdev, .mapdev = moea64_mapdev,
.mapdev_attr = moea64_mapdev_attr, .mapdev_attr = moea64_mapdev_attr,
.unmapdev = moea64_unmapdev, .unmapdev = moea64_unmapdev,
.kextract = moea64_kextract, .kextract = moea64_kextract,
.kenter = moea64_kenter, .kenter = moea64_kenter,
.kenter_attr = moea64_kenter_attr, .kenter_attr = moea64_kenter_attr,
.dev_direct_mapped = moea64_dev_direct_mapped, .dev_direct_mapped = moea64_dev_direct_mapped,
.dumpsys_pa_init = moea64_scan_init, .dumpsys_pa_init = moea64_scan_init,
.dumpsys_scan_pmap = moea64_scan_pmap, .dumpsys_scan_pmap = moea64_scan_pmap,
.dumpsys_dump_pmap_init = moea64_dump_pmap_init, .dumpsys_dump_pmap_init = moea64_dump_pmap_init,
.dumpsys_map_chunk = moea64_dumpsys_map, .dumpsys_map_chunk = moea64_dumpsys_map,
.map_user_ptr = moea64_map_user_ptr, .map_user_ptr = moea64_map_user_ptr,
.decode_kernel_ptr = moea64_decode_kernel_ptr, .decode_kernel_ptr = moea64_decode_kernel_ptr,
}; };
MMU_DEF(oea64_mmu, "mmu_oea64_base", moea64_methods); MMU_DEF(oea64_mmu, "mmu_oea64_base", moea64_methods);
/*
* Get physical address from PVO.
*
* For superpages, the lower bits are not stored on pvo_pte.pa and must be
* obtained from VA.
*/
static __inline vm_paddr_t
moea64_pvo_paddr(struct pvo_entry *pvo)
{
vm_paddr_t pa;
pa = (pvo)->pvo_pte.pa & LPTE_RPGN;
if (PVO_IS_SP(pvo)) {
pa &= ~SP_MASK; /* This is needed to clear LPTE_LP bits. */
pa |= PVO_VADDR(pvo) & SP_MASK;
}
return (pa);
}
static struct pvo_head * static struct pvo_head *
vm_page_to_pvoh(vm_page_t m) vm_page_to_pvoh(vm_page_t m)
{ {
mtx_assert(PV_LOCKPTR(VM_PAGE_TO_PHYS(m)), MA_OWNED); mtx_assert(PV_LOCKPTR(VM_PAGE_TO_PHYS(m)), MA_OWNED);
return (&m->md.mdpg_pvoh); return (&m->md.mdpg_pvoh);
} }
Show All 32 Linesinit_pvo_entry(struct pvo_entry *pvo, pmap_t pmap, vm_offset_t va)
pvo->pvo_pmap = pmap; pvo->pvo_pmap = pmap;
va &= ~ADDR_POFF; va &= ~ADDR_POFF;
pvo->pvo_vaddr |= va; pvo->pvo_vaddr |= va;
vsid = va_to_vsid(pmap, va); vsid = va_to_vsid(pmap, va);
pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT) pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT)
| (vsid << 16); | (vsid << 16);
shift = (pvo->pvo_vaddr & PVO_LARGE) ? moea64_large_page_shift : if (pmap == kernel_pmap && (pvo->pvo_vaddr & PVO_LARGE) != 0)
ADDR_PIDX_SHFT; shift = moea64_large_page_shift;
else
shift = ADDR_PIDX_SHFT;
hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)va & ADDR_PIDX) >> shift); hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)va & ADDR_PIDX) >> shift);
pvo->pvo_pte.slot = (hash & moea64_pteg_mask) << 3; pvo->pvo_pte.slot = (hash & moea64_pteg_mask) << 3;
} }
static void static void
free_pvo_entry(struct pvo_entry *pvo) free_pvo_entry(struct pvo_entry *pvo)
{ {
▲ Show 20 Lines • Show All 327 Lines • ▼ Show 20 Lines
void void
moea64_early_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend) moea64_early_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
{ {
int i, j; int i, j;
vm_size_t physsz, hwphyssz; vm_size_t physsz, hwphyssz;
vm_paddr_t kernelphysstart, kernelphysend; vm_paddr_t kernelphysstart, kernelphysend;
int rm_pavail; int rm_pavail;
/* Level 0 reservations consist of 4096 pages (16MB superpage). */
vm_level_0_order = 12;
#ifndef __powerpc64__ #ifndef __powerpc64__
/* We don't have a direct map since there is no BAT */ /* We don't have a direct map since there is no BAT */
hw_direct_map = 0; hw_direct_map = 0;
/* Make sure battable is zero, since we have no BAT */ /* Make sure battable is zero, since we have no BAT */
for (i = 0; i < 16; i++) { for (i = 0; i < 16; i++) {
battable[i].batu = 0; battable[i].batu = 0;
battable[i].batl = 0; battable[i].batl = 0;
▲ Show 20 Lines • Show All 415 Lines • ▼ Show 20 Linesmoea64_unwire(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
vm_page_t m; vm_page_t m;
int64_t refchg; int64_t refchg;
key.pvo_vaddr = sva; key.pvo_vaddr = sva;
PMAP_LOCK(pm); PMAP_LOCK(pm);
for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
pvo != NULL && PVO_VADDR(pvo) < eva; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) { pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
if (PVO_IS_SP(pvo)) {
if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
pvo = moea64_sp_unwire(pvo);
continue;
} else {
CTR1(KTR_PMAP, "%s: demote before unwire",
__func__);
moea64_sp_demote(pvo);
}
}
if ((pvo->pvo_vaddr & PVO_WIRED) == 0) if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
panic("moea64_unwire: pvo %p is missing PVO_WIRED", panic("moea64_unwire: pvo %p is missing PVO_WIRED",
pvo); pvo);
pvo->pvo_vaddr &= ~PVO_WIRED; pvo->pvo_vaddr &= ~PVO_WIRED;
refchg = moea64_pte_replace(pvo, 0 /* No invalidation */); refchg = moea64_pte_replace(pvo, 0 /* No invalidation */);
if ((pvo->pvo_vaddr & PVO_MANAGED) && if ((pvo->pvo_vaddr & PVO_MANAGED) &&
(pvo->pvo_pte.prot & VM_PROT_WRITE)) { (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
if (refchg < 0) if (refchg < 0)
▲ Show 20 Lines • Show All 224 Lines • ▼ Show 20 Lines
* target pmap with the protection requested. If specified the page * target pmap with the protection requested. If specified the page
* will be wired down. * will be wired down.
*/ */
int int
moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, u_int flags, int8_t psind) vm_prot_t prot, u_int flags, int8_t psind)
{ {
struct pvo_entry *pvo, *oldpvo; struct pvo_entry *pvo, *oldpvo, *tpvo;
struct pvo_head *pvo_head; struct pvo_head *pvo_head;
uint64_t pte_lo; uint64_t pte_lo;
int error; int error;
if ((m->oflags & VPO_UNMANAGED) == 0) { if ((m->oflags & VPO_UNMANAGED) == 0) {
if ((flags & PMAP_ENTER_QUICK_LOCKED) == 0) if ((flags & PMAP_ENTER_QUICK_LOCKED) == 0)
VM_PAGE_OBJECT_BUSY_ASSERT(m); VM_PAGE_OBJECT_BUSY_ASSERT(m);
else else
VM_OBJECT_ASSERT_LOCKED(m->object); VM_OBJECT_ASSERT_LOCKED(m->object);
} }
if (psind > 0)
return (moea64_sp_enter(pmap, va, m, prot, flags, psind));
pvo = alloc_pvo_entry(0); pvo = alloc_pvo_entry(0);
if (pvo == NULL) if (pvo == NULL)
return (KERN_RESOURCE_SHORTAGE); return (KERN_RESOURCE_SHORTAGE);
pvo->pvo_pmap = NULL; /* to be filled in later */ pvo->pvo_pmap = NULL; /* to be filled in later */
pvo->pvo_pte.prot = prot; pvo->pvo_pte.prot = prot;
pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m)); pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m));
pvo->pvo_pte.pa = VM_PAGE_TO_PHYS(m) | pte_lo; pvo->pvo_pte.pa = VM_PAGE_TO_PHYS(m) | pte_lo;
if ((flags & PMAP_ENTER_WIRED) != 0) if ((flags & PMAP_ENTER_WIRED) != 0)
pvo->pvo_vaddr |= PVO_WIRED; pvo->pvo_vaddr |= PVO_WIRED;
if ((m->oflags & VPO_UNMANAGED) != 0 || !moea64_initialized) { if ((m->oflags & VPO_UNMANAGED) != 0 || !moea64_initialized) {
pvo_head = NULL; pvo_head = NULL;
} else { } else {
pvo_head = &m->md.mdpg_pvoh; pvo_head = &m->md.mdpg_pvoh;
pvo->pvo_vaddr |= PVO_MANAGED; pvo->pvo_vaddr |= PVO_MANAGED;
} }
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
if (pvo->pvo_pmap == NULL) if (pvo->pvo_pmap == NULL)
init_pvo_entry(pvo, pmap, va); init_pvo_entry(pvo, pmap, va);
tpvo = moea64_pvo_find_va(pmap, va & ~SP_MASK);
if (tpvo && PVO_IS_SP(tpvo)) {
/* Demote SP before entering a regular page */
CTR2(KTR_PMAP, "%s: demote before enter: va=%#jx",
__func__, (uintmax_t)va);
moea64_sp_demote_aligned(tpvo);
}
if (prot & VM_PROT_WRITE) if (prot & VM_PROT_WRITE)
if (pmap_bootstrapped && if (pmap_bootstrapped &&
(m->oflags & VPO_UNMANAGED) == 0) (m->oflags & VPO_UNMANAGED) == 0)
vm_page_aflag_set(m, PGA_WRITEABLE); vm_page_aflag_set(m, PGA_WRITEABLE);
error = moea64_pvo_enter(pvo, pvo_head, &oldpvo); error = moea64_pvo_enter(pvo, pvo_head, &oldpvo);
if (error == EEXIST) { if (error == EEXIST) {
if (oldpvo->pvo_vaddr == pvo->pvo_vaddr && if (oldpvo->pvo_vaddr == pvo->pvo_vaddr &&
oldpvo->pvo_pte.pa == pvo->pvo_pte.pa && oldpvo->pvo_pte.pa == pvo->pvo_pte.pa &&
oldpvo->pvo_pte.prot == prot) { oldpvo->pvo_pte.prot == prot) {
/* Identical mapping already exists */ /* Identical mapping already exists */
error = 0; error = 0;
/* If not in page table, reinsert it */ /* If not in page table, reinsert it */
if (moea64_pte_synch(oldpvo) < 0) { if (moea64_pte_synch(oldpvo) < 0) {
STAT_MOEA64(moea64_pte_overflow--); STAT_MOEA64(moea64_pte_overflow--);
moea64_pte_insert(oldpvo); moea64_pte_insert(oldpvo);
} }
/* Then just clean up and go home */ /* Then just clean up and go home */
PV_PAGE_UNLOCK(m); PV_PAGE_UNLOCK(m);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
free_pvo_entry(pvo); free_pvo_entry(pvo);
pvo = NULL;
goto out; goto out;
} else { } else {
/* Otherwise, need to kill it first */ /* Otherwise, need to kill it first */
KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old " KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old "
"mapping does not match new mapping")); "mapping does not match new mapping"));
moea64_pvo_remove_from_pmap(oldpvo); moea64_pvo_remove_from_pmap(oldpvo);
moea64_pvo_enter(pvo, pvo_head, NULL); moea64_pvo_enter(pvo, pvo_head, NULL);
} }
Show All 12 Linesout:
* Flush the page from the instruction cache if this page is * Flush the page from the instruction cache if this page is
* mapped executable and cacheable. * mapped executable and cacheable.
*/ */
if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 && if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 &&
(pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { (pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
vm_page_aflag_set(m, PGA_EXECUTABLE); vm_page_aflag_set(m, PGA_EXECUTABLE);
moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE); moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE);
} }
/*
* Try to promote pages.
*
* If the VA of the entered page is not aligned with its PA,
* don't try page promotion as it is not possible.
* This reduces the number of promotion failures dramatically.
*/
if (moea64_ps_enabled(pmap) && pmap != kernel_pmap && pvo != NULL &&
markjUnsubmitted
Not Done
Inline Actions

Why do we not promote superpages in the kernel map? Most kernel mappings are unmanaged, so the check below will catch them, but exec_map and pipe_map create managed mappings that can in principle be promoted.

markj: Why do we not promote superpages in the kernel map? Most kernel mappings are unmanaged, so the…
luporlAuthorUnsubmitted
Done
Inline Actions

I was afraid that using superpages in kernel map could make things more complex and cause some breaks and also wasn't sure about the benefits.
But I can give it a try now and see how things go.

luporl: I was afraid that using superpages in kernel map could make things more complex and cause some…
luporlAuthorUnsubmitted
Done
Inline Actions

As this can have a big impact in performance and stability, I'll try it after improving current changes' performance.

luporl: As this can have a big impact in performance and stability, I'll try it after improving current…
(pvo->pvo_vaddr & PVO_MANAGED) != 0 &&
(va & SP_MASK) == (VM_PAGE_TO_PHYS(m) & SP_MASK) &&
(m->flags & PG_FICTITIOUS) == 0 &&
vm_reserv_level_iffullpop(m) == 0)
moea64_sp_promote(pmap, va, m);
return (KERN_SUCCESS); return (KERN_SUCCESS);
} }
static void static void
moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
vm_size_t sz) vm_size_t sz)
{ {
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
* corresponding offset from m_start are mapped. * corresponding offset from m_start are mapped.
*/ */
void void
moea64_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, moea64_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end,
vm_page_t m_start, vm_prot_t prot) vm_page_t m_start, vm_prot_t prot)
{ {
vm_page_t m; vm_page_t m;
vm_pindex_t diff, psize; vm_pindex_t diff, psize;
vm_offset_t va;
int8_t psind;
VM_OBJECT_ASSERT_LOCKED(m_start->object); VM_OBJECT_ASSERT_LOCKED(m_start->object);
psize = atop(end - start); psize = atop(end - start);
m = m_start; m = m_start;
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
moea64_enter(pm, start + ptoa(diff), m, prot & va = start + ptoa(diff);
(VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP | if ((va & SP_MASK) == 0 && va + SP_SIZE <= end &&
PMAP_ENTER_QUICK_LOCKED, 0); m->psind == 1 && moea64_ps_enabled(pm))
psind = 1;
else
psind = 0;
moea64_enter(pm, va, m, prot &
(VM_PROT_READ | VM_PROT_EXECUTE),
PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, psind);
if (psind == 1)
m = &m[SP_SIZE / PAGE_SIZE - 1];
m = TAILQ_NEXT(m, listq); m = TAILQ_NEXT(m, listq);
} }
} }
void void
moea64_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, moea64_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m,
vm_prot_t prot) vm_prot_t prot)
{ {
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines
{ {
CTR0(KTR_PMAP, "moea64_init"); CTR0(KTR_PMAP, "moea64_init");
moea64_pvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), moea64_pvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry),
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
UMA_ZONE_VM | UMA_ZONE_NOFREE); UMA_ZONE_VM | UMA_ZONE_NOFREE);
/*
* Are large page mappings enabled?
*/
TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
if (superpages_enabled) {
KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
("moea64_init: can't assign to pagesizes[1]"));
if (moea64_large_page_size == 0) {
printf("mmu_oea64: HW does not support large pages. "
"Disabling superpages...\n");
superpages_enabled = 0;
} else if (!moea64_has_lp_4k_16m) {
jhibbitsUnsubmitted
Done
Inline Actions

Is a DMAP really a prerequisite for superpages? Or do they just share the same prerequisites?

jhibbits: Is a DMAP really a prerequisite for superpages? Or do they just share the same prerequisites?
luporlAuthorUnsubmitted
Done
Inline Actions

I guess they share the same prerequisite: large pages availability.
What if I change !hw_direct_map to moea64_large_page_size == 0?

luporl: I guess they share the same prerequisite: large pages availability. What if I change `!
luporlAuthorUnsubmitted
Done
Inline Actions

Changed to moea64_large_page_size == 0.

luporl: Changed to `moea64_large_page_size == 0`.
printf("mmu_oea64: "
"HW does not support mixed 4KB/16MB page sizes. "
"Disabling superpages...\n");
superpages_enabled = 0;
} else
pagesizes[1] = SP_SIZE;
}
if (!hw_direct_map) { if (!hw_direct_map) {
uma_zone_set_allocf(moea64_pvo_zone, moea64_uma_page_alloc); uma_zone_set_allocf(moea64_pvo_zone, moea64_uma_page_alloc);
} }
#ifdef COMPAT_FREEBSD32 #ifdef COMPAT_FREEBSD32
elf32_nxstack = 1; elf32_nxstack = 1;
#endif #endif
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Linesmoea64_remove_write(vm_page_t m)
int64_t refchg, ret; int64_t refchg, ret;
pmap_t pmap; pmap_t pmap;
KASSERT((m->oflags & VPO_UNMANAGED) == 0, KASSERT((m->oflags & VPO_UNMANAGED) == 0,
("moea64_remove_write: page %p is not managed", m)); ("moea64_remove_write: page %p is not managed", m));
vm_page_assert_busied(m); vm_page_assert_busied(m);
if (!pmap_page_is_write_mapped(m)) if (!pmap_page_is_write_mapped(m))
return return;
powerpc_sync(); powerpc_sync();
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
refchg = 0; refchg = 0;
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
pmap = pvo->pvo_pmap; pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
if (!(pvo->pvo_vaddr & PVO_DEAD) && if (!(pvo->pvo_vaddr & PVO_DEAD) &&
(pvo->pvo_pte.prot & VM_PROT_WRITE)) { (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
if (PVO_IS_SP(pvo)) {
CTR1(KTR_PMAP, "%s: demote before remwr",
__func__);
moea64_sp_demote(pvo);
}
pvo->pvo_pte.prot &= ~VM_PROT_WRITE; pvo->pvo_pte.prot &= ~VM_PROT_WRITE;
ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE); ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
if (ret < 0) if (ret < 0)
ret = LPTE_CHG; ret = LPTE_CHG;
refchg |= ret; refchg |= ret;
if (pvo->pvo_pmap == kernel_pmap) if (pvo->pvo_pmap == kernel_pmap)
isync(); isync();
} }
Show All 32 Lines
void void
moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma) moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma)
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
int64_t refchg; int64_t refchg;
pmap_t pmap; pmap_t pmap;
uint64_t lo; uint64_t lo;
CTR3(KTR_PMAP, "%s: pa=%#jx, ma=%#x",
__func__, (uintmax_t)VM_PAGE_TO_PHYS(m), ma);
if ((m->oflags & VPO_UNMANAGED) != 0) { if ((m->oflags & VPO_UNMANAGED) != 0) {
m->md.mdpg_cache_attrs = ma; m->md.mdpg_cache_attrs = ma;
return; return;
} }
lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), ma); lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), ma);
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
pmap = pvo->pvo_pmap; pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
if (!(pvo->pvo_vaddr & PVO_DEAD)) { if (!(pvo->pvo_vaddr & PVO_DEAD)) {
if (PVO_IS_SP(pvo)) {
CTR1(KTR_PMAP,
"%s: demote before set_memattr", __func__);
moea64_sp_demote(pvo);
}
pvo->pvo_pte.pa &= ~LPTE_WIMG; pvo->pvo_pte.pa &= ~LPTE_WIMG;
pvo->pvo_pte.pa |= lo; pvo->pvo_pte.pa |= lo;
refchg = moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE); refchg = moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
if (refchg < 0) if (refchg < 0)
refchg = (pvo->pvo_pte.prot & VM_PROT_WRITE) ? refchg = (pvo->pvo_pte.prot & VM_PROT_WRITE) ?
LPTE_CHG : 0; LPTE_CHG : 0;
if ((pvo->pvo_vaddr & PVO_MANAGED) && if ((pvo->pvo_vaddr & PVO_MANAGED) &&
(pvo->pvo_pte.prot & VM_PROT_WRITE)) { (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
▲ Show 20 Lines • Show All 436 Lines • ▼ Show 20 Lines if (refchg & LPTE_REF)
vm_page_aflag_set(pg, PGA_REFERENCED); vm_page_aflag_set(pg, PGA_REFERENCED);
} }
} }
void void
moea64_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, moea64_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva,
vm_prot_t prot) vm_prot_t prot)
{ {
struct pvo_entry *pvo, *tpvo, key; struct pvo_entry *pvo, key;
CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm,
sva, eva, prot); sva, eva, prot);
KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap,
("moea64_protect: non current pmap")); ("moea64_protect: non current pmap"));
if ((prot & VM_PROT_READ) == VM_PROT_NONE) { if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
moea64_remove(pm, sva, eva); moea64_remove(pm, sva, eva);
return; return;
} }
PMAP_LOCK(pm); PMAP_LOCK(pm);
key.pvo_vaddr = sva; key.pvo_vaddr = sva;
for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { pvo != NULL && PVO_VADDR(pvo) < eva;
tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
if (PVO_IS_SP(pvo)) {
if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
pvo = moea64_sp_protect(pvo, prot);
continue;
} else {
CTR1(KTR_PMAP, "%s: demote before protect",
__func__);
moea64_sp_demote(pvo);
}
}
moea64_pvo_protect(pm, pvo, prot); moea64_pvo_protect(pm, pvo, prot);
} }
PMAP_UNLOCK(pm); PMAP_UNLOCK(pm);
} }
/* /*
* Map a list of wired pages into kernel virtual address space. This is * Map a list of wired pages into kernel virtual address space. This is
* intended for temporary mappings which do not need page modification or * intended for temporary mappings which do not need page modification or
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Linesmoea64_remove_pages(pmap_t pm)
while (!SLIST_EMPTY(&tofree)) { while (!SLIST_EMPTY(&tofree)) {
pvo = SLIST_FIRST(&tofree); pvo = SLIST_FIRST(&tofree);
SLIST_REMOVE_HEAD(&tofree, pvo_dlink); SLIST_REMOVE_HEAD(&tofree, pvo_dlink);
moea64_pvo_remove_from_page(pvo); moea64_pvo_remove_from_page(pvo);
free_pvo_entry(pvo); free_pvo_entry(pvo);
} }
} }
/* static void
* Remove the given range of addresses from the specified map. moea64_remove_locked(pmap_t pm, vm_offset_t sva, vm_offset_t eva,
*/ struct pvo_dlist *tofree)
void
moea64_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{ {
struct pvo_entry *pvo, *tpvo, key; struct pvo_entry *pvo, *tpvo, key;
struct pvo_dlist tofree;
/* PMAP_LOCK_ASSERT(pm, MA_OWNED);
* Perform an unsynchronized read. This is, however, safe.
*/
if (pm->pm_stats.resident_count == 0)
return;
key.pvo_vaddr = sva; key.pvo_vaddr = sva;
SLIST_INIT(&tofree);
PMAP_LOCK(pm);
for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) {
if (PVO_IS_SP(pvo)) {
if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
tpvo = moea64_sp_remove(pvo, tofree);
continue;
} else {
CTR1(KTR_PMAP, "%s: demote before remove",
__func__);
moea64_sp_demote(pvo);
}
}
tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo);
/* /*
* For locking reasons, remove this from the page table and * For locking reasons, remove this from the page table and
* pmap, but save delinking from the vm_page for a second * pmap, but save delinking from the vm_page for a second
* pass * pass
*/ */
moea64_pvo_remove_from_pmap(pvo); moea64_pvo_remove_from_pmap(pvo);
SLIST_INSERT_HEAD(&tofree, pvo, pvo_dlink); SLIST_INSERT_HEAD(tofree, pvo, pvo_dlink);
} }
}
/*
* Remove the given range of addresses from the specified map.
*/
void
moea64_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{
struct pvo_entry *pvo;
struct pvo_dlist tofree;
/*
* Perform an unsynchronized read. This is, however, safe.
*/
if (pm->pm_stats.resident_count == 0)
return;
SLIST_INIT(&tofree);
PMAP_LOCK(pm);
moea64_remove_locked(pm, sva, eva, &tofree);
PMAP_UNLOCK(pm); PMAP_UNLOCK(pm);
while (!SLIST_EMPTY(&tofree)) { while (!SLIST_EMPTY(&tofree)) {
pvo = SLIST_FIRST(&tofree); pvo = SLIST_FIRST(&tofree);
SLIST_REMOVE_HEAD(&tofree, pvo_dlink); SLIST_REMOVE_HEAD(&tofree, pvo_dlink);
moea64_pvo_remove_from_page(pvo); moea64_pvo_remove_from_page(pvo);
free_pvo_entry(pvo); free_pvo_entry(pvo);
} }
Show All 13 Linesmoea64_remove_all(vm_page_t m)
LIST_INIT(&freequeue); LIST_INIT(&freequeue);
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) { LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) {
pmap = pvo->pvo_pmap; pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap); PMAP_LOCK(pmap);
wasdead = (pvo->pvo_vaddr & PVO_DEAD); wasdead = (pvo->pvo_vaddr & PVO_DEAD);
if (!wasdead) if (!wasdead) {
if (PVO_IS_SP(pvo)) {
CTR1(KTR_PMAP, "%s: demote before remove_all",
__func__);
moea64_sp_demote(pvo);
}
moea64_pvo_remove_from_pmap(pvo); moea64_pvo_remove_from_pmap(pvo);
}
moea64_pvo_remove_from_page_locked(pvo, m); moea64_pvo_remove_from_page_locked(pvo, m);
if (!wasdead) if (!wasdead)
LIST_INSERT_HEAD(&freequeue, pvo, pvo_vlink); LIST_INSERT_HEAD(&freequeue, pvo, pvo_vlink);
PMAP_UNLOCK(pmap); PMAP_UNLOCK(pmap);
} }
KASSERT(!pmap_page_is_mapped(m), ("Page still has mappings")); KASSERT(!pmap_page_is_mapped(m), ("Page still has mappings"));
KASSERT((m->a.flags & PGA_WRITEABLE) == 0, ("Page still writable")); KASSERT((m->a.flags & PGA_WRITEABLE) == 0, ("Page still writable"));
▲ Show 20 Lines • Show All 216 Lines • ▼ Show 20 Lines
} }
static boolean_t static boolean_t
moea64_query_bit(vm_page_t m, uint64_t ptebit) moea64_query_bit(vm_page_t m, uint64_t ptebit)
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
int64_t ret; int64_t ret;
boolean_t rv; boolean_t rv;
vm_page_t sp;
/* /*
* See if this bit is stored in the page already. * See if this bit is stored in the page already.
*
* For superpages, the bit is stored in the first vm page.
*/ */
if (m->md.mdpg_attrs & ptebit) if ((m->md.mdpg_attrs & ptebit) != 0 ||
((sp = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(m) & ~SP_MASK)) != NULL &&
(sp->md.mdpg_attrs & (ptebit | MDPG_ATTR_SP)) ==
(ptebit | MDPG_ATTR_SP))
)
return (TRUE); return (TRUE);
/* /*
* Examine each PTE. Sync so that any pending REF/CHG bits are * Examine each PTE. Sync so that any pending REF/CHG bits are
* flushed to the PTEs. * flushed to the PTEs.
*/ */
rv = FALSE; rv = FALSE;
powerpc_sync(); powerpc_sync();
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
if (PVO_IS_SP(pvo)) {
ret = moea64_sp_query(pvo, ptebit);
/*
* If SP was not demoted, check its REF/CHG bits here.
*/
if (ret != -1) {
if ((ret & ptebit) != 0) {
rv = TRUE;
break;
}
continue;
}
/* else, fallthrough */
}
ret = 0; ret = 0;
/* /*
* See if this pvo has a valid PTE. if so, fetch the * See if this pvo has a valid PTE. if so, fetch the
* REF/CHG bits from the valid PTE. If the appropriate * REF/CHG bits from the valid PTE. If the appropriate
* ptebit is set, return success. * ptebit is set, return success.
*/ */
PMAP_LOCK(pvo->pvo_pmap); PMAP_LOCK(pvo->pvo_pmap);
Show All 29 Linesmoea64_clear_bit(vm_page_t m, u_int64_t ptebit)
powerpc_sync(); powerpc_sync();
/* /*
* For each pvo entry, clear the pte's ptebit. * For each pvo entry, clear the pte's ptebit.
*/ */
count = 0; count = 0;
PV_PAGE_LOCK(m); PV_PAGE_LOCK(m);
LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
if (PVO_IS_SP(pvo)) {
if ((ret = moea64_sp_clear(pvo, m, ptebit)) != -1) {
count += ret;
continue;
}
}
ret = 0; ret = 0;
PMAP_LOCK(pvo->pvo_pmap); PMAP_LOCK(pvo->pvo_pmap);
if (!(pvo->pvo_vaddr & PVO_DEAD)) if (!(pvo->pvo_vaddr & PVO_DEAD))
ret = moea64_pte_clear(pvo, ptebit); ret = moea64_pte_clear(pvo, ptebit);
PMAP_UNLOCK(pvo->pvo_pmap); PMAP_UNLOCK(pvo->pvo_pmap);
if (ret > 0 && (ret & ptebit)) if (ret > 0 && (ret & ptebit))
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines
{ {
*va = (void *)(uintptr_t)pa; *va = (void *)(uintptr_t)pa;
} }
extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1]; extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1];
void void
moea64_scan_init() moea64_scan_init()
jhibbitsUnsubmitted
Done
Inline Actions

I think adding 'void' is only necessary for prototypes, not for definitions.

jhibbits: I think adding 'void' is only necessary for prototypes, not for definitions.
luporlAuthorUnsubmitted
Done
Inline Actions

Right, I always add 'void' to functions with no arguments, but it's not really necessary in definitions, so I'll remove it.

luporl: Right, I always add 'void' to functions with no arguments, but it's not really necessary in…
{ {
struct pvo_entry *pvo; struct pvo_entry *pvo;
vm_offset_t va; vm_offset_t va;
int i; int i;
if (!do_minidump) { if (!do_minidump) {
/* Initialize phys. segments for dumpsys(). */ /* Initialize phys. segments for dumpsys(). */
memset(&dump_map, 0, sizeof(dump_map)); memset(&dump_map, 0, sizeof(dump_map));
▲ Show 20 Lines • Show All 258 Lines • ▼ Show 20 Lines
DEFINE_OEA64_IFUNC(int64_t, pte_replace, (struct pvo_entry *, int), DEFINE_OEA64_IFUNC(int64_t, pte_replace, (struct pvo_entry *, int),
moea64_pte_replace_default) moea64_pte_replace_default)
DEFINE_OEA64_IFUNC(int64_t, pte_insert, (struct pvo_entry *), moea64_null_method) DEFINE_OEA64_IFUNC(int64_t, pte_insert, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_unset, (struct pvo_entry *), moea64_null_method) DEFINE_OEA64_IFUNC(int64_t, pte_unset, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_clear, (struct pvo_entry *, uint64_t), DEFINE_OEA64_IFUNC(int64_t, pte_clear, (struct pvo_entry *, uint64_t),
moea64_null_method) moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_synch, (struct pvo_entry *), moea64_null_method) DEFINE_OEA64_IFUNC(int64_t, pte_synch, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_insert_sp, (struct pvo_entry *), moea64_null_method)
DEFINE_OEA64_IFUNC(int64_t, pte_unset_sp, (struct pvo_entry *), moea64_null_method)
jhibbitsUnsubmitted
Done
Inline Actions

Unless there's a statistically significant performance penalty associated with it, this can be unconditional.

jhibbits: Unless there's a statistically significant performance penalty associated with it, this can be…
luporlAuthorUnsubmitted
Done
Inline Actions

Right, buildworld times showed a 1% difference only, when sp_enabled=0, that doesn't seem statistically significant.

luporl: Right, buildworld times showed a 1% difference only, when sp_enabled=0, that doesn't seem…
DEFINE_OEA64_IFUNC(int64_t, pte_replace_sp, (struct pvo_entry *), moea64_null_method)
/* Superpage functions */
/* MMU interface */
static bool
moea64_ps_enabled(pmap_t pmap)
{
return (superpages_enabled);
}
static void
moea64_align_superpage(vm_object_t object, vm_ooffset_t offset,
vm_offset_t *addr, vm_size_t size)
{
vm_offset_t sp_offset;
if (size < SP_SIZE)
return;
CTR4(KTR_PMAP, "%s: offs=%#jx, addr=%p, size=%#jx",
__func__, (uintmax_t)offset, addr, (uintmax_t)size);
if (object != NULL && (object->flags & OBJ_COLORED) != 0)
offset += ptoa(object->pg_color);
sp_offset = offset & SP_MASK;
if (size - ((SP_SIZE - sp_offset) & SP_MASK) < SP_SIZE ||
(*addr & SP_MASK) == sp_offset)
return;
if ((*addr & SP_MASK) < sp_offset)
*addr = (*addr & ~SP_MASK) + sp_offset;
else
*addr = ((*addr + SP_MASK) & ~SP_MASK) + sp_offset;
}
/* Helpers */
static __inline void
moea64_pvo_cleanup(struct pvo_dlist *tofree)
{
struct pvo_entry *pvo;
/* clean up */
while (!SLIST_EMPTY(tofree)) {
pvo = SLIST_FIRST(tofree);
SLIST_REMOVE_HEAD(tofree, pvo_dlink);
if (pvo->pvo_vaddr & PVO_DEAD)
moea64_pvo_remove_from_page(pvo);
free_pvo_entry(pvo);
}
}
static __inline uint16_t
pvo_to_vmpage_flags(struct pvo_entry *pvo)
{
uint16_t flags;
flags = 0;
if ((pvo->pvo_pte.prot & VM_PROT_WRITE) != 0)
flags |= PGA_WRITEABLE;
if ((pvo->pvo_pte.prot & VM_PROT_EXECUTE) != 0)
flags |= PGA_EXECUTABLE;
return (flags);
}
/*
* Check if the given pvo and its superpage are in sva-eva range.
*/
static __inline bool
moea64_sp_pvo_in_range(struct pvo_entry *pvo, vm_offset_t sva, vm_offset_t eva)
{
vm_offset_t spva;
spva = PVO_VADDR(pvo) & ~SP_MASK;
if (spva >= sva && spva + SP_SIZE <= eva) {
/*
* Because this function is intended to be called from loops
* that iterate over ordered pvo entries, if the condition
* above is true then the pvo must be the first of its
* superpage.
*/
KASSERT(PVO_VADDR(pvo) == spva,
("%s: unexpected unaligned superpage pvo", __func__));
return (true);
}
return (false);
}
/*
* Update vm about the REF/CHG bits if the superpage is managed and
* has (or had) write access.
*/
static void
moea64_sp_refchg_process(struct pvo_entry *sp, vm_page_t m,
int64_t sp_refchg, vm_prot_t prot)
{
vm_page_t m_end;
int64_t refchg;
if ((sp->pvo_vaddr & PVO_MANAGED) != 0 && (prot & VM_PROT_WRITE) != 0) {
for (m_end = &m[SP_PAGES]; m < m_end; m++) {
refchg = sp_refchg |
atomic_readandclear_32(&m->md.mdpg_attrs);
if (refchg & LPTE_CHG)
vm_page_dirty(m);
if (refchg & LPTE_REF)
vm_page_aflag_set(m, PGA_REFERENCED);
}
}
}
/* Superpage ops */
static int
moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot, u_int flags, int8_t psind)
{
struct pvo_entry *pvo, **pvos;
struct pvo_head *pvo_head;
vm_offset_t sva;
vm_page_t sm;
vm_paddr_t pa;
bool sync;
struct pvo_dlist tofree;
int error, i;
uint16_t aflags;
KASSERT((va & SP_MASK) == 0, ("%s: va %#jx unaligned",
__func__, (uintmax_t)va));
KASSERT(psind == 1, ("%s: invalid psind: %d", __func__, psind));
KASSERT(m->psind == 1, ("%s: invalid m->psind: %d",
__func__, m->psind));
KASSERT(pmap != kernel_pmap,
("%s: function called with kernel pmap", __func__));
CTR5(KTR_PMAP, "%s: va=%#jx, pa=%#jx, prot=%#x, flags=%#x, psind=1",
__func__, (uintmax_t)va, (uintmax_t)VM_PAGE_TO_PHYS(m),
prot, flags);
SLIST_INIT(&tofree);
sva = va;
sm = m;
pa = VM_PAGE_TO_PHYS(sm);
/* Try to allocate all PVOs first, to make failure handling easier. */
pvos = malloc(SP_PAGES * sizeof(struct pvo_entry *), M_TEMP, M_NOWAIT);
if (pvos == NULL) {
CTR1(KTR_PMAP, "%s: failed to alloc pvo array", __func__);
return (KERN_RESOURCE_SHORTAGE);
}
for (i = 0; i < SP_PAGES; i++) {
pvos[i] = alloc_pvo_entry(0);
if (pvos[i] == NULL) {
CTR1(KTR_PMAP, "%s: failed to alloc pvo", __func__);
for (i = i - 1; i >= 0; i--)
free_pvo_entry(pvos[i]);
free(pvos, M_TEMP);
return (KERN_RESOURCE_SHORTAGE);
}
}
PV_PAGE_LOCK(sm);
PMAP_LOCK(pmap);
/* Note: moea64_remove_locked() also clears cached REF/CHG bits. */
moea64_remove_locked(pmap, va, va + SP_SIZE, &tofree);
/* Enter pages */
for (i = 0; i < SP_PAGES;
i++, va += PAGE_SIZE, pa += PAGE_SIZE, m++) {
pvo = pvos[i];
pvo->pvo_pte.prot = prot;
pvo->pvo_pte.pa = (pa & ~LPTE_LP_MASK) | LPTE_LP_4K_16M |
moea64_calc_wimg(pa, pmap_page_get_memattr(m));
if ((flags & PMAP_ENTER_WIRED) != 0)
pvo->pvo_vaddr |= PVO_WIRED;
pvo->pvo_vaddr |= PVO_LARGE;
if ((m->oflags & VPO_UNMANAGED) != 0)
pvo_head = NULL;
else {
pvo_head = &m->md.mdpg_pvoh;
pvo->pvo_vaddr |= PVO_MANAGED;
}
init_pvo_entry(pvo, pmap, va);
error = moea64_pvo_enter(pvo, pvo_head, NULL);
/*
* All superpage PVOs were previously removed, so no errors
* should occur while inserting the new ones.
*/
KASSERT(error == 0, ("%s: unexpected error "
"when inserting superpage PVO: %d",
__func__, error));
}
PMAP_UNLOCK(pmap);
PV_PAGE_UNLOCK(sm);
sync = (sm->a.flags & PGA_EXECUTABLE) == 0;
/* Note: moea64_pvo_cleanup() also clears page prot. flags. */
moea64_pvo_cleanup(&tofree);
pvo = pvos[0];
/* Set vm page flags */
aflags = pvo_to_vmpage_flags(pvo);
if (aflags != 0)
for (m = sm; m < &sm[SP_PAGES]; m++)
vm_page_aflag_set(m, aflags);
/*
* Flush the page from the instruction cache if this page is
* mapped executable and cacheable.
*/
if (sync && (pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)
moea64_syncicache(pmap, sva, VM_PAGE_TO_PHYS(sm), SP_SIZE);
atomic_add_long(&sp_mappings, 1);
CTR3(KTR_PMAP, "%s: SP success for va %#jx in pmap %p",
__func__, (uintmax_t)sva, pmap);
free(pvos, M_TEMP);
return (KERN_SUCCESS);
}
static void
moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m)
{
struct pvo_entry *first, *pvo;
vm_paddr_t pa, pa_end;
vm_offset_t sva, va_end;
int64_t sp_refchg;
/* This CTR may generate a lot of output. */
markjUnsubmitted
Done
Inline Actions

In other implementations, this check is in the caller. Why not here?

I would check moea64_ps_enabled() earlier, so as not to pessimize systems where superpages are disabled, since vm_reserv_level_iffullpop() is more expensive.

markj: In other implementations, this check is in the caller. Why not here? I would check…
luporlAuthorUnsubmitted
Done
Inline Actions

No special reason. I'll move the check to the caller.

Nice tip, I'll check moea64_ps_enabled() earlier.

luporl: No special reason. I'll move the check to the caller. Nice tip, I'll check moea64_ps_enabled()…
/* CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)va); */
va &= ~SP_MASK;
sva = va;
/* Get superpage */
m = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(m) & ~SP_MASK);
PV_PAGE_LOCK(m);
PMAP_LOCK(pmap);
/*
* Check if all pages meet promotion criteria.
*
* XXX In some cases the loop below may be executed for each or most
* of the entered pages of a superpage, which can be expensive
* (although it was not profiled) and need some optimization.
*
* Some cases where this seems to happen are:
* - When a superpage is first entered read-only and later becomes
* read-write.
* - When some of the superpage's virtual addresses map to previously
* wired/cached pages while others map to pages allocated from a
* different physical address range. A common scenario where this
* happens is when mmap'ing a file that is already present in FS
* block cache and doesn't fill a superpage.
*/
first = pvo = moea64_pvo_find_va(pmap, sva);
for (pa = VM_PAGE_TO_PHYS(m), pa_end = pa + SP_SIZE;
pa < pa_end; pa += PAGE_SIZE, va += PAGE_SIZE) {
if (pvo == NULL || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR3(KTR_PMAP,
"%s: NULL or dead PVO: pmap=%p, va=%#jx",
__func__, pmap, (uintmax_t)va);
goto error;
}
if (PVO_PADDR(pvo) != pa) {
CTR5(KTR_PMAP, "%s: PAs don't match: "
"pmap=%p, va=%#jx, pvo_pa=%#jx, exp_pa=%#jx",
__func__, pmap, (uintmax_t)va,
(uintmax_t)PVO_PADDR(pvo), (uintmax_t)pa);
atomic_add_long(&sp_p_fail_pa, 1);
goto error;
}
if ((first->pvo_vaddr & PVO_FLAGS_PROMOTE) !=
(pvo->pvo_vaddr & PVO_FLAGS_PROMOTE)) {
CTR5(KTR_PMAP, "%s: PVO flags don't match: "
"pmap=%p, va=%#jx, pvo_flags=%#jx, exp_flags=%#jx",
__func__, pmap, (uintmax_t)va,
(uintmax_t)(pvo->pvo_vaddr & PVO_FLAGS_PROMOTE),
(uintmax_t)(first->pvo_vaddr & PVO_FLAGS_PROMOTE));
atomic_add_long(&sp_p_fail_flags, 1);
goto error;
}
if (first->pvo_pte.prot != pvo->pvo_pte.prot) {
CTR5(KTR_PMAP, "%s: PVO protections don't match: "
"pmap=%p, va=%#jx, pvo_prot=%#x, exp_prot=%#x",
__func__, pmap, (uintmax_t)va,
pvo->pvo_pte.prot, first->pvo_pte.prot);
atomic_add_long(&sp_p_fail_prot, 1);
goto error;
}
if ((first->pvo_pte.pa & LPTE_WIMG) !=
(pvo->pvo_pte.pa & LPTE_WIMG)) {
CTR5(KTR_PMAP, "%s: WIMG bits don't match: "
"pmap=%p, va=%#jx, pvo_wimg=%#jx, exp_wimg=%#jx",
__func__, pmap, (uintmax_t)va,
(uintmax_t)(pvo->pvo_pte.pa & LPTE_WIMG),
(uintmax_t)(first->pvo_pte.pa & LPTE_WIMG));
atomic_add_long(&sp_p_fail_wimg, 1);
goto error;
}
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);
}
/* All OK, promote. */
/*
* Handle superpage REF/CHG bits. If REF or CHG is set in
* any page, then it must be set in the superpage.
*
* Instead of querying each page, we take advantage of two facts:
* 1- If a page is being promoted, it was referenced.
* 2- If promoted pages are writable, they were modified.
*/
sp_refchg = LPTE_REF |
((first->pvo_pte.prot & VM_PROT_WRITE) != 0 ? LPTE_CHG : 0);
/* Promote pages */
for (pvo = first, va_end = PVO_VADDR(pvo) + SP_SIZE;
pvo != NULL && PVO_VADDR(pvo) < va_end;
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
pvo->pvo_pte.pa &= ~LPTE_LP_MASK;
pvo->pvo_pte.pa |= LPTE_LP_4K_16M;
pvo->pvo_vaddr |= PVO_LARGE;
}
moea64_pte_replace_sp(first);
/* Send REF/CHG bits to VM */
moea64_sp_refchg_process(first, m, sp_refchg, first->pvo_pte.prot);
markjUnsubmitted
Done
Inline Actions

Ouch, won't this be quite expensive? Do you actually need to invalidate the small mappings? This is one area where the various superpage implementations differ. ARM has an annoying requirement that a physical page not be aliased by both 4KB and large mappings, so promotions require a "break-before-make", in which the 4KB mappings are cleared (by clearing the L2 entry), TLBs are invalidated, and then the large mapping is created. See pmap_update_entry() on arm64 for example. amd64, i386 and riscv do not have this requirement and therefore do not perform TLB invalidation during promotion. I am not sure about the requirements on POWER, but it would be better to avoid or batch these invalidations if possible.

markj: Ouch, won't this be quite expensive? Do you actually need to invalidate the small mappings?
bdragonUnsubmitted
Done
Inline Actions

(Note: IBM differentiates between "Effective Address" and "Virtual Address" -- VA is an (up to) 78 bit address that is generally unique (although I suppose if two processes were to share the same VSID at any point, they could theoretically have 256M or 1T chunks of shared address space), EA is the normal 64 bit address that is the address that is stored in pointers and such)

There is no L2 entry, HPT is not a radix table, it's a segment + hash scheme, where the EA is converted into a VA by using the (32-entry) SLB to look up a VSID (or taking a segment fault, or using a process scoped segment table if we were to set one up), and then using that VSID and the page bits of the EA (as per the L,LP bits in the SLBE, for FreeBSD we can assume 4k base pages) to get the VPN, and then using that VPN as a hash key into the pagetable to identify the address of the correct PTEG to search. (As well as possibly doing a secondary search to come up with a second PTEG to scan)

The 8 records of the PTEG (and 8 of the secondary if enabled) are then matched against the properties of the SLBE to look for a match that satisfies the requirements.

It is vitally important that there not be overlapping matches in VA space because that can lead to multihits in the TLB if two different page sizes are observed at any point. This causes a machine check, which brings down the whole machine.

So yes, it is necessary to do a bunch of invalidation as far as I know.

See section 5.7 in Book III of the ISA. (Preferably 3.0) -- 5.7.7 is the main description of address translation.

FWIW, I wonder if we could optimize the pvo_entry structure by not tracking the entire address of the PTE slot and instead reconstructing it by running the hash function given the VSID.

bdragon: (Note: IBM differentiates between "Effective Address" and "Virtual Address" -- VA is an (up to)…
bdragonUnsubmitted
Done
Inline Actions

(that is, only tracking the offset into the PTEG and calculating the PTEG base address when we need it)

bdragon: (that is, only tracking the offset into the PTEG and calculating the PTEG base address when we…
bdragonUnsubmitted
Done
Inline Actions

Additional note: the main benefit of large pages on HPT is that the *tlb hit rate* goes up. You still need a PTE for every 4k of it if you want to avoid page faults, but a match will get cached in the TLB / ERATs as covering the entire range.

bdragon: Additional note: the main benefit of large pages on HPT is that the *tlb hit rate* goes up. You…
luporlAuthorUnsubmitted
Done
Inline Actions

As bdragon explained, we really need to invalidate all mappings.
We don't have a batched invalidation mechanism on PPC yet, but it could be a nice addition.

I've tried to optimize the PVO tree by using only one PVO to track a superpage, but I started facing some issues and gave up at that moment, to make everything work correctly first, before trying more optimizations.
From what I remember, the main issue was caused by not saving the PVO_HID bit for each 4K PTE. IIUC, we can't guess what it will be for a given 4K range within a superpage, because it is set when we fail to insert the PTE in the primary hash table and use the second one instead. But maybe we can save only this bit for each PTE, in a PVO bitmap, and make the idea of using only one PVO for superpage work.

luporl: As bdragon explained, we really need to invalidate all mappings. We don't have a batched…
luporlAuthorUnsubmitted
Done
Inline Actions

The last diff implements batch PTE unset and insert. TLB invalidations are performed during PTE unset.

It would be possible to perform all TLB invalidations as a separate step, but it would require more work and I'm don't know how much it would improve performance, compared to the current loop that removes the PTE and invalidates its TLB entry, since on POWER9 (and probably on POWER8 too) TLBIE don't need locks.

luporl: The last diff implements batch PTE unset and insert. TLB invalidations are performed during PTE…
/* Use first page to cache REF/CHG bits */
atomic_set_32(&m->md.mdpg_attrs, sp_refchg | MDPG_ATTR_SP);
PMAP_UNLOCK(pmap);
PV_PAGE_UNLOCK(m);
atomic_add_long(&sp_mappings, 1);
atomic_add_long(&sp_promotions, 1);
CTR3(KTR_PMAP, "%s: success for va %#jx in pmap %p",
__func__, (uintmax_t)sva, pmap);
return;
error:
atomic_add_long(&sp_p_failures, 1);
PMAP_UNLOCK(pmap);
PV_PAGE_UNLOCK(m);
}
markjUnsubmitted
Done
Inline Actions

The return value is unused.

markj: The return value is unused.
luporlAuthorUnsubmitted
Done
Inline Actions

Right, I'll remove it.

luporl: Right, I'll remove it.
static void
moea64_sp_demote_aligned(struct pvo_entry *sp)
{
struct pvo_entry *pvo;
vm_offset_t va, va_end;
vm_paddr_t pa;
vm_page_t m;
pmap_t pmap;
int64_t refchg;
CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
pmap = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
pvo = sp;
/* Demote pages */
va = PVO_VADDR(pvo);
pa = PVO_PADDR(pvo);
m = PHYS_TO_VM_PAGE(pa);
for (pvo = sp, va_end = va + SP_SIZE;
pvo != NULL && PVO_VADDR(pvo) < va_end;
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo),
va += PAGE_SIZE, pa += PAGE_SIZE) {
KASSERT(pvo && PVO_VADDR(pvo) == va,
("%s: missing PVO for va %#jx", __func__, (uintmax_t)va));
pvo->pvo_vaddr &= ~PVO_LARGE;
pvo->pvo_pte.pa &= ~LPTE_RPGN;
pvo->pvo_pte.pa |= pa;
}
refchg = moea64_pte_replace_sp(sp);
/* Clear SP flag */
atomic_clear_32(&m->md.mdpg_attrs, MDPG_ATTR_SP);
/*
* Handle superpage REF/CHG bits. A bit set in the superpage
* means all pages should consider it set.
*/
moea64_sp_refchg_process(sp, m, refchg, sp->pvo_pte.prot);
atomic_add_long(&sp_demotions, 1);
CTR3(KTR_PMAP, "%s: success for va %#jx in pmap %p",
__func__, (uintmax_t)PVO_VADDR(sp), pmap);
}
static void
moea64_sp_demote(struct pvo_entry *pvo)
{
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
if ((PVO_VADDR(pvo) & SP_MASK) != 0) {
pvo = moea64_pvo_find_va(pvo->pvo_pmap,
PVO_VADDR(pvo) & ~SP_MASK);
KASSERT(pvo != NULL, ("%s: missing PVO for va %#jx",
__func__, (uintmax_t)(PVO_VADDR(pvo) & ~SP_MASK)));
}
moea64_sp_demote_aligned(pvo);
}
static struct pvo_entry *
moea64_sp_unwire(struct pvo_entry *sp)
{
struct pvo_entry *pvo, *prev;
vm_offset_t eva;
pmap_t pm;
int64_t ret, refchg;
CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
pm = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
prev = pvo, pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
panic("%s: pvo %p is missing PVO_WIRED",
__func__, pvo);
pvo->pvo_vaddr &= ~PVO_WIRED;
ret = moea64_pte_replace(pvo, 0 /* No invalidation */);
if (ret < 0)
refchg |= LPTE_CHG;
else
refchg |= ret;
pm->pm_stats.wired_count--;
}
/* Send REF/CHG bits to VM */
moea64_sp_refchg_process(sp, PHYS_TO_VM_PAGE(PVO_PADDR(sp)),
refchg, sp->pvo_pte.prot);
return (prev);
}
static struct pvo_entry *
moea64_sp_protect(struct pvo_entry *sp, vm_prot_t prot)
{
struct pvo_entry *pvo, *prev;
vm_offset_t eva;
pmap_t pm;
vm_page_t m, m_end;
int64_t ret, refchg;
vm_prot_t oldprot;
CTR3(KTR_PMAP, "%s: va=%#jx, prot=%x",
__func__, (uintmax_t)PVO_VADDR(sp), prot);
pm = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
oldprot = sp->pvo_pte.prot;
m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
KASSERT(m != NULL, ("%s: missing vm page for pa %#jx",
__func__, (uintmax_t)PVO_PADDR(sp)));
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
prev = pvo, pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
pvo->pvo_pte.prot = prot;
/*
* If the PVO is in the page table, update mapping
*/
ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
if (ret < 0)
refchg |= LPTE_CHG;
else
refchg |= ret;
}
/* Send REF/CHG bits to VM */
moea64_sp_refchg_process(sp, m, refchg, oldprot);
/* Handle pages that became executable */
if ((m->a.flags & PGA_EXECUTABLE) == 0 &&
(sp->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
if ((m->oflags & VPO_UNMANAGED) == 0)
for (m_end = &m[SP_PAGES]; m < m_end; m++)
vm_page_aflag_set(m, PGA_EXECUTABLE);
moea64_syncicache(pm, PVO_VADDR(sp), PVO_PADDR(sp), SP_SIZE);
}
return (prev);
}
static struct pvo_entry *
moea64_sp_remove(struct pvo_entry *sp, struct pvo_dlist *tofree)
{
struct pvo_entry *pvo, *tpvo;
vm_offset_t eva;
pmap_t pm;
CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
pm = sp->pvo_pmap;
PMAP_LOCK_ASSERT(pm, MA_OWNED);
eva = PVO_VADDR(sp) + SP_SIZE;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) {
tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo);
/*
* For locking reasons, remove this from the page table and
* pmap, but save delinking from the vm_page for a second
* pass
*/
moea64_pvo_remove_from_pmap(pvo);
SLIST_INSERT_HEAD(tofree, pvo, pvo_dlink);
}
/* Clear SP bit */
atomic_clear_32(&PHYS_TO_VM_PAGE(PVO_PADDR(sp))->md.mdpg_attrs,
MDPG_ATTR_SP);
return (tpvo);
}
static int64_t
moea64_sp_query_locked(struct pvo_entry *pvo, uint64_t ptebit)
{
int64_t refchg, ret;
vm_offset_t eva;
vm_page_t m;
pmap_t pmap;
struct pvo_entry *sp;
pmap = pvo->pvo_pmap;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/* Get first SP PVO */
if ((PVO_VADDR(pvo) & SP_MASK) != 0) {
sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~SP_MASK);
KASSERT(sp != NULL, ("%s: missing PVO for va %#jx",
__func__, (uintmax_t)(PVO_VADDR(pvo) & ~SP_MASK)));
} else
sp = pvo;
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
ret = moea64_pte_synch(pvo);
if (ret > 0) {
refchg |= ret & (LPTE_CHG | LPTE_REF);
if ((refchg & ptebit) != 0)
break;
}
}
/* Save results */
if (refchg != 0) {
m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
atomic_set_32(&m->md.mdpg_attrs, refchg | MDPG_ATTR_SP);
}
return (refchg);
}
static int64_t
moea64_sp_query(struct pvo_entry *pvo, uint64_t ptebit)
{
int64_t refchg;
pmap_t pmap;
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
/*
* Check if SP was demoted/removed before pmap lock was acquired.
*/
if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
__func__, (uintmax_t)PVO_PADDR(pvo));
PMAP_UNLOCK(pmap);
return (-1);
}
refchg = moea64_sp_query_locked(pvo, ptebit);
PMAP_UNLOCK(pmap);
CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",
__func__, (uintmax_t)PVO_VADDR(pvo),
(uintmax_t)PVO_PADDR(pvo), (uintmax_t)refchg);
return (refchg);
}
static int64_t
moea64_sp_pvo_clear(struct pvo_entry *pvo, uint64_t ptebit)
{
int64_t refchg, ret;
pmap_t pmap;
struct pvo_entry *sp;
vm_offset_t eva;
vm_page_t m;
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
/*
* Check if SP was demoted/removed before pmap lock was acquired.
*/
if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
__func__, (uintmax_t)PVO_PADDR(pvo));
PMAP_UNLOCK(pmap);
return (-1);
}
/* Get first SP PVO */
if ((PVO_VADDR(pvo) & SP_MASK) != 0) {
sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~SP_MASK);
KASSERT(sp != NULL, ("%s: missing PVO for va %#jx",
__func__, (uintmax_t)(PVO_VADDR(pvo) & ~SP_MASK)));
} else
sp = pvo;
eva = PVO_VADDR(sp) + SP_SIZE;
refchg = 0;
for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
ret = moea64_pte_clear(pvo, ptebit);
if (ret > 0)
refchg |= ret & (LPTE_CHG | LPTE_REF);
}
m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
atomic_clear_32(&m->md.mdpg_attrs, ptebit);
PMAP_UNLOCK(pmap);
CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",
__func__, (uintmax_t)PVO_VADDR(sp),
(uintmax_t)PVO_PADDR(sp), (uintmax_t)refchg);
return (refchg);
}
static int64_t
moea64_sp_clear(struct pvo_entry *pvo, vm_page_t m, uint64_t ptebit)
{
int64_t count, ret;
pmap_t pmap;
count = 0;
pmap = pvo->pvo_pmap;
/*
* Since this reference bit is shared by 4096 4KB pages, it
* should not be cleared every time it is tested. Apply a
* simple "hash" function on the physical page number, the
* virtual superpage number, and the pmap address to select
* one 4KB page out of the 4096 on which testing the
* reference bit will result in clearing that reference bit.
* This function is designed to avoid the selection of the
* same 4KB page for every 16MB page mapping.
*
* Always leave the reference bit of a wired mapping set, as
* the current state of its reference bit won't affect page
* replacement.
*/
if (ptebit == LPTE_REF && (((VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) ^
(PVO_VADDR(pvo) >> SP_SHIFT) ^ (uintptr_t)pmap) &
(SP_PAGES - 1)) == 0 && (pvo->pvo_vaddr & PVO_WIRED) == 0) {
if ((ret = moea64_sp_pvo_clear(pvo, ptebit)) == -1)
return (-1);
if ((ret & ptebit) != 0)
count++;
/*
* If this page was not selected by the hash function, then assume
* its REF bit was set.
*/
} else if (ptebit == LPTE_REF) {
count++;
/*
* To clear the CHG bit of a single SP page, first it must be demoted.
* But if no CHG bit is set, no bit clear and thus no SP demotion is
* needed.
*/
} else {
CTR4(KTR_PMAP, "%s: ptebit=%#jx, va=%#jx, pa=%#jx",
__func__, (uintmax_t)ptebit, (uintmax_t)PVO_VADDR(pvo),
(uintmax_t)PVO_PADDR(pvo));
PMAP_LOCK(pmap);
/*
* Make sure SP wasn't demoted/removed before pmap lock
* was acquired.
*/
if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
__func__, (uintmax_t)PVO_PADDR(pvo));
PMAP_UNLOCK(pmap);
return (-1);
}
ret = moea64_sp_query_locked(pvo, ptebit);
if ((ret & ptebit) != 0)
count++;
else {
PMAP_UNLOCK(pmap);
return (0);
}
moea64_sp_demote(pvo);
moea64_pte_clear(pvo, ptebit);
/*
* Write protect the mapping to a single page so that a
* subsequent write access may repromote.
*/
if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
moea64_pvo_protect(pmap, pvo,
pvo->pvo_pte.prot & ~VM_PROT_WRITE);
PMAP_UNLOCK(pmap);
}
return (count);
}