Differential D24993 Diff 72280 sys/powerpc/pseries/mmu_phyp.c

Changeset View

Standalone View

sys/powerpc/pseries/mmu_phyp.c

Show First 20 Lines • Show All 48 Lines • ▼ Show 20 Lines
#include <vm/vm_map.h>		#include <vm/vm_map.h>
#include <vm/vm_object.h>		#include <vm/vm_object.h>
#include <vm/vm_extern.h>		#include <vm/vm_extern.h>
#include <vm/vm_pageout.h>		#include <vm/vm_pageout.h>
#include <vm/uma.h>		#include <vm/uma.h>

#include <powerpc/aim/mmu_oea64.h>		#include <powerpc/aim/mmu_oea64.h>

#include "mmu_if.h"
#include "moea64_if.h"

#include "phyp-hvcall.h"		#include "phyp-hvcall.h"

#define MMU_PHYP_DEBUG 0		#define MMU_PHYP_DEBUG 0
#define MMU_PHYP_ID "mmu_phyp: "		#define MMU_PHYP_ID "mmu_phyp: "
#if MMU_PHYP_DEBUG		#if MMU_PHYP_DEBUG
#define dprintf(fmt, ...) printf(fmt, ## __VA_ARGS__)		#define dprintf(fmt, ...) printf(fmt, ## __VA_ARGS__)
#define dprintf0(fmt, ...) dprintf(MMU_PHYP_ID fmt, ## __VA_ARGS__)		#define dprintf0(fmt, ...) dprintf(MMU_PHYP_ID fmt, ## __VA_ARGS__)
#else		#else
#define dprintf(fmt, args...) do { ; } while(0)		#define dprintf(fmt, args...) do { ; } while(0)
#define dprintf0(fmt, args...) do { ; } while(0)		#define dprintf0(fmt, args...) do { ; } while(0)
#endif		#endif

static struct rmlock mphyp_eviction_lock;		static struct rmlock mphyp_eviction_lock;

/*		/*
* Kernel MMU interface		* Kernel MMU interface
*/		*/

static void mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart,		static void mphyp_install(void);
		static void mphyp_bootstrap(vm_offset_t kernelstart,
vm_offset_t kernelend);		vm_offset_t kernelend);
static void mphyp_cpu_bootstrap(mmu_t mmup, int ap);		static void mphyp_cpu_bootstrap(int ap);
static void mphyp_dump_pmap(mmu_t mmu, void ctx, void *buf,		static void mphyp_dump_pmap(void ctx, void *buf,
u_long *nbytes);		u_long *nbytes);
static int64_t mphyp_pte_synch(mmu_t, struct pvo_entry *pvo);		static int64_t mphyp_pte_synch(struct pvo_entry *pvo);
static int64_t mphyp_pte_clear(mmu_t, struct pvo_entry *pvo, uint64_t ptebit);		static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit);
static int64_t mphyp_pte_unset(mmu_t, struct pvo_entry *pvo);		static int64_t mphyp_pte_unset(struct pvo_entry *pvo);
static int mphyp_pte_insert(mmu_t, struct pvo_entry *pvo);		static int64_t mphyp_pte_insert(struct pvo_entry *pvo);

static mmu_method_t mphyp_methods[] = {		static struct pmap_funcs mphyp_methods = {
MMUMETHOD(mmu_bootstrap, mphyp_bootstrap),		.install = mphyp_install,
MMUMETHOD(mmu_cpu_bootstrap, mphyp_cpu_bootstrap),		.bootstrap = mphyp_bootstrap,
MMUMETHOD(mmu_dump_pmap, mphyp_dump_pmap),		.cpu_bootstrap = mphyp_cpu_bootstrap,
		.dumpsys_dump_pmap = mphyp_dump_pmap,
		};

MMUMETHOD(moea64_pte_synch, mphyp_pte_synch),		static struct moea64_funcs mmu_phyp_funcs = {
MMUMETHOD(moea64_pte_clear, mphyp_pte_clear),		.pte_synch = mphyp_pte_synch,
MMUMETHOD(moea64_pte_unset, mphyp_pte_unset),		.pte_clear = mphyp_pte_clear,
MMUMETHOD(moea64_pte_insert, mphyp_pte_insert),		.pte_unset = mphyp_pte_unset,
		.pte_insert = mphyp_pte_insert,
/* XXX: pmap_copy_page, pmap_init_page with H_PAGE_INIT */

{ 0, 0 }
};		};

MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu);		MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu);

static int brokenkvm = 0;		static int brokenkvm = 0;

static void		static void
print_kvm_bug_warning(void *data)		print_kvm_bug_warning(void *data)
{		{

if (brokenkvm)		if (brokenkvm)
printf("WARNING: Running on a broken hypervisor that does "		printf("WARNING: Running on a broken hypervisor that does "
"not support mandatory H_CLEAR_MOD and H_CLEAR_REF "		"not support mandatory H_CLEAR_MOD and H_CLEAR_REF "
"hypercalls. Performance will be suboptimal.\n");		"hypercalls. Performance will be suboptimal.\n");
}		}

SYSINIT(kvmbugwarn1, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1,		SYSINIT(kvmbugwarn1, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1,
print_kvm_bug_warning, NULL);		print_kvm_bug_warning, NULL);
SYSINIT(kvmbugwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1, print_kvm_bug_warning,		SYSINIT(kvmbugwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1, print_kvm_bug_warning,
NULL);		NULL);

static void		static void
mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)		mphyp_install()
{		{

		moea64_ops = &mmu_phyp_funcs;
		}

		static void
		mphyp_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
		{
uint64_t final_pteg_count = 0;		uint64_t final_pteg_count = 0;
char buf[8];		char buf[8];
uint32_t prop[2];		uint32_t prop[2];
uint32_t nptlp, shift = 0, slb_encoding = 0;		uint32_t nptlp, shift = 0, slb_encoding = 0;
uint32_t lp_size, lp_encoding;		uint32_t lp_size, lp_encoding;
struct lpte old;		struct lpte old;
uint64_t vsid;		uint64_t vsid;
phandle_t dev, node, root;		phandle_t dev, node, root;
int idx, len, res;		int idx, len, res;

rm_init(&mphyp_eviction_lock, "pte eviction");		rm_init(&mphyp_eviction_lock, "pte eviction");

moea64_early_bootstrap(mmup, kernelstart, kernelend);		moea64_early_bootstrap(kernelstart, kernelend);

root = OF_peer(0);		root = OF_peer(0);

dev = OF_child(root);		dev = OF_child(root);
while (dev != 0) {		while (dev != 0) {
res = OF_getprop(dev, "name", buf, sizeof(buf));		res = OF_getprop(dev, "name", buf, sizeof(buf));
if (res > 0 && strcmp(buf, "cpus") == 0)		if (res > 0 && strcmp(buf, "cpus") == 0)
break;		break;
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Lines	if (len > 0) {
moea64_large_page_shift = 0;		moea64_large_page_shift = 0;
moea64_large_page_mask = 0;		moea64_large_page_mask = 0;
hw_direct_map = 0;		hw_direct_map = 0;
printf(MMU_PHYP_ID		printf(MMU_PHYP_ID
"Support for hugepages not found\n");		"Support for hugepages not found\n");
}		}
}		}

moea64_mid_bootstrap(mmup, kernelstart, kernelend);		moea64_mid_bootstrap(kernelstart, kernelend);
moea64_late_bootstrap(mmup, kernelstart, kernelend);		moea64_late_bootstrap(kernelstart, kernelend);

/* Test for broken versions of KVM that don't conform to the spec */		/* Test for broken versions of KVM that don't conform to the spec */
if (phyp_hcall(H_CLEAR_MOD, 0, 0) == H_FUNCTION)		if (phyp_hcall(H_CLEAR_MOD, 0, 0) == H_FUNCTION)
brokenkvm = 1;		brokenkvm = 1;
}		}

static void		static void
mphyp_cpu_bootstrap(mmu_t mmup, int ap)		mphyp_cpu_bootstrap(int ap)
{		{
struct slb *slb = PCPU_GET(aim.slb);		struct slb *slb = PCPU_GET(aim.slb);
register_t seg0;		register_t seg0;
int i;		int i;

/*		/*
* Install kernel SLB entries		* Install kernel SLB entries
*/		*/

__asm __volatile ("slbia");		__asm __volatile ("slbia");
__asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));		__asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
for (i = 0; i < 64; i++) {		for (i = 0; i < 64; i++) {
if (!(slb[i].slbe & SLBE_VALID))		if (!(slb[i].slbe & SLBE_VALID))
continue;		continue;

__asm __volatile ("slbmte %0, %1" ::		__asm __volatile ("slbmte %0, %1" ::
"r"(slb[i].slbv), "r"(slb[i].slbe));		"r"(slb[i].slbv), "r"(slb[i].slbe));
}		}
}		}

static int64_t		static int64_t
mphyp_pte_synch(mmu_t mmu, struct pvo_entry *pvo)		mphyp_pte_synch(struct pvo_entry *pvo)
{		{
struct lpte pte;		struct lpte pte;
uint64_t junk;		uint64_t junk;

__asm __volatile("ptesync");		__asm __volatile("ptesync");
phyp_pft_hcall(H_READ, 0, pvo->pvo_pte.slot, 0, 0, &pte.pte_hi,		phyp_pft_hcall(H_READ, 0, pvo->pvo_pte.slot, 0, 0, &pte.pte_hi,
&pte.pte_lo, &junk);		&pte.pte_lo, &junk);
if ((pte.pte_hi & LPTE_AVPN_MASK) !=		if ((pte.pte_hi & LPTE_AVPN_MASK) !=
((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) &		((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) &
LPTE_AVPN_MASK))		LPTE_AVPN_MASK))
return (-1);		return (-1);
if (!(pte.pte_hi & LPTE_VALID))		if (!(pte.pte_hi & LPTE_VALID))
return (-1);		return (-1);

return (pte.pte_lo & (LPTE_CHG \| LPTE_REF));		return (pte.pte_lo & (LPTE_CHG \| LPTE_REF));
}		}

static int64_t		static int64_t
mphyp_pte_clear(mmu_t mmu, struct pvo_entry *pvo, uint64_t ptebit)		mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit)
{		{
struct rm_priotracker track;		struct rm_priotracker track;
int64_t refchg;		int64_t refchg;
uint64_t ptelo, junk;		uint64_t ptelo, junk;
int err;		int err;

/*		/*
* This involves two steps (synch and clear) so we need the entry		* This involves two steps (synch and clear) so we need the entry
* not to change in the middle. We are protected against deliberate		* not to change in the middle. We are protected against deliberate
* unset by virtue of holding the pmap lock. Protection against		* unset by virtue of holding the pmap lock. Protection against
* incidental unset (page table eviction) comes from holding the		* incidental unset (page table eviction) comes from holding the
* shared eviction lock.		* shared eviction lock.
*/		*/
PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);		PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
rm_rlock(&mphyp_eviction_lock, &track);		rm_rlock(&mphyp_eviction_lock, &track);

refchg = mphyp_pte_synch(mmu, pvo);		refchg = mphyp_pte_synch(pvo);
if (refchg < 0) {		if (refchg < 0) {
rm_runlock(&mphyp_eviction_lock, &track);		rm_runlock(&mphyp_eviction_lock, &track);
return (refchg);		return (refchg);
}		}

if (brokenkvm) {		if (brokenkvm) {
/*		/*
* No way to clear either bit, which is total madness.		* No way to clear either bit, which is total madness.
Show All 20 Lines	mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit)
}		}

rm_runlock(&mphyp_eviction_lock, &track);		rm_runlock(&mphyp_eviction_lock, &track);

return (refchg);		return (refchg);
}		}

static int64_t		static int64_t
mphyp_pte_unset(mmu_t mmu, struct pvo_entry *pvo)		mphyp_pte_unset(struct pvo_entry *pvo)
{		{
struct lpte pte;		struct lpte pte;
uint64_t junk;		uint64_t junk;
int err;		int err;

PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);		PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);

moea64_pte_from_pvo(pvo, &pte);		moea64_pte_from_pvo(pvo, &pte);
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	mphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict)
if (k == -1)		if (k == -1)
return (k);		return (k);

phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi,		phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi,
&to_evict->pte_lo, &junk);		&to_evict->pte_lo, &junk);
return (k);		return (k);
}		}

static int		static int64_t
mphyp_pte_insert(mmu_t mmu, struct pvo_entry *pvo)		mphyp_pte_insert(struct pvo_entry *pvo)
{		{
struct rm_priotracker track;		struct rm_priotracker track;
int64_t result;		int64_t result;
struct lpte evicted, pte;		struct lpte evicted, pte;
uint64_t index, junk, lastptelo;		uint64_t index, junk, lastptelo;

PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);		PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);

▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines	mphyp_pte_insert(struct pvo_entry *pvo)
if (result == H_SUCCESS)		if (result == H_SUCCESS)
return (0);		return (0);

panic("Page replacement error: %ld", result);		panic("Page replacement error: %ld", result);
return (result);		return (result);
}		}

static void *		static void *
mphyp_dump_pmap(mmu_t mmu, void ctx, void buf, u_long *nbytes)		mphyp_dump_pmap(void ctx, void buf, u_long *nbytes)
{		{
struct dump_context *dctx;		struct dump_context *dctx;
struct lpte p, *pbuf;		struct lpte p, *pbuf;
int bufidx;		int bufidx;
uint64_t junk;		uint64_t junk;
u_long ptex, ptex_end;		u_long ptex, ptex_end;

dctx = (struct dump_context *)ctx;		dctx = (struct dump_context *)ctx;
Show All 19 Lines