Differential D25237 Diff 76890 sys/powerpc/pseries/mmu_phyp.c

Changeset View

Standalone View

sys/powerpc/pseries/mmu_phyp.c

Show First 20 Lines • Show All 76 Lines • ▼ Show 20 Lines	static void mphyp_bootstrap(vm_offset_t kernelstart,
vm_offset_t kernelend);		vm_offset_t kernelend);
static void mphyp_cpu_bootstrap(int ap);		static void mphyp_cpu_bootstrap(int ap);
static void mphyp_dump_pmap(void ctx, void *buf,		static void mphyp_dump_pmap(void ctx, void *buf,
u_long *nbytes);		u_long *nbytes);
static int64_t mphyp_pte_synch(struct pvo_entry *pvo);		static int64_t mphyp_pte_synch(struct pvo_entry *pvo);
static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit);		static int64_t mphyp_pte_clear(struct pvo_entry *pvo, uint64_t ptebit);
static int64_t mphyp_pte_unset(struct pvo_entry *pvo);		static int64_t mphyp_pte_unset(struct pvo_entry *pvo);
static int64_t mphyp_pte_insert(struct pvo_entry *pvo);		static int64_t mphyp_pte_insert(struct pvo_entry *pvo);
		static int64_t mphyp_pte_unset_sp(struct pvo_entry *pvo);
		static int64_t mphyp_pte_insert_sp(struct pvo_entry *pvo);
		static int64_t mphyp_pte_replace_sp(struct pvo_entry *pvo);

static struct pmap_funcs mphyp_methods = {		static struct pmap_funcs mphyp_methods = {
.install = mphyp_install,		.install = mphyp_install,
.bootstrap = mphyp_bootstrap,		.bootstrap = mphyp_bootstrap,
.cpu_bootstrap = mphyp_cpu_bootstrap,		.cpu_bootstrap = mphyp_cpu_bootstrap,
.dumpsys_dump_pmap = mphyp_dump_pmap,		.dumpsys_dump_pmap = mphyp_dump_pmap,
};		};

static struct moea64_funcs mmu_phyp_funcs = {		static struct moea64_funcs mmu_phyp_funcs = {
.pte_synch = mphyp_pte_synch,		.pte_synch = mphyp_pte_synch,
.pte_clear = mphyp_pte_clear,		.pte_clear = mphyp_pte_clear,
.pte_unset = mphyp_pte_unset,		.pte_unset = mphyp_pte_unset,
.pte_insert = mphyp_pte_insert,		.pte_insert = mphyp_pte_insert,
		.pte_unset_sp = mphyp_pte_unset_sp,
		.pte_insert_sp = mphyp_pte_insert_sp,
		.pte_replace_sp = mphyp_pte_replace_sp,
};		};

MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu);		MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, oea64_mmu);

static int brokenkvm = 0;		static int brokenkvm = 0;

static void		static void
print_kvm_bug_warning(void *data)		print_kvm_bug_warning(void *data)
Show All 24 Lines	mphyp_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
char buf[8];		char buf[8];
uint32_t prop[2];		uint32_t prop[2];
uint32_t nptlp, shift = 0, slb_encoding = 0;		uint32_t nptlp, shift = 0, slb_encoding = 0;
uint32_t lp_size, lp_encoding;		uint32_t lp_size, lp_encoding;
struct lpte old;		struct lpte old;
uint64_t vsid;		uint64_t vsid;
phandle_t dev, node, root;		phandle_t dev, node, root;
int idx, len, res;		int idx, len, res;
		bool has_lp;

rm_init(&mphyp_eviction_lock, "pte eviction");		rm_init(&mphyp_eviction_lock, "pte eviction");

moea64_early_bootstrap(kernelstart, kernelend);		moea64_early_bootstrap(kernelstart, kernelend);

root = OF_peer(0);		root = OF_peer(0);

dev = OF_child(root);		dev = OF_child(root);
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines	if (len > 0) {
* We have to use a variable length array on the stack		* We have to use a variable length array on the stack
* since we have very limited stack space.		* since we have very limited stack space.
*/		*/
pcell_t arr[len/sizeof(cell_t)];		pcell_t arr[len/sizeof(cell_t)];
res = OF_getencprop(node, "ibm,segment-page-sizes", arr,		res = OF_getencprop(node, "ibm,segment-page-sizes", arr,
sizeof(arr));		sizeof(arr));
len /= 4;		len /= 4;
idx = 0;		idx = 0;
		has_lp = false;
while (len > 0) {		while (len > 0) {
shift = arr[idx];		shift = arr[idx];
slb_encoding = arr[idx + 1];		slb_encoding = arr[idx + 1];
nptlp = arr[idx + 2];		nptlp = arr[idx + 2];

dprintf0("Segment Page Size: "		dprintf0("Segment Page Size: "
"%uKB, slb_enc=0x%X: {size, encoding}[%u] =",		"%uKB, slb_enc=0x%X: {size, encoding}[%u] =",
shift > 10? 1 << (shift-10) : 0,		shift > 10? 1 << (shift-10) : 0,
slb_encoding, nptlp);		slb_encoding, nptlp);

idx += 3;		idx += 3;
len -= 3;		len -= 3;
while (len > 0 && nptlp) {		while (len > 0 && nptlp) {
lp_size = arr[idx];		lp_size = arr[idx];
lp_encoding = arr[idx+1];		lp_encoding = arr[idx+1];

dprintf(" {%uKB, 0x%X}",		dprintf(" {%uKB, 0x%X}",
lp_size > 10? 1 << (lp_size-10) : 0,		lp_size > 10? 1 << (lp_size-10) : 0,
lp_encoding);		lp_encoding);

if (slb_encoding == SLBV_L && lp_encoding == 0)		if (slb_encoding == SLBV_L && lp_encoding == 0)
break;		has_lp = true;

		if (slb_encoding == SLB_PGSZ_4K_4K &&
		lp_encoding == LP_4K_16M)
		moea64_has_lp_4k_16m = true;

idx += 2;		idx += 2;
len -= 2;		len -= 2;
nptlp--;		nptlp--;
}		}
dprintf("\n");		dprintf("\n");
if (nptlp && slb_encoding == SLBV_L && lp_encoding == 0)		if (has_lp && moea64_has_lp_4k_16m)
break;		break;
}		}

if (len > 0) {		if (has_lp) {
moea64_large_page_shift = shift;		moea64_large_page_shift = shift;
moea64_large_page_size = 1ULL << lp_size;		moea64_large_page_size = 1ULL << lp_size;
moea64_large_page_mask = moea64_large_page_size - 1;		moea64_large_page_mask = moea64_large_page_size - 1;
hw_direct_map = 1;		hw_direct_map = 1;
printf(MMU_PHYP_ID		printf(MMU_PHYP_ID
"Support for hugepages of %uKB detected\n",		"Support for hugepages of %uKB detected\n",
moea64_large_page_shift > 10?		moea64_large_page_shift > 10?
1 << (moea64_large_page_shift-10) : 0);		1 << (moea64_large_page_shift-10) : 0);
▲ Show 20 Lines • Show All 145 Lines • ▼ Show 20 Lines	mphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict)
/* Start at a random slot */		/* Start at a random slot */
i = mftb() % 8;		i = mftb() % 8;
k = -1;		k = -1;
for (j = 0; j < 8; j++) {		for (j = 0; j < 8; j++) {
slot = ptegbase + (i + j) % 8;		slot = ptegbase + (i + j) % 8;
phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi,		phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi,
&pt.pte_lo, &junk);		&pt.pte_lo, &junk);

if (pt.pte_hi & LPTE_WIRED)		if ((pt.pte_hi & (LPTE_WIRED \| LPTE_BIG)) != 0)
continue;		continue;

/* This is a candidate, so remember it */		/* This is a candidate, so remember it */
k = slot;		k = slot;

/* Try to get a page that has not been used lately */		/* Try to get a page that has not been used lately */
if (!(pt.pte_hi & LPTE_VALID) \|\| !(pt.pte_lo & LPTE_REF)) {		if (!(pt.pte_hi & LPTE_VALID) \|\| !(pt.pte_lo & LPTE_REF)) {
memcpy(to_evict, &pt, sizeof(struct lpte));		memcpy(to_evict, &pt, sizeof(struct lpte));
return (k);		return (k);
}		}
}		}

if (k == -1)		if (k == -1)
return (k);		return (k);

phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi,		phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi,
&to_evict->pte_lo, &junk);		&to_evict->pte_lo, &junk);
return (k);		return (k);
}		}

static int64_t		static __inline int64_t
mphyp_pte_insert(struct pvo_entry *pvo)		mphyp_pte_insert_locked(struct pvo_entry pvo, struct lpte pte)
{		{
struct rm_priotracker track;		struct lpte evicted;
		uint64_t index, junk;
int64_t result;		int64_t result;
struct lpte evicted, pte;
uint64_t index, junk, lastptelo;

PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);

/* Initialize PTE */
moea64_pte_from_pvo(pvo, &pte);
evicted.pte_hi = 0;

/* Make sure further insertion is locked out during evictions */
rm_rlock(&mphyp_eviction_lock, &track);

/*		/*
* First try primary hash.		* First try primary hash.
*/		*/
pvo->pvo_pte.slot &= ~7UL; /* Base slot address */		pvo->pvo_pte.slot &= ~7UL; /* Base slot address */
result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi,		result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte->pte_hi,
pte.pte_lo, &index, &evicted.pte_lo, &junk);		pte->pte_lo, &index, &evicted.pte_lo, &junk);
if (result == H_SUCCESS) {		if (result == H_SUCCESS) {
rm_runlock(&mphyp_eviction_lock, &track);
pvo->pvo_pte.slot = index;		pvo->pvo_pte.slot = index;
return (0);		return (0);
}		}
KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld "		KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld "
"(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot,		"(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot,
moea64_pteg_count, pte.pte_hi, pte.pte_lo));		moea64_pteg_count, pte->pte_hi, pte->pte_lo));

/*		/*
* Next try secondary hash.		* Next try secondary hash.
*/		*/
pvo->pvo_vaddr ^= PVO_HID;		pvo->pvo_vaddr ^= PVO_HID;
pte.pte_hi ^= LPTE_HID;		pte->pte_hi ^= LPTE_HID;
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);		pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);

result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot,		result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot,
pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk);		pte->pte_hi, pte->pte_lo, &index, &evicted.pte_lo, &junk);
if (result == H_SUCCESS) {		if (result == H_SUCCESS) {
rm_runlock(&mphyp_eviction_lock, &track);
pvo->pvo_pte.slot = index;		pvo->pvo_pte.slot = index;
return (0);		return (0);
}		}
KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld",		KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld",
result));		result));

/*		return (-1);
* Out of luck. Find a PTE to sacrifice.		}
*/

/* Lock out all insertions for a bit */
rm_runlock(&mphyp_eviction_lock, &track);
rm_wlock(&mphyp_eviction_lock);

		static __inline int64_t
		mphyp_pte_evict_and_insert_locked(struct pvo_entry pvo, struct lpte pte)
		{
		struct lpte evicted;
		uint64_t index, junk, lastptelo;
		int64_t result;

		evicted.pte_hi = 0;

index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);		index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);
if (index == -1L) {		if (index == -1L) {
/* Try other hash table? */		/* Try other hash table? */
pvo->pvo_vaddr ^= PVO_HID;		pvo->pvo_vaddr ^= PVO_HID;
pte.pte_hi ^= LPTE_HID;		pte->pte_hi ^= LPTE_HID;
pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);		pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);		index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);
}		}

if (index == -1L) {		if (index == -1L) {
/* No freeable slots in either PTEG? We're hosed. */		/* No freeable slots in either PTEG? We're hosed. */
rm_wunlock(&mphyp_eviction_lock);		rm_wunlock(&mphyp_eviction_lock);
panic("mphyp_pte_insert: overflow");		panic("mphyp_pte_insert: overflow");
return (-1);		return (-1);
}		}

/* Victim acquired: update page before waving goodbye */		/* Victim acquired: update page before waving goodbye */
if (evicted.pte_hi & LPTE_VALID) {		if (evicted.pte_hi & LPTE_VALID) {
result = phyp_pft_hcall(H_REMOVE, H_AVPN, index,		result = phyp_pft_hcall(H_REMOVE, H_AVPN, index,
evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo,		evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo,
&junk);		&junk);
STAT_MOEA64(moea64_pte_overflow++);		STAT_MOEA64(moea64_pte_overflow++);
KASSERT(result == H_SUCCESS \|\| result == H_NOT_FOUND,		KASSERT(result == H_SUCCESS \|\| result == H_NOT_FOUND,
("Error evicting page: %d", (int)result));		("Error evicting page: %d", (int)result));
}		}

/*		/*
* Set the new PTE.		* Set the new PTE.
*/		*/
result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte.pte_hi,		result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte->pte_hi,
pte.pte_lo, &index, &evicted.pte_lo, &junk);		pte->pte_lo, &index, &evicted.pte_lo, &junk);
rm_wunlock(&mphyp_eviction_lock); /* All clear */

pvo->pvo_pte.slot = index;		pvo->pvo_pte.slot = index;
if (result == H_SUCCESS)		if (result == H_SUCCESS)
return (0);		return (0);

		rm_wunlock(&mphyp_eviction_lock);
panic("Page replacement error: %ld", result);		panic("Page replacement error: %ld", result);
return (result);		return (result);
}		}

		static int64_t
		mphyp_pte_insert(struct pvo_entry *pvo)
		{
		struct rm_priotracker track;
		int64_t ret;
		struct lpte pte;

		PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);

		/* Initialize PTE */
		moea64_pte_from_pvo(pvo, &pte);

		/* Make sure further insertion is locked out during evictions */
		rm_rlock(&mphyp_eviction_lock, &track);

		ret = mphyp_pte_insert_locked(pvo, &pte);
		rm_runlock(&mphyp_eviction_lock, &track);

		if (ret == -1) {
		/*
		* Out of luck. Find a PTE to sacrifice.
		*/

		/* Lock out all insertions for a bit */
		rm_wlock(&mphyp_eviction_lock);
		ret = mphyp_pte_evict_and_insert_locked(pvo, &pte);
		rm_wunlock(&mphyp_eviction_lock); /* All clear */
		}

		return (ret);
		}

static void *		static void *
mphyp_dump_pmap(void ctx, void buf, u_long *nbytes)		mphyp_dump_pmap(void ctx, void buf, u_long *nbytes)
{		{
struct dump_context *dctx;		struct dump_context *dctx;
struct lpte p, *pbuf;		struct lpte p, *pbuf;
int bufidx;		int bufidx;
uint64_t junk;		uint64_t junk;
u_long ptex, ptex_end;		u_long ptex, ptex_end;
Show All 12 Lines	mphyp_dump_pmap(void ctx, void buf, u_long *nbytes)
for (; ptex < ptex_end; ptex++) {		for (; ptex < ptex_end; ptex++) {
phyp_pft_hcall(H_READ, 0, ptex, 0, 0,		phyp_pft_hcall(H_READ, 0, ptex, 0, 0,
&p.pte_hi, &p.pte_lo, &junk);		&p.pte_hi, &p.pte_lo, &junk);
pbuf[bufidx++] = p;		pbuf[bufidx++] = p;
}		}

dctx->ptex = ptex;		dctx->ptex = ptex;
return (buf);		return (buf);
		}

		static int64_t
		mphyp_pte_unset_sp(struct pvo_entry *pvo)
		{
		struct lpte pte;
		uint64_t junk, refchg;
		int err;
		vm_offset_t eva;
		pmap_t pm;

		pm = pvo->pvo_pmap;
		PMAP_LOCK_ASSERT(pm, MA_OWNED);
		KASSERT((PVO_VADDR(pvo) & SP_MASK) == 0,
		("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));

		refchg = 0;
		eva = PVO_VADDR(pvo) + SP_SIZE;

		for (; pvo != NULL && PVO_VADDR(pvo) < eva;
		pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
		moea64_pte_from_pvo(pvo, &pte);

		err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot,
		pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo,
		&junk);
		KASSERT(err == H_SUCCESS \|\| err == H_NOT_FOUND,
		("Error removing page: %d", err));

		if (err == H_NOT_FOUND)
		STAT_MOEA64(moea64_pte_overflow--);
		refchg \|= pte.pte_lo & (LPTE_REF \| LPTE_CHG);
		}

		return (refchg);
		}

		static int64_t
		mphyp_pte_insert_sp(struct pvo_entry *pvo)
		{
		struct rm_priotracker track;
		int64_t ret;
		struct lpte pte;
		vm_offset_t eva;
		pmap_t pm;

		pm = pvo->pvo_pmap;
		PMAP_LOCK_ASSERT(pm, MA_OWNED);
		KASSERT((PVO_VADDR(pvo) & SP_MASK) == 0,
		("%s: va %#jx unaligned", __func__, (uintmax_t)PVO_VADDR(pvo)));

		eva = PVO_VADDR(pvo) + SP_SIZE;

		/* Make sure further insertion is locked out during evictions */
		rm_rlock(&mphyp_eviction_lock, &track);

		for (; pvo != NULL && PVO_VADDR(pvo) < eva;
		pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
		/* Initialize PTE */
		moea64_pte_from_pvo(pvo, &pte);

		ret = mphyp_pte_insert_locked(pvo, &pte);
		if (ret == -1) {
		/*
		* Out of luck. Find a PTE to sacrifice.
		*/

		/* Lock out all insertions for a bit */
		rm_runlock(&mphyp_eviction_lock, &track);
		rm_wlock(&mphyp_eviction_lock);
		mphyp_pte_evict_and_insert_locked(pvo, &pte);
		rm_wunlock(&mphyp_eviction_lock); /* All clear */
		rm_rlock(&mphyp_eviction_lock, &track);
		}
		}

		rm_runlock(&mphyp_eviction_lock, &track);
		return (0);
		}

		static int64_t
		mphyp_pte_replace_sp(struct pvo_entry *pvo)
		{
		int64_t refchg;

		refchg = mphyp_pte_unset_sp(pvo);
		mphyp_pte_insert_sp(pvo);
		return (refchg);
}		}