Index: user/nwhitehorn/ps3/powerpc/include/pte.h =================================================================== --- user/nwhitehorn/ps3/powerpc/include/pte.h (revision 211725) +++ user/nwhitehorn/ps3/powerpc/include/pte.h (revision 211726) @@ -1,260 +1,261 @@ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $NetBSD: pte.h,v 1.2 1998/08/31 14:43:40 tsubai Exp $ * $FreeBSD$ */ #ifndef _MACHINE_PTE_H_ #define _MACHINE_PTE_H_ #if defined(AIM) /* * Page Table Entries */ #ifndef LOCORE /* 32-bit PTE */ struct pte { u_int32_t pte_hi; u_int32_t pte_lo; }; struct pteg { struct pte pt[8]; }; /* 64-bit (long) PTE */ struct lpte { u_int64_t pte_hi; u_int64_t pte_lo; }; struct lpteg { struct lpte pt[8]; }; #endif /* LOCORE */ /* 32-bit PTE definitions */ /* High word: */ #define PTE_VALID 0x80000000 #define PTE_VSID_SHFT 7 #define PTE_HID 0x00000040 #define PTE_API 0x0000003f /* Low word: */ #define PTE_RPGN 0xfffff000 #define PTE_REF 0x00000100 #define PTE_CHG 0x00000080 #define PTE_WIMG 0x00000078 #define PTE_W 0x00000040 #define PTE_I 0x00000020 #define PTE_M 0x00000010 #define PTE_G 0x00000008 #define PTE_PP 0x00000003 #define PTE_SO 0x00000000 /* Super. Only (U: XX, S: RW) */ #define PTE_SW 0x00000001 /* Super. Write-Only (U: RO, S: RW) */ #define PTE_BW 0x00000002 /* Supervisor (U: RW, S: RW) */ #define PTE_BR 0x00000003 /* Both Read Only (U: RO, S: RO) */ #define PTE_RW PTE_BW #define PTE_RO PTE_BR #define PTE_EXEC 0x00000200 /* pseudo bit in attrs; page is exec */ /* 64-bit PTE definitions */ /* High quadword: */ #define LPTE_VSID_SHIFT 12 +#define LPTE_AVPN_MASK 0xFFFFFFFFFFFFFF80ULL #define LPTE_API 0x0000000000000F80ULL #define LPTE_WIRED 0x0000000000000010ULL #define LPTE_LOCKED 0x0000000000000008ULL #define LPTE_BIG 0x0000000000000004ULL /* 4kb/16Mb page */ #define LPTE_HID 0x0000000000000002ULL #define LPTE_VALID 0x0000000000000001ULL /* Low quadword: */ #define EXTEND_PTE(x) UINT64_C(x) /* make constants 64-bit */ #define LPTE_RPGN 0xfffffffffffff000ULL #define LPTE_REF EXTEND_PTE( PTE_REF ) #define LPTE_CHG EXTEND_PTE( PTE_CHG ) #define LPTE_WIMG EXTEND_PTE( PTE_WIMG ) #define LPTE_W EXTEND_PTE( PTE_W ) #define LPTE_I EXTEND_PTE( PTE_I ) #define LPTE_M EXTEND_PTE( PTE_M ) #define LPTE_G EXTEND_PTE( PTE_G ) #define LPTE_NOEXEC 0x0000000000000004ULL #define LPTE_PP EXTEND_PTE( PTE_PP ) #define LPTE_SO EXTEND_PTE( PTE_SO ) /* Super. Only */ #define LPTE_SW EXTEND_PTE( PTE_SW ) /* Super. Write-Only */ #define LPTE_BW EXTEND_PTE( PTE_BW ) /* Supervisor */ #define LPTE_BR EXTEND_PTE( PTE_BR ) /* Both Read Only */ #define LPTE_RW LPTE_BW #define LPTE_RO LPTE_BR #ifndef LOCORE typedef struct pte pte_t; typedef struct lpte lpte_t; #endif /* LOCORE */ /* * Extract bits from address */ #define ADDR_SR_SHFT 28 #define ADDR_PIDX 0x0ffff000UL #define ADDR_PIDX_SHFT 12 #define ADDR_API_SHFT 22 #define ADDR_API_SHFT64 16 #define ADDR_POFF 0x00000fffUL /* * Bits in DSISR: */ #define DSISR_DIRECT 0x80000000 #define DSISR_NOTFOUND 0x40000000 #define DSISR_PROTECT 0x08000000 #define DSISR_INVRX 0x04000000 #define DSISR_STORE 0x02000000 #define DSISR_DABR 0x00400000 #define DSISR_SEGMENT 0x00200000 #define DSISR_EAR 0x00100000 /* * Bits in SRR1 on ISI: */ #define ISSRR1_NOTFOUND 0x40000000 #define ISSRR1_DIRECT 0x10000000 #define ISSRR1_PROTECT 0x08000000 #define ISSRR1_SEGMENT 0x00200000 #ifdef _KERNEL #ifndef LOCORE extern u_int dsisr(void); #endif /* _KERNEL */ #endif /* LOCORE */ #else #include /* * 1st level - page table directory (pdir) * * pdir consists of 1024 entries, each being a pointer to * second level entity, i.e. the actual page table (ptbl). */ #define PDIR_SHIFT 22 #define PDIR_SIZE (1 << PDIR_SHIFT) /* va range mapped by pdir */ #define PDIR_MASK (~(PDIR_SIZE - 1)) #define PDIR_NENTRIES 1024 /* number of page tables in pdir */ /* Returns pdir entry number for given va */ #define PDIR_IDX(va) ((va) >> PDIR_SHIFT) #define PDIR_ENTRY_SHIFT 2 /* entry size is 2^2 = 4 bytes */ /* * 2nd level - page table (ptbl) * * Page table covers 1024 page table entries. Page * table entry (pte) is 32 bit wide and defines mapping * for a single page. */ #define PTBL_SHIFT PAGE_SHIFT #define PTBL_SIZE PAGE_SIZE /* va range mapped by ptbl entry */ #define PTBL_MASK ((PDIR_SIZE - 1) & ~((1 << PAGE_SHIFT) - 1)) #define PTBL_NENTRIES 1024 /* number of pages mapped by ptbl */ /* Returns ptbl entry number for given va */ #define PTBL_IDX(va) (((va) & PTBL_MASK) >> PTBL_SHIFT) /* Size of ptbl in pages, 1024 entries, each sizeof(struct pte_entry). */ #define PTBL_PAGES 2 #define PTBL_ENTRY_SHIFT 3 /* entry size is 2^3 = 8 bytes */ /* * Flags for pte_remove() routine. */ #define PTBL_HOLD 0x00000001 /* do not unhold ptbl pages */ #define PTBL_UNHOLD 0x00000002 /* unhold and attempt to free ptbl pages */ #define PTBL_HOLD_FLAG(pmap) (((pmap) == kernel_pmap) ? PTBL_HOLD : PTBL_UNHOLD) /* * Page Table Entry definitions and macros. */ #ifndef LOCORE struct pte { vm_offset_t rpn; uint32_t flags; }; typedef struct pte pte_t; #endif /* RPN mask, TLB0 4K pages */ #define PTE_PA_MASK PAGE_MASK /* PTE bits assigned to MAS2, MAS3 flags */ #define PTE_W MAS2_W #define PTE_I MAS2_I #define PTE_M MAS2_M #define PTE_G MAS2_G #define PTE_MAS2_MASK (MAS2_G | MAS2_M | MAS2_I | MAS2_W) #define PTE_MAS3_SHIFT 8 #define PTE_UX (MAS3_UX << PTE_MAS3_SHIFT) #define PTE_SX (MAS3_SX << PTE_MAS3_SHIFT) #define PTE_UW (MAS3_UW << PTE_MAS3_SHIFT) #define PTE_SW (MAS3_SW << PTE_MAS3_SHIFT) #define PTE_UR (MAS3_UR << PTE_MAS3_SHIFT) #define PTE_SR (MAS3_SR << PTE_MAS3_SHIFT) #define PTE_MAS3_MASK ((MAS3_UX | MAS3_SX | MAS3_UW \ | MAS3_SW | MAS3_UR | MAS3_SR) << PTE_MAS3_SHIFT) /* Other PTE flags */ #define PTE_VALID 0x80000000 /* Valid */ #define PTE_MODIFIED 0x40000000 /* Modified */ #define PTE_WIRED 0x20000000 /* Wired */ #define PTE_MANAGED 0x10000000 /* Managed */ #define PTE_REFERENCED 0x04000000 /* Referenced */ /* Macro argument must of pte_t type. */ #define PTE_PA(pte) ((pte)->rpn & ~PTE_PA_MASK) #define PTE_ISVALID(pte) ((pte)->flags & PTE_VALID) #define PTE_ISWIRED(pte) ((pte)->flags & PTE_WIRED) #define PTE_ISMANAGED(pte) ((pte)->flags & PTE_MANAGED) #define PTE_ISMODIFIED(pte) ((pte)->flags & PTE_MODIFIED) #define PTE_ISREFERENCED(pte) ((pte)->flags & PTE_REFERENCED) #endif /* #elif defined(E500) */ #endif /* _MACHINE_PTE_H_ */ Index: user/nwhitehorn/ps3/powerpc/ps3/mmu_ps3.c =================================================================== --- user/nwhitehorn/ps3/powerpc/ps3/mmu_ps3.c (revision 211725) +++ user/nwhitehorn/ps3/powerpc/ps3/mmu_ps3.c (revision 211726) @@ -1,332 +1,348 @@ /*- * Copyright (C) 2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mmu_if.h" #include "ps3-hvcall.h" -#define VSID_HASH_MASK 0x0000007fffffffffULL +#define VSID_HASH_MASK 0x0000007fffffffffUL +#define LV1_READ_HTAB_LO_MASK 0xfffUL extern int ps3fb_remap(void); static uint64_t mps3_vas_id; /* * Kernel MMU interface */ static void mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend); static void mps3_cpu_bootstrap(mmu_t mmup, int ap); static void mps3_pte_synch(struct lpte *pt, struct lpte *pvo_pt); static void mps3_pte_clear(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn, u_int64_t ptebit); static void mps3_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn); static void mps3_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn); static int mps3_pte_insert(u_int ptegidx, struct lpte *pvo_pt); static struct lpte *mps3_pvo_to_pte(const struct pvo_entry *pvo); static mmu_method_t mps3_methods[] = { MMUMETHOD(mmu_change_wiring, moea64_change_wiring), MMUMETHOD(mmu_clear_modify, moea64_clear_modify), MMUMETHOD(mmu_clear_reference, moea64_clear_reference), MMUMETHOD(mmu_copy_page, moea64_copy_page), MMUMETHOD(mmu_enter, moea64_enter), MMUMETHOD(mmu_enter_object, moea64_enter_object), MMUMETHOD(mmu_enter_quick, moea64_enter_quick), MMUMETHOD(mmu_extract, moea64_extract), MMUMETHOD(mmu_extract_and_hold, moea64_extract_and_hold), MMUMETHOD(mmu_init, moea64_init), MMUMETHOD(mmu_is_modified, moea64_is_modified), MMUMETHOD(mmu_is_referenced, moea64_is_referenced), MMUMETHOD(mmu_ts_referenced, moea64_ts_referenced), MMUMETHOD(mmu_map, moea64_map), MMUMETHOD(mmu_page_exists_quick,moea64_page_exists_quick), MMUMETHOD(mmu_page_wired_mappings,moea64_page_wired_mappings), MMUMETHOD(mmu_pinit, moea64_pinit), MMUMETHOD(mmu_pinit0, moea64_pinit0), MMUMETHOD(mmu_protect, moea64_protect), MMUMETHOD(mmu_qenter, moea64_qenter), MMUMETHOD(mmu_qremove, moea64_qremove), MMUMETHOD(mmu_release, moea64_release), MMUMETHOD(mmu_remove, moea64_remove), MMUMETHOD(mmu_remove_all, moea64_remove_all), MMUMETHOD(mmu_remove_write, moea64_remove_write), MMUMETHOD(mmu_sync_icache, moea64_sync_icache), MMUMETHOD(mmu_zero_page, moea64_zero_page), MMUMETHOD(mmu_zero_page_area, moea64_zero_page_area), MMUMETHOD(mmu_zero_page_idle, moea64_zero_page_idle), MMUMETHOD(mmu_activate, moea64_activate), MMUMETHOD(mmu_deactivate, moea64_deactivate), /* Internal interfaces */ MMUMETHOD(mmu_bootstrap, mps3_bootstrap), MMUMETHOD(mmu_cpu_bootstrap, mps3_cpu_bootstrap), MMUMETHOD(mmu_mapdev, moea64_mapdev), MMUMETHOD(mmu_unmapdev, moea64_unmapdev), MMUMETHOD(mmu_kextract, moea64_kextract), MMUMETHOD(mmu_kenter, moea64_kenter), MMUMETHOD(mmu_dev_direct_mapped,moea64_dev_direct_mapped), { 0, 0 } }; static mmu_def_t ps3_mmu = { "mmu_ps3", mps3_methods, 0 }; MMU_DEF(ps3_mmu); static void mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { uint64_t final_pteg_count; /* * Set our page table override functions */ moea64_pte_synch_hook = mps3_pte_synch; moea64_pte_clear_hook = mps3_pte_clear; moea64_pte_unset_hook = mps3_pte_unset; moea64_pte_change_hook = mps3_pte_change; moea64_pte_insert_hook = mps3_pte_insert; moea64_pvo_to_pte_hook = mps3_pvo_to_pte; moea64_early_bootstrap(mmup, kernelstart, kernelend); lv1_construct_virtual_address_space( 20 /* log_2(moea64_pteg_count) */, 2 /* n page sizes */, (24UL << 56) | (16UL << 48) /* page sizes 16 MB + 64 KB */, &mps3_vas_id, &final_pteg_count ); moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); moea64_late_bootstrap(mmup, kernelstart, kernelend); } static void mps3_cpu_bootstrap(mmu_t mmup, int ap) { struct slb *slb = PCPU_GET(slb); register_t seg0; int i; mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); /* * Destroy the loader's address space if we are coming up for * the first time, and redo the FB mapping so we can continue * having a console. */ if (!ap) lv1_destruct_virtual_address_space(0); lv1_select_virtual_address_space(mps3_vas_id); if (!ap) ps3fb_remap(); /* * Install kernel SLB entries */ __asm __volatile ("slbia"); __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); for (i = 0; i < 64; i++) { if (!(slb[i].slbe & SLBE_VALID)) continue; __asm __volatile ("slbmte %0, %1" :: "r"(slb[i].slbv), "r"(slb[i].slbe)); } } static void mps3_pte_synch(struct lpte *pt, struct lpte *pvo_pt) { uint64_t halfbucket[4], rcbits; uint64_t slot = (uint64_t)(pt)-1; + __asm __volatile("ptesync"); lv1_read_htab_entries(mps3_vas_id, slot & ~0x3UL, &halfbucket[0], &halfbucket[1], &halfbucket[2], &halfbucket[3], &rcbits); - pvo_pt->pte_lo |= ((rcbits >> (slot & 0x3)) & 0x3) << 7; + /* + * rcbits contains the low 12 bits of each PTEs 2nd part, + * spaced at 16-bit intervals + */ + + KASSERT((halfbucket[slot & 0x3] & LPTE_AVPN_MASK) == + (pvo_pt->pte_hi & LPTE_AVPN_MASK), + ("PTE upper word %#lx != %#lx\n", + halfbucket[slot & 0x3], pvo_pt->pte_hi)); + + pvo_pt->pte_lo &= ~LV1_READ_HTAB_LO_MASK; + pvo_pt->pte_lo |= (rcbits >> ((3 - (slot & 0x3))*16)) & + LV1_READ_HTAB_LO_MASK; } static void mps3_pte_clear(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn, u_int64_t ptebit) { uint64_t slot = (uint64_t)(pt)-1; lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi, pvo_pt->pte_lo & ~ptebit); } static void mps3_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { uint64_t slot = (uint64_t)(pt)-1; + mps3_pte_synch(pt, pvo_pt); pvo_pt->pte_hi &= ~LPTE_VALID; lv1_write_htab_entry(mps3_vas_id, slot, 0, 0); moea64_pte_valid--; } static void mps3_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { uint64_t slot = (uint64_t)(pt)-1; + mps3_pte_synch(pt, pvo_pt); pvo_pt->pte_hi |= LPTE_VALID; lv1_write_htab_entry(mps3_vas_id, slot & ~0x3UL, pvo_pt->pte_hi, pvo_pt->pte_lo); } static int mps3_pte_insert(u_int ptegidx, struct lpte *pvo_pt) { int result; struct lpte evicted; struct pvo_entry *pvo; uint64_t index; pvo_pt->pte_hi |= LPTE_VALID; pvo_pt->pte_hi &= ~LPTE_HID; evicted.pte_hi = 0; result = lv1_insert_htab_entry(mps3_vas_id, ptegidx << 3, pvo_pt->pte_hi, pvo_pt->pte_lo, LPTE_LOCKED | LPTE_WIRED, 0, &index, &evicted.pte_hi, &evicted.pte_lo); if (result != 0) { /* No freeable slots in either PTEG? We're hosed. */ mtmsr(mfmsr() | PSL_DR); panic("moea64_pte_insert: overflow"); return (-1); } /* * See where we ended up. */ if (index >> 3 != ptegidx) pvo_pt->pte_hi |= LPTE_HID; moea64_pte_valid++; if (!evicted.pte_hi) return (index & 0x7); /* * Synchronize the sacrifice PTE with its PVO, then mark both * invalid. The PVO will be reused when/if the VM system comes * here after a fault. */ if (evicted.pte_hi & LPTE_HID) ptegidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { if (pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi) { KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, ("Invalid PVO for valid PTE!")); pvo->pvo_pte.lpte.pte_hi &= ~LPTE_VALID; pvo->pvo_pte.lpte.pte_lo |= evicted.pte_lo & (LPTE_REF | LPTE_CHG); PVO_PTEGIDX_CLR(pvo); moea64_pte_valid--; moea64_pte_overflow++; break; } } return (index & 0x7); } static __inline u_int va_to_pteg(uint64_t vsid, vm_offset_t addr, int large) { uint64_t hash; int shift; shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> shift); return (hash & moea64_pteg_mask); } static struct lpte * mps3_pvo_to_pte(const struct pvo_entry *pvo) { uint64_t slot, vsid; u_int ptegidx; /* If the PTEG index is not set, then there is no page table entry */ if (!PVO_PTEGIDX_ISSET(pvo)) return (NULL); vsid = PVO_VSID(pvo); ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); /* * We can find the actual pte entry without searching by grabbing * the PTEG index from 3 unused bits in pvo_vaddr and by * noticing the HID bit. */ if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) ptegidx ^= moea64_pteg_mask; slot = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo); return ((struct lpte *)(slot + 1)); }