Index: user/nwhitehorn/ps3/powerpc/aim/mmu_oea64.c =================================================================== --- user/nwhitehorn/ps3/powerpc/aim/mmu_oea64.c (revision 212100) +++ user/nwhitehorn/ps3/powerpc/aim/mmu_oea64.c (revision 212101) @@ -1,3062 +1,3065 @@ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Matt Thomas of Allegro Networks, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the NetBSD * Foundation, Inc. and its contributors. * 4. Neither the name of The NetBSD Foundation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $ */ /*- * Copyright (C) 2001 Benno Rice. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * In addition to hardware address maps, this module is called upon to * provide software-use-only maps which may or may not be stored in the * same form as hardware maps. These pseudo-maps are used to store * intermediate results from copy operations to and from address spaces. * * Since the information managed by this module is also stored by the * logical address mapping module, this module may throw away valid virtual * to physical mappings at almost any time. However, invalidations of * mappings must be done as requested. * * In order to cope with hardware architectures which make virtual to * physical map invalidates expensive, this module may delay invalidate * reduced protection operations until such time as they are actually * necessary. This module is given full information as to which processors * are currently using which maps, and to when physical maps must be made * correct. */ #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mmu_oea64.h" #include "mmu_if.h" #define MOEA_DEBUG #define TODO panic("%s: not implemented", __func__); void moea64_release_vsid(uint64_t vsid); uintptr_t moea64_get_unique_vsid(void); static __inline register_t cntlzd(volatile register_t a) { register_t b; __asm ("cntlzd %0, %1" : "=r"(b) : "r"(a)); return b; } #define PTESYNC() __asm __volatile("ptesync"); #define TLBSYNC() __asm __volatile("tlbsync; ptesync"); #define SYNC() __asm __volatile("sync"); #define EIEIO() __asm __volatile("eieio"); /* * The tlbie instruction must be executed in 64-bit mode * so we have to twiddle MSR[SF] around every invocation. * Just to add to the fun, exceptions must be off as well * so that we can't trap in 64-bit mode. What a pain. */ struct mtx tlbie_mutex; static __inline void TLBIE(uint64_t vpn) { #ifndef __powerpc64__ register_t vpn_hi, vpn_lo; register_t msr; register_t scratch; #endif vpn <<= ADDR_PIDX_SHFT; vpn &= ~(0xffffULL << 48); mtx_lock_spin(&tlbie_mutex); #ifdef __powerpc64__ __asm __volatile("\ ptesync; \ tlbie %0; \ eieio; \ tlbsync; \ ptesync;" :: "r"(vpn) : "memory"); #else vpn_hi = (uint32_t)(vpn >> 32); vpn_lo = (uint32_t)vpn; __asm __volatile("\ mfmsr %0; \ mr %1, %0; \ insrdi %1,%5,1,0; \ mtmsrd %1; \ ptesync; \ \ sld %1,%2,%4; \ or %1,%1,%3; \ tlbie %1; \ \ mtmsrd %0; \ eieio; \ tlbsync; \ ptesync;" : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1) : "memory"); #endif mtx_unlock_spin(&tlbie_mutex); } #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR); isync() #define ENABLE_TRANS(msr) mtmsr(msr); isync() #define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4)) #define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff) #define VSID_HASH_MASK 0x0000007fffffffffULL #define MOEA_PVO_CHECK(pvo) #define LOCK_TABLE() mtx_lock(&moea64_table_mutex) #define UNLOCK_TABLE() mtx_unlock(&moea64_table_mutex); #define ASSERT_TABLE_LOCK() mtx_assert(&moea64_table_mutex, MA_OWNED) struct ofw_map { cell_t om_va; cell_t om_len; cell_t om_pa_hi; cell_t om_pa_lo; cell_t om_mode; }; /* * Map of physical memory regions. */ static struct mem_region *regions; static struct mem_region *pregions; static u_int phys_avail_count; static int regions_sz, pregions_sz; extern int ofw_real_mode; extern struct pmap ofw_pmap; extern void bs_remap_earlyboot(void); /* * Lock for the pteg and pvo tables. */ struct mtx moea64_table_mutex; struct mtx moea64_slb_mutex; /* * PTEG data. */ static struct lpteg *moea64_pteg_table; u_int moea64_pteg_count; u_int moea64_pteg_mask; /* * PVO data. */ struct pvo_head *moea64_pvo_table; /* pvo entries by pteg index */ /* lists of unmanaged pages */ struct pvo_head moea64_pvo_kunmanaged = LIST_HEAD_INITIALIZER(moea64_pvo_kunmanaged); struct pvo_head moea64_pvo_unmanaged = LIST_HEAD_INITIALIZER(moea64_pvo_unmanaged); uma_zone_t moea64_upvo_zone; /* zone for pvo entries for unmanaged pages */ uma_zone_t moea64_mpvo_zone; /* zone for pvo entries for managed pages */ #define BPVO_POOL_SIZE 327680 static struct pvo_entry *moea64_bpvo_pool; static int moea64_bpvo_pool_index = 0; #define VSID_NBPW (sizeof(u_int32_t) * 8) #ifdef __powerpc64__ #define NVSIDS (NPMAPS * 16) #define VSID_HASHMASK 0xffffffffUL #else #define NVSIDS NPMAPS #define VSID_HASHMASK 0xfffffUL #endif static u_int moea64_vsid_bitmap[NVSIDS / VSID_NBPW]; static boolean_t moea64_initialized = FALSE; /* * Statistics. */ u_int moea64_pte_valid = 0; u_int moea64_pte_overflow = 0; u_int moea64_pvo_entries = 0; u_int moea64_pvo_enter_calls = 0; u_int moea64_pvo_remove_calls = 0; SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_valid, CTLFLAG_RD, &moea64_pte_valid, 0, ""); SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_overflow, CTLFLAG_RD, &moea64_pte_overflow, 0, ""); SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_entries, CTLFLAG_RD, &moea64_pvo_entries, 0, ""); SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_enter_calls, CTLFLAG_RD, &moea64_pvo_enter_calls, 0, ""); SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD, &moea64_pvo_remove_calls, 0, ""); vm_offset_t moea64_scratchpage_va[2]; uint64_t moea64_scratchpage_vpn[2]; struct lpte *moea64_scratchpage_pte[2]; struct mtx moea64_scratchpage_mtx; uint64_t moea64_large_page_mask = 0; int moea64_large_page_size = 0; int moea64_large_page_shift = 0; /* * Hypervisor Hooks */ void (*moea64_pte_synch_hook)(struct lpte *pt, struct lpte *pvo_pt) = NULL; void (*moea64_pte_clear_hook)(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn, u_int64_t ptebit) = NULL; void (*moea64_pte_unset_hook)(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) = NULL; void (*moea64_pte_change_hook)(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) = NULL; int (*moea64_pte_insert_hook)(u_int ptegidx, struct lpte *pvo_pt) = NULL; struct lpte *(*moea64_pvo_to_pte_hook)(const struct pvo_entry *pvo) = NULL; /* * PTE calls. */ static int moea64_pte_insert_native(u_int, struct lpte *); /* * PVO calls. */ static int moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *, vm_offset_t, vm_offset_t, uint64_t, int); static void moea64_pvo_remove(struct pvo_entry *); static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t); static struct lpte *moea64_pvo_to_pte_native(const struct pvo_entry *); /* * Utility routines. */ static void moea64_bootstrap_native(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend); static void moea64_cpu_bootstrap(mmu_t, int ap); static void moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); static boolean_t moea64_query_bit(vm_page_t, u_int64_t); static u_int moea64_clear_bit(vm_page_t, u_int64_t); static void moea64_kremove(mmu_t, vm_offset_t); static void moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz); static void tlbia(void); #ifdef __powerpc64__ static void slbia(void); #endif /* * Kernel MMU interface */ static mmu_method_t moea64_methods[] = { MMUMETHOD(mmu_change_wiring, moea64_change_wiring), MMUMETHOD(mmu_clear_modify, moea64_clear_modify), MMUMETHOD(mmu_clear_reference, moea64_clear_reference), MMUMETHOD(mmu_copy_page, moea64_copy_page), MMUMETHOD(mmu_enter, moea64_enter), MMUMETHOD(mmu_enter_object, moea64_enter_object), MMUMETHOD(mmu_enter_quick, moea64_enter_quick), MMUMETHOD(mmu_extract, moea64_extract), MMUMETHOD(mmu_extract_and_hold, moea64_extract_and_hold), MMUMETHOD(mmu_init, moea64_init), MMUMETHOD(mmu_is_modified, moea64_is_modified), MMUMETHOD(mmu_is_referenced, moea64_is_referenced), MMUMETHOD(mmu_ts_referenced, moea64_ts_referenced), MMUMETHOD(mmu_map, moea64_map), MMUMETHOD(mmu_page_exists_quick,moea64_page_exists_quick), MMUMETHOD(mmu_page_wired_mappings,moea64_page_wired_mappings), MMUMETHOD(mmu_pinit, moea64_pinit), MMUMETHOD(mmu_pinit0, moea64_pinit0), MMUMETHOD(mmu_protect, moea64_protect), MMUMETHOD(mmu_qenter, moea64_qenter), MMUMETHOD(mmu_qremove, moea64_qremove), MMUMETHOD(mmu_release, moea64_release), MMUMETHOD(mmu_remove, moea64_remove), MMUMETHOD(mmu_remove_all, moea64_remove_all), MMUMETHOD(mmu_remove_write, moea64_remove_write), MMUMETHOD(mmu_sync_icache, moea64_sync_icache), MMUMETHOD(mmu_zero_page, moea64_zero_page), MMUMETHOD(mmu_zero_page_area, moea64_zero_page_area), MMUMETHOD(mmu_zero_page_idle, moea64_zero_page_idle), MMUMETHOD(mmu_activate, moea64_activate), MMUMETHOD(mmu_deactivate, moea64_deactivate), /* Internal interfaces */ MMUMETHOD(mmu_bootstrap, moea64_bootstrap_native), MMUMETHOD(mmu_cpu_bootstrap, moea64_cpu_bootstrap), MMUMETHOD(mmu_mapdev, moea64_mapdev), MMUMETHOD(mmu_unmapdev, moea64_unmapdev), MMUMETHOD(mmu_kextract, moea64_kextract), MMUMETHOD(mmu_kenter, moea64_kenter), MMUMETHOD(mmu_dev_direct_mapped,moea64_dev_direct_mapped), { 0, 0 } }; static mmu_def_t oea64_mmu = { MMU_TYPE_G5, moea64_methods, 0 }; MMU_DEF(oea64_mmu); static __inline u_int va_to_pteg(uint64_t vsid, vm_offset_t addr, int large) { uint64_t hash; int shift; shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> shift); return (hash & moea64_pteg_mask); } static __inline struct pvo_head * pa_to_pvoh(vm_offset_t pa, vm_page_t *pg_p) { struct vm_page *pg; pg = PHYS_TO_VM_PAGE(pa); if (pg_p != NULL) *pg_p = pg; if (pg == NULL) return (&moea64_pvo_unmanaged); return (&pg->md.mdpg_pvoh); } static __inline struct pvo_head * vm_page_to_pvoh(vm_page_t m) { return (&m->md.mdpg_pvoh); } static __inline void moea64_attr_clear(vm_page_t m, u_int64_t ptebit) { mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->md.mdpg_attrs &= ~ptebit; } static __inline u_int64_t moea64_attr_fetch(vm_page_t m) { return (m->md.mdpg_attrs); } static __inline void moea64_attr_save(vm_page_t m, u_int64_t ptebit) { mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->md.mdpg_attrs |= ptebit; } static __inline void moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va, uint64_t pte_lo, int flags) { ASSERT_TABLE_LOCK(); /* * Construct a PTE. Default to IMB initially. Valid bit only gets * set when the real pte is set in memory. * * Note: Don't set the valid bit for correct operation of tlb update. */ pt->pte_hi = (vsid << LPTE_VSID_SHIFT) | (((uint64_t)(va & ADDR_PIDX) >> ADDR_API_SHFT64) & LPTE_API); if (flags & PVO_LARGE) pt->pte_hi |= LPTE_BIG; pt->pte_lo = pte_lo; } static __inline void moea64_pte_synch_native(struct lpte *pt, struct lpte *pvo_pt) { ASSERT_TABLE_LOCK(); pvo_pt->pte_lo |= pt->pte_lo & (LPTE_REF | LPTE_CHG); } static __inline void moea64_pte_clear_native(struct lpte *pt, uint64_t vpn, u_int64_t ptebit) { ASSERT_TABLE_LOCK(); /* * As shown in Section 7.6.3.2.3 */ pt->pte_lo &= ~ptebit; TLBIE(vpn); } static __inline void moea64_pte_set_native(struct lpte *pt, struct lpte *pvo_pt) { ASSERT_TABLE_LOCK(); pvo_pt->pte_hi |= LPTE_VALID; /* * Update the PTE as defined in section 7.6.3.1. * Note that the REF/CHG bits are from pvo_pt and thus should have * been saved so this routine can restore them (if desired). */ pt->pte_lo = pvo_pt->pte_lo; EIEIO(); pt->pte_hi = pvo_pt->pte_hi; PTESYNC(); moea64_pte_valid++; } static __inline void moea64_pte_unset_native(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { ASSERT_TABLE_LOCK(); pvo_pt->pte_hi &= ~LPTE_VALID; /* * Force the reg & chg bits back into the PTEs. */ SYNC(); /* * Invalidate the pte. */ pt->pte_hi &= ~LPTE_VALID; TLBIE(vpn); /* * Save the reg & chg bits. */ moea64_pte_synch_native(pt, pvo_pt); moea64_pte_valid--; } static __inline void moea64_pte_change_native(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { /* * Invalidate the PTE */ moea64_pte_unset_native(pt, pvo_pt, vpn); moea64_pte_set_native(pt, pvo_pt); } static __inline uint64_t moea64_calc_wimg(vm_offset_t pa) { uint64_t pte_lo; int i; /* * Assume the page is cache inhibited and access is guarded unless * it's in our available memory array. */ pte_lo = LPTE_I | LPTE_G; for (i = 0; i < pregions_sz; i++) { if ((pa >= pregions[i].mr_start) && (pa < (pregions[i].mr_start + pregions[i].mr_size))) { pte_lo &= ~(LPTE_I | LPTE_G); pte_lo |= LPTE_M; break; } } return pte_lo; } /* * Quick sort callout for comparing memory regions. */ static int mr_cmp(const void *a, const void *b); static int om_cmp(const void *a, const void *b); static int mr_cmp(const void *a, const void *b) { const struct mem_region *regiona; const struct mem_region *regionb; regiona = a; regionb = b; if (regiona->mr_start < regionb->mr_start) return (-1); else if (regiona->mr_start > regionb->mr_start) return (1); else return (0); } static int om_cmp(const void *a, const void *b) { const struct ofw_map *mapa; const struct ofw_map *mapb; mapa = a; mapb = b; if (mapa->om_pa_hi < mapb->om_pa_hi) return (-1); else if (mapa->om_pa_hi > mapb->om_pa_hi) return (1); else if (mapa->om_pa_lo < mapb->om_pa_lo) return (-1); else if (mapa->om_pa_lo > mapb->om_pa_lo) return (1); else return (0); } static void moea64_cpu_bootstrap(mmu_t mmup, int ap) { int i = 0; #ifdef __powerpc64__ struct slb *slb = PCPU_GET(slb); #endif /* * Initialize segment registers and MMU */ mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); isync(); /* * Install kernel SLB entries */ #ifdef __powerpc64__ slbia(); for (i = 0; i < 64; i++) { if (!(slb[i].slbe & SLBE_VALID)) continue; __asm __volatile ("slbmte %0, %1" :: "r"(slb[i].slbv), "r"(slb[i].slbe)); } #else for (i = 0; i < 16; i++) mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]); #endif /* * Install page table */ __asm __volatile ("ptesync; mtsdr1 %0; isync" :: "r"((uintptr_t)moea64_pteg_table | (64 - cntlzd(moea64_pteg_mask >> 11)))); tlbia(); } static void moea64_add_ofw_mappings(mmu_t mmup, phandle_t mmu, size_t sz) { struct ofw_map translations[sz/sizeof(struct ofw_map)]; register_t msr; vm_offset_t off; vm_paddr_t pa_base; int i, ofw_mappings; bzero(translations, sz); if (OF_getprop(mmu, "translations", translations, sz) == -1) panic("moea64_bootstrap: can't get ofw translations"); CTR0(KTR_PMAP, "moea64_add_ofw_mappings: translations"); sz /= sizeof(*translations); qsort(translations, sz, sizeof (*translations), om_cmp); for (i = 0, ofw_mappings = 0; i < sz; i++) { CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x", (uint32_t)(translations[i].om_pa_lo), translations[i].om_va, translations[i].om_len); if (translations[i].om_pa_lo % PAGE_SIZE) panic("OFW translation not page-aligned!"); pa_base = translations[i].om_pa_lo; #ifdef __powerpc64__ pa_base += (vm_offset_t)translations[i].om_pa_hi << 32; #else if (translations[i].om_pa_hi) panic("OFW translations above 32-bit boundary!"); #endif /* Now enter the pages for this mapping */ DISABLE_TRANS(msr); for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) { if (moea64_pvo_find_va(kernel_pmap, translations[i].om_va + off) != NULL) continue; moea64_kenter(mmup, translations[i].om_va + off, pa_base + off); ofw_mappings++; } ENABLE_TRANS(msr); } } #ifdef __powerpc64__ static void moea64_probe_large_page(void) { uint16_t pvr = mfpvr() >> 16; switch (pvr) { case IBM970: case IBM970FX: case IBM970MP: powerpc_sync(); isync(); mtspr(SPR_HID4, mfspr(SPR_HID4) & ~HID4_970_DISABLE_LG_PG); powerpc_sync(); isync(); /* FALLTHROUGH */ case IBMCELLBE: moea64_large_page_size = 0x1000000; /* 16 MB */ moea64_large_page_shift = 24; break; default: moea64_large_page_size = 0; } moea64_large_page_mask = moea64_large_page_size - 1; } static void moea64_bootstrap_slb_prefault(vm_offset_t va, int large) { struct slb *cache; struct slb entry; uint64_t esid, slbe; uint64_t i; cache = PCPU_GET(slb); esid = va >> ADDR_SR_SHFT; slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; for (i = 0; i < 64; i++) { if (cache[i].slbe == (slbe | i)) return; } entry.slbe = slbe; entry.slbv = KERNEL_VSID(esid) << SLBV_VSID_SHIFT; if (large) entry.slbv |= SLBV_L; slb_insert(kernel_pmap, cache, &entry); } #endif static void moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { register_t msr; vm_paddr_t pa; vm_offset_t size, off; uint64_t pte_lo; int i; if (moea64_large_page_size == 0) hw_direct_map = 0; DISABLE_TRANS(msr); if (hw_direct_map) { PMAP_LOCK(kernel_pmap); for (i = 0; i < pregions_sz; i++) { for (pa = pregions[i].mr_start; pa < pregions[i].mr_start + pregions[i].mr_size; pa += moea64_large_page_size) { pte_lo = LPTE_M; /* * Set memory access as guarded if prefetch within * the page could exit the available physmem area. */ if (pa & moea64_large_page_mask) { pa &= moea64_large_page_mask; pte_lo |= LPTE_G; } if (pa + moea64_large_page_size > pregions[i].mr_start + pregions[i].mr_size) pte_lo |= LPTE_G; moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, &moea64_pvo_kunmanaged, pa, pa, pte_lo, PVO_WIRED | PVO_LARGE | VM_PROT_EXECUTE); } } PMAP_UNLOCK(kernel_pmap); } else { size = moea64_pteg_count * sizeof(struct lpteg); off = (vm_offset_t)(moea64_pteg_table); for (pa = off; pa < off + size; pa += PAGE_SIZE) moea64_kenter(mmup, pa, pa); size = sizeof(struct pvo_head) * moea64_pteg_count; off = (vm_offset_t)(moea64_pvo_table); for (pa = off; pa < off + size; pa += PAGE_SIZE) moea64_kenter(mmup, pa, pa); size = BPVO_POOL_SIZE*sizeof(struct pvo_entry); off = (vm_offset_t)(moea64_bpvo_pool); for (pa = off; pa < off + size; pa += PAGE_SIZE) moea64_kenter(mmup, pa, pa); /* * Map certain important things, like ourselves. * * NOTE: We do not map the exception vector space. That code is * used only in real mode, and leaving it unmapped allows us to * catch NULL pointer deferences, instead of making NULL a valid * address. */ for (pa = kernelstart & ~PAGE_MASK; pa < kernelend; pa += PAGE_SIZE) moea64_kenter(mmup, pa, pa); } ENABLE_TRANS(msr); } void moea64_early_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { int i, j; vm_size_t physsz, hwphyssz; #ifndef __powerpc64__ /* We don't have a direct map since there is no BAT */ hw_direct_map = 0; /* Make sure battable is zero, since we have no BAT */ for (i = 0; i < 16; i++) { battable[i].batu = 0; battable[i].batl = 0; } #else moea64_probe_large_page(); /* Use a direct map if we have large page support */ if (moea64_large_page_size > 0) hw_direct_map = 1; else hw_direct_map = 0; #endif /* Get physical memory regions from firmware */ mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); CTR0(KTR_PMAP, "moea64_bootstrap: physical memory"); qsort(pregions, pregions_sz, sizeof(*pregions), mr_cmp); if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz) panic("moea64_bootstrap: phys_avail too small"); qsort(regions, regions_sz, sizeof(*regions), mr_cmp); phys_avail_count = 0; physsz = 0; hwphyssz = 0; TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); for (i = 0, j = 0; i < regions_sz; i++, j += 2) { CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start, regions[i].mr_start + regions[i].mr_size, regions[i].mr_size); if (hwphyssz != 0 && (physsz + regions[i].mr_size) >= hwphyssz) { if (physsz < hwphyssz) { phys_avail[j] = regions[i].mr_start; phys_avail[j + 1] = regions[i].mr_start + hwphyssz - physsz; physsz = hwphyssz; phys_avail_count++; } break; } phys_avail[j] = regions[i].mr_start; phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; phys_avail_count++; physsz += regions[i].mr_size; } /* Check for overlap with the kernel and exception vectors */ for (j = 0; j < 2*phys_avail_count; j+=2) { if (phys_avail[j] < EXC_LAST) phys_avail[j] += EXC_LAST; if (kernelstart >= phys_avail[j] && kernelstart < phys_avail[j+1]) { if (kernelend < phys_avail[j+1]) { phys_avail[2*phys_avail_count] = (kernelend & ~PAGE_MASK) + PAGE_SIZE; phys_avail[2*phys_avail_count + 1] = phys_avail[j+1]; phys_avail_count++; } phys_avail[j+1] = kernelstart & ~PAGE_MASK; } if (kernelend >= phys_avail[j] && kernelend < phys_avail[j+1]) { if (kernelstart > phys_avail[j]) { phys_avail[2*phys_avail_count] = phys_avail[j]; phys_avail[2*phys_avail_count + 1] = kernelstart & ~PAGE_MASK; phys_avail_count++; } phys_avail[j] = (kernelend & ~PAGE_MASK) + PAGE_SIZE; } } physmem = btoc(physsz); #ifdef PTEGCOUNT moea64_pteg_count = PTEGCOUNT; #else moea64_pteg_count = 0x1000; while (moea64_pteg_count < physmem) moea64_pteg_count <<= 1; moea64_pteg_count >>= 1; #endif /* PTEGCOUNT */ } static void moea64_bootstrap_native(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { int i; vm_size_t size; register_t msr; moea64_early_bootstrap(mmup, kernelstart, kernelend); /* * Allocate PTEG table. */ size = moea64_pteg_count * sizeof(struct lpteg); CTR2(KTR_PMAP, "moea64_bootstrap: %d PTEGs, %d bytes", moea64_pteg_count, size); /* * We now need to allocate memory. This memory, to be allocated, * has to reside in a page table. The page table we are about to * allocate. We don't have BAT. So drop to data real mode for a minute * as a measure of last resort. We do this a couple times. */ moea64_pteg_table = (struct lpteg *)moea64_bootstrap_alloc(size, size); DISABLE_TRANS(msr); bzero((void *)moea64_pteg_table, moea64_pteg_count * sizeof(struct lpteg)); ENABLE_TRANS(msr); CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table); /* * Initialize the TLBIE lock. TLBIE can only be executed by one CPU. */ mtx_init(&tlbie_mutex, "tlbie mutex", NULL, MTX_SPIN); moea64_late_bootstrap(mmup, kernelstart, kernelend); /* * Allocate some things for page zeroing. We put this directly * in the page table, marked with LPTE_LOCKED, to avoid any * of the PVO book-keeping or other parts of the VM system * from even knowing that this hack exists. */ if (!hw_direct_map) { mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL, MTX_DEF); for (i = 0; i < 2; i++) { struct lpte pt; uint64_t vsid; int pteidx, ptegidx; moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE; virtual_end -= PAGE_SIZE; LOCK_TABLE(); vsid = va_to_vsid(kernel_pmap, moea64_scratchpage_va[i]); moea64_pte_create(&pt, vsid, moea64_scratchpage_va[i], LPTE_NOEXEC, 0); pt.pte_hi |= LPTE_LOCKED; moea64_scratchpage_vpn[i] = (vsid << 16) | ((moea64_scratchpage_va[i] & ADDR_PIDX) >> ADDR_PIDX_SHFT); ptegidx = va_to_pteg(vsid, moea64_scratchpage_va[i], 0); pteidx = moea64_pte_insert_native(ptegidx, &pt); if (pt.pte_hi & LPTE_HID) ptegidx ^= moea64_pteg_mask; moea64_scratchpage_pte[i] = &moea64_pteg_table[ptegidx].pt[pteidx]; UNLOCK_TABLE(); } } } void moea64_late_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { ihandle_t mmui; phandle_t chosen; phandle_t mmu; size_t sz; int i; vm_size_t size; vm_offset_t pa, va; register_t msr; void *dpcpu; /* * Set PTEG mask */ moea64_pteg_mask = moea64_pteg_count - 1; /* * Allocate pv/overflow lists. */ size = sizeof(struct pvo_head) * moea64_pteg_count; moea64_pvo_table = (struct pvo_head *)moea64_bootstrap_alloc(size, PAGE_SIZE); CTR1(KTR_PMAP, "moea64_bootstrap: PVO table at %p", moea64_pvo_table); DISABLE_TRANS(msr); for (i = 0; i < moea64_pteg_count; i++) LIST_INIT(&moea64_pvo_table[i]); ENABLE_TRANS(msr); /* * Initialize the lock that synchronizes access to the pteg and pvo * tables. */ mtx_init(&moea64_table_mutex, "pmap table", NULL, MTX_DEF | MTX_RECURSE); mtx_init(&moea64_slb_mutex, "SLB table", NULL, MTX_DEF); /* * Initialise the unmanaged pvo pool. */ moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc( BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0); moea64_bpvo_pool_index = 0; /* * Make sure kernel vsid is allocated as well as VSID 0. */ #ifndef __powerpc64__ moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NVSIDS - 1)) / VSID_NBPW] |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); moea64_vsid_bitmap[0] |= 1; #endif /* * Initialize the kernel pmap (which is statically allocated). */ #ifdef __powerpc64__ for (i = 0; i < 64; i++) { pcpup->pc_slb[i].slbv = 0; pcpup->pc_slb[i].slbe = 0; } #else for (i = 0; i < 16; i++) kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i; #endif kernel_pmap->pmap_phys = kernel_pmap; kernel_pmap->pm_active = ~0; PMAP_LOCK_INIT(kernel_pmap); /* * Now map in all the other buffers we allocated earlier */ moea64_setup_direct_map(mmup, kernelstart, kernelend); /* * Set up the Open Firmware pmap and add its mappings if not in real * mode. */ if (!ofw_real_mode) { #ifndef __powerpc64__ moea64_pinit(mmup, &ofw_pmap); for (i = 0; i < 16; i++) ofw_pmap.pm_sr[i] = kernel_pmap->pm_sr[i]; #endif if ((chosen = OF_finddevice("/chosen")) == -1) panic("moea64_bootstrap: can't find /chosen"); OF_getprop(chosen, "mmu", &mmui, 4); if ((mmu = OF_instance_to_package(mmui)) == -1) panic("moea64_bootstrap: can't get mmu package"); if ((sz = OF_getproplen(mmu, "translations")) == -1) panic("moea64_bootstrap: can't get ofw translation count"); if (sz > 6144 /* tmpstksz - 2 KB headroom */) panic("moea64_bootstrap: too many ofw translations"); moea64_add_ofw_mappings(mmup, mmu, sz); } /* * Calculate the last available physical address. */ for (i = 0; phys_avail[i + 2] != 0; i += 2) ; Maxmem = powerpc_btop(phys_avail[i + 1]); /* * Initialize MMU and remap early physical mappings */ MMU_CPU_BOOTSTRAP(mmup,0); mtmsr(mfmsr() | PSL_DR | PSL_IR); isync(); pmap_bootstrapped++; bs_remap_earlyboot(); /* * Set the start and end of kva. */ virtual_avail = VM_MIN_KERNEL_ADDRESS; virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; /* * Map the entire KVA range into the SLB. We must not fault there. */ #ifdef __powerpc64__ for (va = virtual_avail; va < virtual_end; va += SEGMENT_LENGTH) moea64_bootstrap_slb_prefault(va, 0); #endif /* * Figure out how far we can extend virtual_end into segment 16 * without running into existing mappings. Segment 16 is guaranteed * to contain neither RAM nor devices (at least on Apple hardware), * but will generally contain some OFW mappings we should not * step on. */ #ifndef __powerpc64__ /* KVA is in high memory on PPC64 */ PMAP_LOCK(kernel_pmap); while (virtual_end < VM_MAX_KERNEL_ADDRESS && moea64_pvo_find_va(kernel_pmap, virtual_end+1) == NULL) virtual_end += PAGE_SIZE; PMAP_UNLOCK(kernel_pmap); #endif /* * Allocate a kernel stack with a guard page for thread0 and map it * into the kernel page map. */ pa = moea64_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, PAGE_SIZE); va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; virtual_avail = va + KSTACK_PAGES * PAGE_SIZE; CTR2(KTR_PMAP, "moea_bootstrap: kstack0 at %#x (%#x)", pa, va); thread0.td_kstack = va; thread0.td_kstack_pages = KSTACK_PAGES; for (i = 0; i < KSTACK_PAGES; i++) { moea64_kenter(mmup, va, pa); pa += PAGE_SIZE; va += PAGE_SIZE; } /* * Allocate virtual address space for the message buffer. */ pa = msgbuf_phys = moea64_bootstrap_alloc(MSGBUF_SIZE, PAGE_SIZE); msgbufp = (struct msgbuf *)virtual_avail; va = virtual_avail; virtual_avail += round_page(MSGBUF_SIZE); while (va < virtual_avail) { moea64_kenter(mmup, va, pa); pa += PAGE_SIZE; va += PAGE_SIZE; } /* * Allocate virtual address space for the dynamic percpu area. */ pa = moea64_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE); dpcpu = (void *)virtual_avail; va = virtual_avail; virtual_avail += DPCPU_SIZE; while (va < virtual_avail) { moea64_kenter(mmup, va, pa); pa += PAGE_SIZE; va += PAGE_SIZE; } dpcpu_init(dpcpu, 0); } /* * Activate a user pmap. The pmap must be activated before its address * space can be accessed in any way. */ void moea64_activate(mmu_t mmu, struct thread *td) { pmap_t pm; pm = &td->td_proc->p_vmspace->vm_pmap; pm->pm_active |= PCPU_GET(cpumask); #ifdef __powerpc64__ PCPU_SET(userslb, pm->pm_slb); #else PCPU_SET(curpmap, pm->pmap_phys); #endif } void moea64_deactivate(mmu_t mmu, struct thread *td) { pmap_t pm; pm = &td->td_proc->p_vmspace->vm_pmap; pm->pm_active &= ~(PCPU_GET(cpumask)); #ifdef __powerpc64__ PCPU_SET(userslb, NULL); #else PCPU_SET(curpmap, NULL); #endif } void moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired) { struct pvo_entry *pvo; struct lpte *pt; uint64_t vsid; int i, ptegidx; PMAP_LOCK(pm); pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF); if (pvo != NULL) { LOCK_TABLE(); if (moea64_pvo_to_pte_hook != NULL) pt = moea64_pvo_to_pte_hook(pvo); else pt = moea64_pvo_to_pte_native(pvo); if (wired) { if ((pvo->pvo_vaddr & PVO_WIRED) == 0) pm->pm_stats.wired_count++; pvo->pvo_vaddr |= PVO_WIRED; pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED; } else { if ((pvo->pvo_vaddr & PVO_WIRED) != 0) pm->pm_stats.wired_count--; pvo->pvo_vaddr &= ~PVO_WIRED; pvo->pvo_pte.lpte.pte_hi &= ~LPTE_WIRED; } if (pt != NULL) { /* Update wiring flag in page table. */ if (moea64_pte_change_hook != NULL) moea64_pte_change_hook(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); else moea64_pte_change_native(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); } else if (wired) { /* * If we are wiring the page, and it wasn't in the * page table before, add it. */ vsid = PVO_VSID(pvo); ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); if (moea64_pte_insert_hook != NULL) i = moea64_pte_insert_hook(ptegidx, &pvo->pvo_pte.lpte); else i = moea64_pte_insert_native(ptegidx, &pvo->pvo_pte.lpte); if (i >= 0) { PVO_PTEGIDX_CLR(pvo); PVO_PTEGIDX_SET(pvo, i); } } UNLOCK_TABLE(); } PMAP_UNLOCK(pm); } /* * This goes through and sets the physical address of our * special scratch PTE to the PA we want to zero or copy. Because * of locking issues (this can get called in pvo_enter() by * the UMA allocator), we can't use most other utility functions here */ static __inline void moea64_set_scratchpage_pa(int which, vm_offset_t pa) { KASSERT(!hw_direct_map, ("Using OEA64 scratchpage with a direct map!")); mtx_assert(&moea64_scratchpage_mtx, MA_OWNED); moea64_scratchpage_pte[which]->pte_hi &= ~LPTE_VALID; TLBIE(moea64_scratchpage_vpn[which]); moea64_scratchpage_pte[which]->pte_lo &= ~(LPTE_WIMG | LPTE_RPGN); moea64_scratchpage_pte[which]->pte_lo |= moea64_calc_wimg(pa) | (uint64_t)pa; EIEIO(); moea64_scratchpage_pte[which]->pte_hi |= LPTE_VALID; PTESYNC(); isync(); } void moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst) { vm_offset_t dst; vm_offset_t src; dst = VM_PAGE_TO_PHYS(mdst); src = VM_PAGE_TO_PHYS(msrc); if (hw_direct_map) { kcopy((void *)src, (void *)dst, PAGE_SIZE); } else { mtx_lock(&moea64_scratchpage_mtx); moea64_set_scratchpage_pa(0,src); moea64_set_scratchpage_pa(1,dst); kcopy((void *)moea64_scratchpage_va[0], (void *)moea64_scratchpage_va[1], PAGE_SIZE); mtx_unlock(&moea64_scratchpage_mtx); } } void moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size) { vm_offset_t pa = VM_PAGE_TO_PHYS(m); if (!moea64_initialized) panic("moea64_zero_page: can't zero pa %#" PRIxPTR, pa); if (size + off > PAGE_SIZE) panic("moea64_zero_page: size + off > PAGE_SIZE"); if (hw_direct_map) { bzero((caddr_t)pa + off, size); } else { mtx_lock(&moea64_scratchpage_mtx); moea64_set_scratchpage_pa(0,pa); bzero((caddr_t)moea64_scratchpage_va[0] + off, size); mtx_unlock(&moea64_scratchpage_mtx); } } /* * Zero a page of physical memory by temporarily mapping it */ void moea64_zero_page(mmu_t mmu, vm_page_t m) { vm_offset_t pa = VM_PAGE_TO_PHYS(m); vm_offset_t va, off; if (!moea64_initialized) panic("moea64_zero_page: can't zero pa %#zx", pa); if (!hw_direct_map) { mtx_lock(&moea64_scratchpage_mtx); moea64_set_scratchpage_pa(0,pa); va = moea64_scratchpage_va[0]; } else { va = pa; } for (off = 0; off < PAGE_SIZE; off += cacheline_size) __asm __volatile("dcbz 0,%0" :: "r"(va + off)); if (!hw_direct_map) mtx_unlock(&moea64_scratchpage_mtx); } void moea64_zero_page_idle(mmu_t mmu, vm_page_t m) { moea64_zero_page(mmu, m); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. */ void moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { vm_page_lock_queues(); PMAP_LOCK(pmap); moea64_enter_locked(pmap, va, m, prot, wired); vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. * * The page queues and pmap must be locked. */ static void moea64_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { struct pvo_head *pvo_head; uma_zone_t zone; vm_page_t pg; uint64_t pte_lo; u_int pvo_flags; int error; if (!moea64_initialized) { pvo_head = &moea64_pvo_kunmanaged; pg = NULL; zone = moea64_upvo_zone; pvo_flags = 0; } else { pvo_head = vm_page_to_pvoh(m); pg = m; zone = moea64_mpvo_zone; pvo_flags = PVO_MANAGED; } if (pmap_bootstrapped) mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || (m->oflags & VPO_BUSY) != 0 || VM_OBJECT_LOCKED(m->object), ("moea64_enter_locked: page %p is not busy", m)); /* XXX change the pvo head for fake pages */ if ((m->flags & PG_FICTITIOUS) == PG_FICTITIOUS) { pvo_flags &= ~PVO_MANAGED; pvo_head = &moea64_pvo_kunmanaged; zone = moea64_upvo_zone; } pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m)); if (prot & VM_PROT_WRITE) { pte_lo |= LPTE_BW; if (pmap_bootstrapped && (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) vm_page_flag_set(m, PG_WRITEABLE); } else pte_lo |= LPTE_BR; if (prot & VM_PROT_EXECUTE) pvo_flags |= VM_PROT_EXECUTE; if (wired) pvo_flags |= PVO_WIRED; if ((m->flags & PG_FICTITIOUS) != 0) pvo_flags |= PVO_FAKE; error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), pte_lo, pvo_flags); /* * Flush the page from the instruction cache if this page is * mapped executable and cacheable. */ if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE); } } static void moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz) { /* * This is much trickier than on older systems because * we can't sync the icache on physical addresses directly * without a direct map. Instead we check a couple of cases * where the memory is already mapped in and, failing that, * use the same trick we use for page zeroing to create * a temporary mapping for this physical address. */ if (!pmap_bootstrapped) { /* * If PMAP is not bootstrapped, we are likely to be * in real mode. */ __syncicache((void *)pa, sz); } else if (pmap == kernel_pmap) { __syncicache((void *)va, sz); } else if (hw_direct_map) { __syncicache((void *)pa, sz); } else { /* Use the scratch page to set up a temp mapping */ mtx_lock(&moea64_scratchpage_mtx); moea64_set_scratchpage_pa(1,pa & ~ADDR_POFF); __syncicache((void *)(moea64_scratchpage_va[1] + (va & ADDR_POFF)), sz); mtx_unlock(&moea64_scratchpage_mtx); } } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void moea64_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_page_t m; vm_pindex_t diff, psize; psize = atop(end - start); m = m_start; vm_page_lock_queues(); PMAP_LOCK(pm); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { moea64_enter_locked(pm, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); m = TAILQ_NEXT(m, listq); } vm_page_unlock_queues(); PMAP_UNLOCK(pm); } void moea64_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) { vm_page_lock_queues(); PMAP_LOCK(pm); moea64_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); vm_page_unlock_queues(); PMAP_UNLOCK(pm); } vm_paddr_t moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va) { struct pvo_entry *pvo; vm_paddr_t pa; PMAP_LOCK(pm); pvo = moea64_pvo_find_va(pm, va); if (pvo == NULL) pa = 0; else pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va - PVO_VADDR(pvo)); PMAP_UNLOCK(pm); return (pa); } /* * Atomically extract and hold the physical page with the given * pmap and virtual address pair if that mapping permits the given * protection. */ vm_page_t moea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot) { struct pvo_entry *pvo; vm_page_t m; vm_paddr_t pa; m = NULL; pa = 0; PMAP_LOCK(pmap); retry: pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF); if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW || (prot & VM_PROT_WRITE) == 0)) { if (vm_page_pa_tryrelock(pmap, pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, &pa)) goto retry; m = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); vm_page_hold(m); } PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } static void * moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) { /* * This entire routine is a horrible hack to avoid bothering kmem * for new KVA addresses. Because this can get called from inside * kmem allocation routines, calling kmem for a new address here * can lead to multiply locking non-recursive mutexes. */ static vm_pindex_t color; vm_offset_t va; vm_page_t m; int pflags, needed_lock; *flags = UMA_SLAB_PRIV; needed_lock = !PMAP_LOCKED(kernel_pmap); if (needed_lock) PMAP_LOCK(kernel_pmap); if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT) pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED; else pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED; if (wait & M_ZERO) pflags |= VM_ALLOC_ZERO; for (;;) { m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ); if (m == NULL) { if (wait & M_NOWAIT) return (NULL); VM_WAIT; } else break; } va = VM_PAGE_TO_PHYS(m); moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, &moea64_pvo_kunmanaged, va, VM_PAGE_TO_PHYS(m), LPTE_M, PVO_WIRED | PVO_BOOTSTRAP); if (needed_lock) PMAP_UNLOCK(kernel_pmap); if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) bzero((void *)va, PAGE_SIZE); return (void *)va; } void moea64_init(mmu_t mmu) { CTR0(KTR_PMAP, "moea64_init"); moea64_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); moea64_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); if (!hw_direct_map) { uma_zone_set_allocf(moea64_upvo_zone,moea64_uma_page_alloc); uma_zone_set_allocf(moea64_mpvo_zone,moea64_uma_page_alloc); } moea64_initialized = TRUE; } boolean_t moea64_is_referenced(mmu_t mmu, vm_page_t m) { KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_is_referenced: page %p is not managed", m)); return (moea64_query_bit(m, PTE_REF)); } boolean_t moea64_is_modified(mmu_t mmu, vm_page_t m) { KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_is_modified: page %p is not managed", m)); /* * If the page is not VPO_BUSY, then PG_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PG_WRITEABLE * is clear, no PTEs can have LPTE_CHG set. */ VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if ((m->oflags & VPO_BUSY) == 0 && (m->flags & PG_WRITEABLE) == 0) return (FALSE); return (moea64_query_bit(m, LPTE_CHG)); } void moea64_clear_reference(mmu_t mmu, vm_page_t m) { KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_clear_reference: page %p is not managed", m)); moea64_clear_bit(m, LPTE_REF); } void moea64_clear_modify(mmu_t mmu, vm_page_t m) { KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_clear_modify: page %p is not managed", m)); VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); KASSERT((m->oflags & VPO_BUSY) == 0, ("moea64_clear_modify: page %p is busy", m)); /* * If the page is not PG_WRITEABLE, then no PTEs can have LPTE_CHG * set. If the object containing the page is locked and the page is * not VPO_BUSY, then PG_WRITEABLE cannot be concurrently set. */ if ((m->flags & PG_WRITEABLE) == 0) return; moea64_clear_bit(m, LPTE_CHG); } /* * Clear the write and modified bits in each of the given page's mappings. */ void moea64_remove_write(mmu_t mmu, vm_page_t m) { struct pvo_entry *pvo; struct lpte *pt; pmap_t pmap; uint64_t lo; KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_remove_write: page %p is not managed", m)); /* * If the page is not VPO_BUSY, then PG_WRITEABLE cannot be set by * another thread while the object is locked. Thus, if PG_WRITEABLE * is clear, no page table entries need updating. */ VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if ((m->oflags & VPO_BUSY) == 0 && (m->flags & PG_WRITEABLE) == 0) return; vm_page_lock_queues(); lo = moea64_attr_fetch(m); SYNC(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { pmap = pvo->pvo_pmap; PMAP_LOCK(pmap); LOCK_TABLE(); if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) { if (moea64_pvo_to_pte_hook != NULL) pt = moea64_pvo_to_pte_hook(pvo); else pt = moea64_pvo_to_pte_native(pvo); pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP; pvo->pvo_pte.lpte.pte_lo |= LPTE_BR; if (pt != NULL) { if (moea64_pte_synch_hook != NULL) moea64_pte_synch_hook(pt, &pvo->pvo_pte.lpte); else moea64_pte_synch_native(pt, &pvo->pvo_pte.lpte); lo |= pvo->pvo_pte.lpte.pte_lo; pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG; if (moea64_pte_change_hook != NULL) moea64_pte_change_hook(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); else moea64_pte_change_native(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); if (pvo->pvo_pmap == kernel_pmap) isync(); } } UNLOCK_TABLE(); PMAP_UNLOCK(pmap); } if ((lo & LPTE_CHG) != 0) { moea64_attr_clear(m, LPTE_CHG); vm_page_dirty(m); } vm_page_flag_clear(m, PG_WRITEABLE); vm_page_unlock_queues(); } /* * moea64_ts_referenced: * * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ boolean_t moea64_ts_referenced(mmu_t mmu, vm_page_t m) { KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_ts_referenced: page %p is not managed", m)); return (moea64_clear_bit(m, LPTE_REF)); } /* * Map a wired page into kernel virtual address space. */ void moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa) { uint64_t pte_lo; int error; #if 0 if (!pmap_bootstrapped) { if (va >= VM_MIN_KERNEL_ADDRESS && va < virtual_end) panic("Trying to enter an address in KVA -- %#" PRIxPTR "!\n",pa); } #endif pte_lo = moea64_calc_wimg(pa); PMAP_LOCK(kernel_pmap); error = moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, &moea64_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED | VM_PROT_EXECUTE); if (error != 0 && error != ENOENT) panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va, pa, error); /* * Flush the memory from the instruction cache. */ if ((pte_lo & (LPTE_I | LPTE_G)) == 0) { __syncicache((void *)va, PAGE_SIZE); } PMAP_UNLOCK(kernel_pmap); } /* * Extract the physical page address associated with the given kernel virtual * address. */ vm_offset_t moea64_kextract(mmu_t mmu, vm_offset_t va) { struct pvo_entry *pvo; vm_paddr_t pa; /* * Shortcut the direct-mapped case when applicable. We never put * anything but 1:1 mappings below VM_MIN_KERNEL_ADDRESS. */ if (va < VM_MIN_KERNEL_ADDRESS) return (va); PMAP_LOCK(kernel_pmap); pvo = moea64_pvo_find_va(kernel_pmap, va); KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR, va)); pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) + (va - PVO_VADDR(pvo)); PMAP_UNLOCK(kernel_pmap); return (pa); } /* * Remove a wired page from kernel virtual address space. */ void moea64_kremove(mmu_t mmu, vm_offset_t va) { moea64_remove(mmu, kernel_pmap, va, va + PAGE_SIZE); } /* * Map a range of physical addresses into kernel virtual address space. * * The value passed in *virt is a suggested virtual address for the mapping. * Architectures which can support a direct-mapped physical to virtual region * can return the appropriate address within that region, leaving '*virt' * unchanged. We cannot and therefore do not; *virt is updated with the * first usable address after the mapped region. */ vm_offset_t moea64_map(mmu_t mmu, vm_offset_t *virt, vm_offset_t pa_start, vm_offset_t pa_end, int prot) { vm_offset_t sva, va; sva = *virt; va = sva; for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE) moea64_kenter(mmu, va, pa_start); *virt = va; return (sva); } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t moea64_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) { int loops; struct pvo_entry *pvo; boolean_t rv; KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_page_exists_quick: page %p is not managed", m)); loops = 0; rv = FALSE; vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { if (pvo->pvo_pmap == pmap) { rv = TRUE; break; } if (++loops >= 16) break; } vm_page_unlock_queues(); return (rv); } /* * Return the number of managed mappings to the given physical page * that are wired. */ int moea64_page_wired_mappings(mmu_t mmu, vm_page_t m) { struct pvo_entry *pvo; int count; count = 0; if ((m->flags & PG_FICTITIOUS) != 0) return (count); vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) if ((pvo->pvo_vaddr & PVO_WIRED) != 0) count++; vm_page_unlock_queues(); return (count); } static uintptr_t moea64_vsidcontext; uintptr_t moea64_get_unique_vsid(void) { u_int entropy; register_t hash; uint32_t mask; int i; entropy = 0; __asm __volatile("mftb %0" : "=r"(entropy)); mtx_lock(&moea64_slb_mutex); for (i = 0; i < NVSIDS; i += VSID_NBPW) { u_int n; /* * Create a new value by mutiplying by a prime and adding in * entropy from the timebase register. This is to make the * VSID more random so that the PT hash function collides * less often. (Note that the prime casues gcc to do shifts * instead of a multiply.) */ moea64_vsidcontext = (moea64_vsidcontext * 0x1105) + entropy; hash = moea64_vsidcontext & (NVSIDS - 1); if (hash == 0) /* 0 is special, avoid it */ continue; n = hash >> 5; mask = 1 << (hash & (VSID_NBPW - 1)); hash = (moea64_vsidcontext & VSID_HASHMASK); if (moea64_vsid_bitmap[n] & mask) { /* collision? */ /* anything free in this bucket? */ if (moea64_vsid_bitmap[n] == 0xffffffff) { entropy = (moea64_vsidcontext >> 20); continue; } i = ffs(~moea64_vsid_bitmap[i]) - 1; mask = 1 << i; hash &= VSID_HASHMASK & ~(VSID_NBPW - 1); hash |= i; } moea64_vsid_bitmap[n] |= mask; mtx_unlock(&moea64_slb_mutex); return (hash); } mtx_unlock(&moea64_slb_mutex); panic("%s: out of segments",__func__); } #ifdef __powerpc64__ void moea64_pinit(mmu_t mmu, pmap_t pmap) { PMAP_LOCK_INIT(pmap); SPLAY_INIT(&pmap->pm_slbtree); pmap->pm_slb = slb_alloc_user_cache(); } #else void moea64_pinit(mmu_t mmu, pmap_t pmap) { int i; register_t hash; PMAP_LOCK_INIT(pmap); if (pmap_bootstrapped) pmap->pmap_phys = (pmap_t)moea64_kextract(mmu, (vm_offset_t)pmap); else pmap->pmap_phys = pmap; /* * Allocate some segment registers for this pmap. */ hash = moea64_get_unique_vsid(); for (i = 0; i < 16; i++) pmap->pm_sr[i] = VSID_MAKE(i, hash); } #endif /* * Initialize the pmap associated with process 0. */ void moea64_pinit0(mmu_t mmu, pmap_t pm) { moea64_pinit(mmu, pm); bzero(&pm->pm_stats, sizeof(pm->pm_stats)); } /* * Set the physical protection on the specified range of this map as requested. */ void moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { struct pvo_entry *pvo; struct lpte *pt; CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva, eva, prot); KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap, ("moea64_protect: non current pmap")); if ((prot & VM_PROT_READ) == VM_PROT_NONE) { moea64_remove(mmu, pm, sva, eva); return; } vm_page_lock_queues(); PMAP_LOCK(pm); for (; sva < eva; sva += PAGE_SIZE) { pvo = moea64_pvo_find_va(pm, sva); if (pvo == NULL) continue; /* * Grab the PTE pointer before we diddle with the cached PTE * copy. */ LOCK_TABLE(); if (moea64_pvo_to_pte_hook != NULL) pt = moea64_pvo_to_pte_hook(pvo); else pt = moea64_pvo_to_pte_native(pvo); /* * Change the protection of the page. */ pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP; pvo->pvo_pte.lpte.pte_lo |= LPTE_BR; pvo->pvo_pte.lpte.pte_lo &= ~LPTE_NOEXEC; if ((prot & VM_PROT_EXECUTE) == 0) pvo->pvo_pte.lpte.pte_lo |= LPTE_NOEXEC; /* * If the PVO is in the page table, update that pte as well. */ if (pt != NULL) { if (moea64_pte_change_hook != NULL) moea64_pte_change_hook(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); else moea64_pte_change_native(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); if ((pvo->pvo_pte.lpte.pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { moea64_syncicache(pm, sva, pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, PAGE_SIZE); } } UNLOCK_TABLE(); } vm_page_unlock_queues(); PMAP_UNLOCK(pm); } /* * Map a list of wired pages into kernel virtual address space. This is * intended for temporary mappings which do not need page modification or * references recorded. Existing mappings in the region are overwritten. */ void moea64_qenter(mmu_t mmu, vm_offset_t va, vm_page_t *m, int count) { while (count-- > 0) { moea64_kenter(mmu, va, VM_PAGE_TO_PHYS(*m)); va += PAGE_SIZE; m++; } } /* * Remove page mappings from kernel virtual address space. Intended for * temporary mappings entered by moea64_qenter. */ void moea64_qremove(mmu_t mmu, vm_offset_t va, int count) { while (count-- > 0) { moea64_kremove(mmu, va); va += PAGE_SIZE; } } void moea64_release_vsid(uint64_t vsid) { int idx, mask; mtx_lock(&moea64_slb_mutex); idx = vsid & (NVSIDS-1); mask = 1 << (idx % VSID_NBPW); idx /= VSID_NBPW; moea64_vsid_bitmap[idx] &= ~mask; mtx_unlock(&moea64_slb_mutex); } void moea64_release(mmu_t mmu, pmap_t pmap) { /* * Free segment registers' VSIDs */ #ifdef __powerpc64__ free_vsids(pmap); slb_free_user_cache(pmap->pm_slb); #else if (pmap->pm_sr[0] == 0) panic("moea64_release: pm_sr[0] = 0"); moea64_release_vsid(pmap->pm_sr[0]); #endif PMAP_LOCK_DESTROY(pmap); } /* * Remove the given range of addresses from the specified map. */ void moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) { struct pvo_entry *pvo; vm_page_lock_queues(); PMAP_LOCK(pm); for (; sva < eva; sva += PAGE_SIZE) { pvo = moea64_pvo_find_va(pm, sva); if (pvo != NULL) moea64_pvo_remove(pvo); } vm_page_unlock_queues(); PMAP_UNLOCK(pm); } /* * Remove physical page from all pmaps in which it resides. moea64_pvo_remove() * will reflect changes in pte's back to the vm_page. */ void moea64_remove_all(mmu_t mmu, vm_page_t m) { struct pvo_head *pvo_head; struct pvo_entry *pvo, *next_pvo; pmap_t pmap; vm_page_lock_queues(); pvo_head = vm_page_to_pvoh(m); for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { next_pvo = LIST_NEXT(pvo, pvo_vlink); MOEA_PVO_CHECK(pvo); /* sanity check */ pmap = pvo->pvo_pmap; PMAP_LOCK(pmap); moea64_pvo_remove(pvo); PMAP_UNLOCK(pmap); } if ((m->flags & PG_WRITEABLE) && moea64_is_modified(mmu, m)) { moea64_attr_clear(m, LPTE_CHG); vm_page_dirty(m); } vm_page_flag_clear(m, PG_WRITEABLE); vm_page_unlock_queues(); } /* * Allocate a physical page of memory directly from the phys_avail map. * Can only be called from moea64_bootstrap before avail start and end are * calculated. */ vm_offset_t moea64_bootstrap_alloc(vm_size_t size, u_int align) { vm_offset_t s, e; int i, j; size = round_page(size); for (i = 0; phys_avail[i + 1] != 0; i += 2) { if (align != 0) s = (phys_avail[i] + align - 1) & ~(align - 1); else s = phys_avail[i]; e = s + size; if (s < phys_avail[i] || e > phys_avail[i + 1]) continue; + if (s + size > platform_real_maxaddr()) + continue; + if (s == phys_avail[i]) { phys_avail[i] += size; } else if (e == phys_avail[i + 1]) { phys_avail[i + 1] -= size; } else { for (j = phys_avail_count * 2; j > i; j -= 2) { phys_avail[j] = phys_avail[j - 2]; phys_avail[j + 1] = phys_avail[j - 1]; } phys_avail[i + 3] = phys_avail[i + 1]; phys_avail[i + 1] = s; phys_avail[i + 2] = e; phys_avail_count++; } return (s); } panic("moea64_bootstrap_alloc: could not allocate memory"); } static void tlbia(void) { vm_offset_t i; #ifndef __powerpc64__ register_t msr, scratch; #endif TLBSYNC(); for (i = 0; i < 0xFF000; i += 0x00001000) { #ifdef __powerpc64__ __asm __volatile("tlbiel %0" :: "r"(i)); #else __asm __volatile("\ mfmsr %0; \ mr %1, %0; \ insrdi %1,%3,1,0; \ mtmsrd %1; \ isync; \ \ tlbiel %2; \ \ mtmsrd %0; \ isync;" : "=r"(msr), "=r"(scratch) : "r"(i), "r"(1)); #endif } EIEIO(); TLBSYNC(); } #ifdef __powerpc64__ static void slbia(void) { register_t seg0; __asm __volatile ("slbia"); __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); } #endif static int moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head, vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags) { struct pvo_entry *pvo; uint64_t vsid; int first; u_int ptegidx; int i; int bootstrap; /* * One nasty thing that can happen here is that the UMA calls to * allocate new PVOs need to map more memory, which calls pvo_enter(), * which calls UMA... * * We break the loop by detecting recursion and allocating out of * the bootstrap pool. */ moea64_pvo_enter_calls++; first = 0; bootstrap = (flags & PVO_BOOTSTRAP); if (!moea64_initialized) bootstrap = 1; /* * Compute the PTE Group index. */ va &= ~ADDR_POFF; vsid = va_to_vsid(pm, va); ptegidx = va_to_pteg(vsid, va, flags & PVO_LARGE); /* * Remove any existing mapping for this page. Reuse the pvo entry if * there is a mapping. */ LOCK_TABLE(); LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa && (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == (pte_lo & LPTE_PP)) { if (!(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID)) { /* Re-insert if spilled */ if (moea64_pte_insert_hook != NULL) i = moea64_pte_insert_hook(ptegidx, &pvo->pvo_pte.lpte); else i = moea64_pte_insert_native(ptegidx, &pvo->pvo_pte.lpte); if (i >= 0) PVO_PTEGIDX_SET(pvo, i); moea64_pte_overflow--; } UNLOCK_TABLE(); return (0); } moea64_pvo_remove(pvo); break; } } /* * If we aren't overwriting a mapping, try to allocate. */ if (bootstrap) { if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) { panic("moea64_enter: bpvo pool exhausted, %d, %d, %zd", moea64_bpvo_pool_index, BPVO_POOL_SIZE, BPVO_POOL_SIZE * sizeof(struct pvo_entry)); } pvo = &moea64_bpvo_pool[moea64_bpvo_pool_index]; moea64_bpvo_pool_index++; bootstrap = 1; } else { /* * Note: drop the table lock around the UMA allocation in * case the UMA allocator needs to manipulate the page * table. The mapping we are working with is already * protected by the PMAP lock. */ UNLOCK_TABLE(); pvo = uma_zalloc(zone, M_NOWAIT); LOCK_TABLE(); } if (pvo == NULL) { UNLOCK_TABLE(); return (ENOMEM); } moea64_pvo_entries++; pvo->pvo_vaddr = va; pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT) | (vsid << 16); pvo->pvo_pmap = pm; LIST_INSERT_HEAD(&moea64_pvo_table[ptegidx], pvo, pvo_olink); pvo->pvo_vaddr &= ~ADDR_POFF; if (!(flags & VM_PROT_EXECUTE)) pte_lo |= LPTE_NOEXEC; if (flags & PVO_WIRED) pvo->pvo_vaddr |= PVO_WIRED; if (pvo_head != &moea64_pvo_kunmanaged) pvo->pvo_vaddr |= PVO_MANAGED; if (bootstrap) pvo->pvo_vaddr |= PVO_BOOTSTRAP; if (flags & PVO_FAKE) pvo->pvo_vaddr |= PVO_FAKE; if (flags & PVO_LARGE) pvo->pvo_vaddr |= PVO_LARGE; moea64_pte_create(&pvo->pvo_pte.lpte, vsid, va, (uint64_t)(pa) | pte_lo, flags); /* * Remember if the list was empty and therefore will be the first * item. */ if (LIST_FIRST(pvo_head) == NULL) first = 1; LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); if (pvo->pvo_vaddr & PVO_WIRED) { pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED; pm->pm_stats.wired_count++; } pm->pm_stats.resident_count++; /* * We hope this succeeds but it isn't required. */ if (moea64_pte_insert_hook != NULL) i = moea64_pte_insert_hook(ptegidx, &pvo->pvo_pte.lpte); else i = moea64_pte_insert_native(ptegidx, &pvo->pvo_pte.lpte); if (i >= 0) { PVO_PTEGIDX_SET(pvo, i); } else { panic("moea64_pvo_enter: overflow"); moea64_pte_overflow++; } if (pm == kernel_pmap) isync(); UNLOCK_TABLE(); #ifdef __powerpc64__ /* * Make sure all our bootstrap mappings are in the SLB as soon * as virtual memory is switched on. */ if (!pmap_bootstrapped) moea64_bootstrap_slb_prefault(va, flags & PVO_LARGE); #endif return (first ? ENOENT : 0); } static void moea64_pvo_remove(struct pvo_entry *pvo) { struct lpte *pt; /* * If there is an active pte entry, we need to deactivate it (and * save the ref & cfg bits). */ LOCK_TABLE(); if (moea64_pvo_to_pte_hook != NULL) pt = moea64_pvo_to_pte_hook(pvo); else pt = moea64_pvo_to_pte_native(pvo); if (pt != NULL) { if (moea64_pte_unset_hook != NULL) moea64_pte_unset_hook(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); else moea64_pte_unset_native(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); PVO_PTEGIDX_CLR(pvo); } else { moea64_pte_overflow--; } /* * Update our statistics. */ pvo->pvo_pmap->pm_stats.resident_count--; if (pvo->pvo_vaddr & PVO_WIRED) pvo->pvo_pmap->pm_stats.wired_count--; /* * Save the REF/CHG bits into their cache if the page is managed. */ if ((pvo->pvo_vaddr & (PVO_MANAGED|PVO_FAKE)) == PVO_MANAGED) { struct vm_page *pg; pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); if (pg != NULL) { moea64_attr_save(pg, pvo->pvo_pte.lpte.pte_lo & (LPTE_REF | LPTE_CHG)); } } /* * Remove this PVO from the PV list. */ LIST_REMOVE(pvo, pvo_vlink); /* * Remove this from the overflow list and return it to the pool * if we aren't going to reuse it. */ LIST_REMOVE(pvo, pvo_olink); UNLOCK_TABLE(); if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP)) uma_zfree((pvo->pvo_vaddr & PVO_MANAGED) ? moea64_mpvo_zone : moea64_upvo_zone, pvo); moea64_pvo_entries--; moea64_pvo_remove_calls++; } static struct pvo_entry * moea64_pvo_find_va(pmap_t pm, vm_offset_t va) { struct pvo_entry *pvo; int ptegidx; uint64_t vsid; #ifdef __powerpc64__ struct slb slb; /* The page is not mapped if the segment isn't */ if (va_to_slb_entry(pm, va, &slb) != 0) return NULL; vsid = (slb.slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT; if (slb.slbv & SLBV_L) va &= ~moea64_large_page_mask; else va &= ~ADDR_POFF; ptegidx = va_to_pteg(vsid, va, slb.slbv & SLBV_L); #else va &= ~ADDR_POFF; vsid = va_to_vsid(pm, va); ptegidx = va_to_pteg(vsid, va, 0); #endif LOCK_TABLE(); LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) break; } UNLOCK_TABLE(); return (pvo); } static struct lpte * moea64_pvo_to_pte_native(const struct pvo_entry *pvo) { struct lpte *pt; int pteidx, ptegidx; uint64_t vsid; ASSERT_TABLE_LOCK(); /* If the PTEG index is not set, then there is no page table entry */ if (!PVO_PTEGIDX_ISSET(pvo)) return (NULL); /* * Calculate the ptegidx */ vsid = PVO_VSID(pvo); ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); /* * We can find the actual pte entry without searching by grabbing * the PTEG index from 3 unused bits in pvo_vaddr and by * noticing the HID bit. */ if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) ptegidx ^= moea64_pteg_mask; pteidx = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo); if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { panic("moea64_pvo_to_pte: pvo %p has valid pte in pvo but no " "valid pte index", pvo); } if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { panic("moea64_pvo_to_pte: pvo %p has valid pte index in pvo " "pvo but no valid pte", pvo); } pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7]; if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) == LPTE_VALID) { if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) { panic("moea64_pvo_to_pte: pvo %p has valid pte in " "moea64_pteg_table %p but invalid in pvo", pvo, pt); } if (((pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo) & ~(LPTE_M|LPTE_CHG|LPTE_REF)) != 0) { panic("moea64_pvo_to_pte: pvo %p pte does not match " "pte %p in moea64_pteg_table difference is %#x", pvo, pt, (uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo)); } return (pt); } if (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) { panic("moea64_pvo_to_pte: pvo %p has invalid pte %p in " "moea64_pteg_table but valid in pvo", pvo, pt); } return (NULL); } static __inline int moea64_pte_spillable_ident(u_int ptegidx) { struct lpte *pt; int i, j, k; /* Start at a random slot */ i = mftb() % 8; k = -1; for (j = 0; j < 8; j++) { pt = &moea64_pteg_table[ptegidx].pt[(i + j) % 8]; if (pt->pte_hi & (LPTE_LOCKED | LPTE_WIRED)) continue; /* This is a candidate, so remember it */ k = (i + j) % 8; /* Try to get a page that has not been used lately */ if (!(pt->pte_lo & LPTE_REF)) return (k); } return (k); } static int moea64_pte_insert_native(u_int ptegidx, struct lpte *pvo_pt) { struct lpte *pt; struct pvo_entry *pvo; u_int pteg_bktidx; int i; ASSERT_TABLE_LOCK(); /* * First try primary hash. */ pteg_bktidx = ptegidx; for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) { if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) { pvo_pt->pte_hi &= ~LPTE_HID; moea64_pte_set_native(pt, pvo_pt); return (i); } } /* * Now try secondary hash. */ pteg_bktidx ^= moea64_pteg_mask; for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) { if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) { pvo_pt->pte_hi |= LPTE_HID; moea64_pte_set_native(pt, pvo_pt); return (i); } } /* * Out of luck. Find a PTE to sacrifice. */ pteg_bktidx = ptegidx; i = moea64_pte_spillable_ident(pteg_bktidx); if (i < 0) { pteg_bktidx ^= moea64_pteg_mask; i = moea64_pte_spillable_ident(pteg_bktidx); } if (i < 0) { /* No freeable slots in either PTEG? We're hosed. */ panic("moea64_pte_insert: overflow"); return (-1); } if (pteg_bktidx == ptegidx) pvo_pt->pte_hi &= ~LPTE_HID; else pvo_pt->pte_hi |= LPTE_HID; /* * Synchronize the sacrifice PTE with its PVO, then mark both * invalid. The PVO will be reused when/if the VM system comes * here after a fault. */ pt = &moea64_pteg_table[pteg_bktidx].pt[i]; if (pt->pte_hi & LPTE_HID) pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) { if (pvo->pvo_pte.lpte.pte_hi == pt->pte_hi) { KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, ("Invalid PVO for valid PTE!")); moea64_pte_unset_native(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); PVO_PTEGIDX_CLR(pvo); moea64_pte_overflow++; break; } } KASSERT(pvo->pvo_pte.lpte.pte_hi == pt->pte_hi, ("Unable to find PVO for spilled PTE")); /* * Set the new PTE. */ moea64_pte_set_native(pt, pvo_pt); return (i); } static boolean_t moea64_query_bit(vm_page_t m, u_int64_t ptebit) { struct pvo_entry *pvo; struct lpte *pt; if (moea64_attr_fetch(m) & ptebit) return (TRUE); vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { MOEA_PVO_CHECK(pvo); /* sanity check */ /* * See if we saved the bit off. If so, cache it and return * success. */ if (pvo->pvo_pte.lpte.pte_lo & ptebit) { moea64_attr_save(m, ptebit); MOEA_PVO_CHECK(pvo); /* sanity check */ vm_page_unlock_queues(); return (TRUE); } } /* * No luck, now go through the hard part of looking at the PTEs * themselves. Sync so that any pending REF/CHG bits are flushed to * the PTEs. */ SYNC(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { MOEA_PVO_CHECK(pvo); /* sanity check */ /* * See if this pvo has a valid PTE. if so, fetch the * REF/CHG bits from the valid PTE. If the appropriate * ptebit is set, cache it and return success. */ LOCK_TABLE(); if (moea64_pvo_to_pte_hook) pt = moea64_pvo_to_pte_hook(pvo); else pt = moea64_pvo_to_pte_native(pvo); if (pt != NULL) { if (moea64_pte_synch_hook != NULL) moea64_pte_synch_hook(pt, &pvo->pvo_pte.lpte); else moea64_pte_synch_native(pt, &pvo->pvo_pte.lpte); if (pvo->pvo_pte.lpte.pte_lo & ptebit) { UNLOCK_TABLE(); moea64_attr_save(m, ptebit); MOEA_PVO_CHECK(pvo); /* sanity check */ vm_page_unlock_queues(); return (TRUE); } } UNLOCK_TABLE(); } vm_page_unlock_queues(); return (FALSE); } static u_int moea64_clear_bit(vm_page_t m, u_int64_t ptebit) { u_int count; struct pvo_entry *pvo; struct lpte *pt; vm_page_lock_queues(); /* * Clear the cached value. */ moea64_attr_clear(m, ptebit); /* * Sync so that any pending REF/CHG bits are flushed to the PTEs (so * we can reset the right ones). note that since the pvo entries and * list heads are accessed via BAT0 and are never placed in the page * table, we don't have to worry about further accesses setting the * REF/CHG bits. */ SYNC(); /* * For each pvo entry, clear the pvo's ptebit. If this pvo has a * valid pte clear the ptebit from the valid pte. */ count = 0; LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { MOEA_PVO_CHECK(pvo); /* sanity check */ LOCK_TABLE(); if (moea64_pvo_to_pte_hook != NULL) pt = moea64_pvo_to_pte_hook(pvo); else pt = moea64_pvo_to_pte_native(pvo); if (pt != NULL) { if (moea64_pte_synch_hook != NULL) moea64_pte_synch_hook(pt, &pvo->pvo_pte.lpte); else moea64_pte_synch_native(pt, &pvo->pvo_pte.lpte); if (pvo->pvo_pte.lpte.pte_lo & ptebit) { count++; if (moea64_pte_clear_hook != NULL) moea64_pte_clear_hook(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn, ptebit); else moea64_pte_clear_native(pt, pvo->pvo_vpn, ptebit); } } pvo->pvo_pte.lpte.pte_lo &= ~ptebit; MOEA_PVO_CHECK(pvo); /* sanity check */ UNLOCK_TABLE(); } vm_page_unlock_queues(); return (count); } boolean_t moea64_dev_direct_mapped(mmu_t mmu, vm_offset_t pa, vm_size_t size) { struct pvo_entry *pvo; vm_offset_t ppa; int error = 0; PMAP_LOCK(kernel_pmap); for (ppa = pa & ~ADDR_POFF; ppa < pa + size; ppa += PAGE_SIZE) { pvo = moea64_pvo_find_va(kernel_pmap, ppa); if (pvo == NULL || (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) != ppa) { error = EFAULT; break; } } PMAP_UNLOCK(kernel_pmap); return (error); } /* * Map a set of physical memory pages into the kernel virtual * address space. Return a pointer to where it is mapped. This * routine is intended to be used for mapping device memory, * NOT real memory. */ void * moea64_mapdev(mmu_t mmu, vm_offset_t pa, vm_size_t size) { vm_offset_t va, tmpva, ppa, offset; ppa = trunc_page(pa); offset = pa & PAGE_MASK; size = roundup(offset + size, PAGE_SIZE); va = kmem_alloc_nofault(kernel_map, size); if (!va) panic("moea64_mapdev: Couldn't alloc kernel virtual memory"); for (tmpva = va; size > 0;) { moea64_kenter(mmu, tmpva, ppa); size -= PAGE_SIZE; tmpva += PAGE_SIZE; ppa += PAGE_SIZE; } return ((void *)(va + offset)); } void moea64_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size) { vm_offset_t base, offset; base = trunc_page(va); offset = va & PAGE_MASK; size = roundup(offset + size, PAGE_SIZE); kmem_free(kernel_map, base, size); } void moea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) { struct pvo_entry *pvo; vm_offset_t lim; vm_paddr_t pa; vm_size_t len; PMAP_LOCK(pm); while (sz > 0) { lim = round_page(va); len = MIN(lim - va, sz); pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF); if (pvo != NULL) { pa = (pvo->pvo_pte.pte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF); moea64_syncicache(pm, va, pa, len); } va += len; sz -= len; } PMAP_UNLOCK(pm); } Index: user/nwhitehorn/ps3/powerpc/aim/slb.c =================================================================== --- user/nwhitehorn/ps3/powerpc/aim/slb.c (revision 212100) +++ user/nwhitehorn/ps3/powerpc/aim/slb.c (revision 212101) @@ -1,296 +1,314 @@ /*- * Copyright (c) 2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include uintptr_t moea64_get_unique_vsid(void); void moea64_release_vsid(uint64_t vsid); +void *uma_small_alloc_core(uma_zone_t zone, int bytes, u_int8_t *flags, + int wait, vm_offset_t minphys, vm_offset_t maxphys); struct slbcontainer { struct slb slb; SPLAY_ENTRY(slbcontainer) slb_node; }; static int slb_compare(struct slbcontainer *a, struct slbcontainer *b); static void slb_zone_init(void *); SPLAY_PROTOTYPE(slb_tree, slbcontainer, slb_node, slb_compare); SPLAY_GENERATE(slb_tree, slbcontainer, slb_node, slb_compare); uma_zone_t slb_zone; uma_zone_t slb_cache_zone; SYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL); int va_to_slb_entry(pmap_t pm, vm_offset_t va, struct slb *slb) { struct slbcontainer cont, *found; uint64_t esid; esid = (uintptr_t)va >> ADDR_SR_SHFT; slb->slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; if (pm == kernel_pmap) { /* Set kernel VSID to deterministic value */ slb->slbv = va_to_vsid(kernel_pmap, va) << SLBV_VSID_SHIFT; /* Figure out if this is a large-page mapping */ if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) { /* * XXX: If we have set up a direct map, assumes * all physical memory is mapped with large pages. */ if (mem_valid(va, 0) == 0) slb->slbv |= SLBV_L; } return (0); } PMAP_LOCK_ASSERT(pm, MA_OWNED); cont.slb.slbe = slb->slbe; found = SPLAY_FIND(slb_tree, &pm->pm_slbtree, &cont); if (found == NULL) return (-1); slb->slbv = found->slb.slbv; return (0); } uint64_t va_to_vsid(pmap_t pm, vm_offset_t va) { struct slb entry; /* Shortcut kernel case */ if (pm == kernel_pmap) return (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)); /* * If there is no vsid for this VA, we need to add a new entry * to the PMAP's segment table. */ if (va_to_slb_entry(pm, va, &entry) != 0) return (allocate_vsid(pm, (uintptr_t)va >> ADDR_SR_SHFT, 0)); return ((entry.slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT); } uint64_t allocate_vsid(pmap_t pm, uint64_t esid, int large) { uint64_t vsid; struct slbcontainer *slb_entry, kern_entry; struct slb *prespill; prespill = NULL; if (pm == kernel_pmap) { vsid = va_to_vsid(pm, esid << ADDR_SR_SHFT); slb_entry = &kern_entry; prespill = PCPU_GET(slb); } else { vsid = moea64_get_unique_vsid(); slb_entry = uma_zalloc(slb_zone, M_NOWAIT); if (slb_entry == NULL) panic("Could not allocate SLB mapping!"); prespill = pm->pm_slb; } slb_entry->slb.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; slb_entry->slb.slbv = vsid << SLBV_VSID_SHIFT; if (large) slb_entry->slb.slbv |= SLBV_L; if (pm != kernel_pmap) { PMAP_LOCK_ASSERT(pm, MA_OWNED); SPLAY_INSERT(slb_tree, &pm->pm_slbtree, slb_entry); } /* * Someone probably wants this soon, and it may be a wired * SLB mapping, so pre-spill this entry. */ if (prespill != NULL) slb_insert(pm, prespill, &slb_entry->slb); return (vsid); } /* Lock entries mapping kernel text and stacks */ #define SLB_SPILLABLE(slbe) \ (((slbe & SLBE_ESID_MASK) < VM_MIN_KERNEL_ADDRESS && \ (slbe & SLBE_ESID_MASK) > 16*SEGMENT_LENGTH) || \ (slbe & SLBE_ESID_MASK) > VM_MAX_KERNEL_ADDRESS) void slb_insert(pmap_t pm, struct slb *slbcache, struct slb *slb_entry) { uint64_t slbe, slbv; int i, j, to_spill; /* We don't want to be preempted while modifying the kernel map */ critical_enter(); to_spill = -1; slbv = slb_entry->slbv; slbe = slb_entry->slbe; /* Hunt for a likely candidate */ for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) { if (pm == kernel_pmap && i == USER_SR) continue; if (!(slbcache[i].slbe & SLBE_VALID)) { to_spill = i; break; } if (to_spill < 0 && (pm != kernel_pmap || SLB_SPILLABLE(slbcache[i].slbe))) to_spill = i; } if (to_spill < 0) panic("SLB spill on ESID %#lx, but no available candidates!\n", (slbe & SLBE_ESID_MASK) >> SLBE_ESID_SHIFT); if (slbcache[to_spill].slbe & SLBE_VALID) { /* Invalidate this first to avoid races */ slbcache[to_spill].slbe = 0; mb(); } slbcache[to_spill].slbv = slbv; slbcache[to_spill].slbe = slbe | (uint64_t)to_spill; /* If it is for this CPU, put it in the SLB right away */ if (pm == kernel_pmap && pmap_bootstrapped) { /* slbie not required */ __asm __volatile ("slbmte %0, %1" :: "r"(slbcache[to_spill].slbv), "r"(slbcache[to_spill].slbe)); } critical_exit(); } int vsid_to_esid(pmap_t pm, uint64_t vsid, uint64_t *esid) { uint64_t slbv; struct slbcontainer *entry; #ifdef INVARIANTS if (pm == kernel_pmap) panic("vsid_to_esid only works on user pmaps"); PMAP_LOCK_ASSERT(pm, MA_OWNED); #endif slbv = vsid << SLBV_VSID_SHIFT; SPLAY_FOREACH(entry, slb_tree, &pm->pm_slbtree) { if (slbv == entry->slb.slbv) { *esid = entry->slb.slbe >> SLBE_ESID_SHIFT; return (0); } } return (-1); } void free_vsids(pmap_t pm) { struct slbcontainer *entry; while (!SPLAY_EMPTY(&pm->pm_slbtree)) { entry = SPLAY_MIN(slb_tree, &pm->pm_slbtree); SPLAY_REMOVE(slb_tree, &pm->pm_slbtree, entry); moea64_release_vsid(entry->slb.slbv >> SLBV_VSID_SHIFT); uma_zfree(slb_zone, entry); } } static int slb_compare(struct slbcontainer *a, struct slbcontainer *b) { if (a->slb.slbe == b->slb.slbe) return (0); else if (a->slb.slbe < b->slb.slbe) return (-1); else return (1); } +static void * +slb_uma_cache_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) +{ + static vm_offset_t realmax = 0; + + if (realmax == 0) + realmax = platform_real_maxaddr(); + + return (uma_small_alloc_core(zone, bytes, flags, wait, 0, realmax)); +} + + static void slb_zone_init(void *dummy) { slb_zone = uma_zcreate("SLB segment", sizeof(struct slbcontainer), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); + + if (platform_real_maxaddr() != VM_MAX_ADDRESS) + uma_zone_set_allocf(slb_cache_zone, slb_uma_cache_alloc); } struct slb * slb_alloc_user_cache(void) { return (uma_zalloc(slb_cache_zone, M_ZERO)); } void slb_free_user_cache(struct slb *slb) { uma_zfree(slb_cache_zone, slb); } Index: user/nwhitehorn/ps3/powerpc/aim/uma_machdep.c =================================================================== --- user/nwhitehorn/ps3/powerpc/aim/uma_machdep.c (revision 212100) +++ user/nwhitehorn/ps3/powerpc/aim/uma_machdep.c (revision 212101) @@ -1,103 +1,120 @@ /*- * Copyright (c) 2003 The FreeBSD Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #include static int hw_uma_mdpages; SYSCTL_INT(_hw, OID_AUTO, uma_mdpages, CTLFLAG_RD, &hw_uma_mdpages, 0, "UMA MD pages in use"); +void *uma_small_alloc_core(uma_zone_t zone, int bytes, u_int8_t *flags, + int wait, vm_offset_t minphys, vm_offset_t maxphys); + void * uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) { + return (uma_small_alloc_core(zone, bytes, flags, wait, + 0, VM_MAX_ADDRESS)); +} + +void * +uma_small_alloc_core(uma_zone_t zone, int bytes, u_int8_t *flags, int wait, + vm_offset_t minphys, vm_offset_t maxphys) +{ static vm_pindex_t color; void *va; vm_page_t m; int pflags; *flags = UMA_SLAB_PRIV; if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT) pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED; else pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED; if (wait & M_ZERO) pflags |= VM_ALLOC_ZERO; for (;;) { - m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ); + if (minphys == 0 && maxphys == VM_MAX_ADDRESS) + m = vm_page_alloc(NULL, color++, + pflags | VM_ALLOC_NOOBJ); + else + m = vm_phys_alloc_contig(1, minphys, maxphys, PAGE_SIZE, + PAGE_SIZE); if (m == NULL) { if (wait & M_NOWAIT) return (NULL); VM_WAIT; } else break; } va = (void *) VM_PAGE_TO_PHYS(m); if (!hw_direct_map) pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m)); if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) bzero(va, PAGE_SIZE); atomic_add_int(&hw_uma_mdpages, 1); return (va); } void uma_small_free(void *mem, int size, u_int8_t flags) { vm_page_t m; if (!hw_direct_map) pmap_remove(kernel_pmap,(vm_offset_t)mem, (vm_offset_t)mem + PAGE_SIZE); m = PHYS_TO_VM_PAGE((vm_offset_t)mem); m->wire_count--; vm_page_free(m); atomic_subtract_int(&cnt.v_wire_count, 1); atomic_subtract_int(&hw_uma_mdpages, 1); } Index: user/nwhitehorn/ps3/powerpc/include/platform.h =================================================================== --- user/nwhitehorn/ps3/powerpc/include/platform.h (revision 212100) +++ user/nwhitehorn/ps3/powerpc/include/platform.h (revision 212101) @@ -1,58 +1,59 @@ /*- * Copyright (C) 1996 Wolfgang Solfrank. * Copyright (C) 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $NetBSD: powerpc.h,v 1.3 2000/06/01 00:49:59 matt Exp $ * $FreeBSD$ */ #ifndef _MACHINE_PLATFORM_H_ #define _MACHINE_PLATFORM_H_ #include #include struct mem_region { vm_offset_t mr_start; vm_size_t mr_size; }; void mem_regions(struct mem_region **, int *, struct mem_region **, int *); +vm_offset_t platform_real_maxaddr(void); u_long platform_timebase_freq(struct cpuref *); int platform_smp_first_cpu(struct cpuref *); int platform_smp_next_cpu(struct cpuref *); int platform_smp_get_bsp(struct cpuref *); int platform_smp_start_cpu(struct pcpu *); const char *installed_platform(void); void platform_probe_and_attach(void); #endif /* _MACHINE_PLATFORM_H_ */ Index: user/nwhitehorn/ps3/powerpc/include/vmparam.h =================================================================== --- user/nwhitehorn/ps3/powerpc/include/vmparam.h (revision 212100) +++ user/nwhitehorn/ps3/powerpc/include/vmparam.h (revision 212101) @@ -1,205 +1,209 @@ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $NetBSD: vmparam.h,v 1.11 2000/02/11 19:25:16 thorpej Exp $ * $FreeBSD$ */ #ifndef _MACHINE_VMPARAM_H_ #define _MACHINE_VMPARAM_H_ #define USRSTACK VM_MAXUSER_ADDRESS #ifndef MAXTSIZ #define MAXTSIZ (64*1024*1024) /* max text size */ #endif #ifndef DFLDSIZ #define DFLDSIZ (128*1024*1024) /* default data size */ #endif #ifndef MAXDSIZ #define MAXDSIZ (1*1024*1024*1024) /* max data size */ #endif #ifndef DFLSSIZ #define DFLSSIZ (8*1024*1024) /* default stack size */ #endif #ifndef MAXSSIZ #define MAXSSIZ (64*1024*1024) /* max stack size */ #endif /* * The time for a process to be blocked before being very swappable. * This is a number of seconds which the system takes as being a non-trivial * amount of real time. You probably shouldn't change this; * it is used in subtle ways (fractions and multiples of it are, that is, like * half of a ``long time'', almost a long time, etc.) * It is related to human patience and other factors which don't really * change over time. */ #define MAXSLP 20 /* * Would like to have MAX addresses = 0, but this doesn't (currently) work */ #if !defined(LOCORE) #ifdef __powerpc64__ #define VM_MIN_ADDRESS (0x0000000000000000UL) #define VM_MAXUSER_ADDRESS (0x7ffffffffffff000UL) #define VM_MAX_ADDRESS (0xffffffffffffffffUL) #else #define VM_MIN_ADDRESS ((vm_offset_t)0) #define VM_MAXUSER_ADDRESS ((vm_offset_t)0x7ffff000) #define VM_MAX_ADDRESS VM_MAXUSER_ADDRESS #endif #else /* LOCORE */ #ifndef __powerpc64__ #define VM_MIN_ADDRESS 0 #define VM_MAXUSER_ADDRESS 0x7ffff000 #endif #endif /* LOCORE */ #define FREEBSD32_USRSTACK 0x7ffff000 #ifdef AIM #define KERNBASE 0x00100000UL /* start of kernel virtual */ #ifdef __powerpc64__ #define VM_MIN_KERNEL_ADDRESS 0xc000000000000000UL #define VM_MAX_KERNEL_ADDRESS 0xc0000001c7ffffffUL #define VM_MAX_SAFE_KERNEL_ADDRESS VM_MAX_KERNEL_ADDRESS #else #define VM_MIN_KERNEL_ADDRESS ((vm_offset_t)KERNEL_SR << ADDR_SR_SHFT) #define VM_MAX_SAFE_KERNEL_ADDRESS (VM_MIN_KERNEL_ADDRESS + 2*SEGMENT_LENGTH -1) #define VM_MAX_KERNEL_ADDRESS (VM_MIN_KERNEL_ADDRESS + 3*SEGMENT_LENGTH - 1) #endif /* * Use the direct-mapped BAT registers for UMA small allocs. This * takes pressure off the small amount of available KVA. */ #define UMA_MD_SMALL_ALLOC #else /* Book-E */ /* * Kernel CCSRBAR location. We make this the reset location. */ #define CCSRBAR_VA 0xfef00000 #define CCSRBAR_SIZE 0x00100000 #define KERNBASE 0xc0000000 /* start of kernel virtual */ #define VM_MIN_KERNEL_ADDRESS KERNBASE #define VM_MAX_KERNEL_ADDRESS 0xf8000000 #endif /* AIM/E500 */ /* XXX max. amount of KVM to be used by buffers. */ #ifndef VM_MAX_KERNEL_BUF #define VM_MAX_KERNEL_BUF (SEGMENT_LENGTH * 7 / 10) #endif #if !defined(LOCORE) struct pmap_physseg { struct pv_entry *pvent; char *attrs; }; #endif #define VM_PHYSSEG_MAX 16 /* 1? */ /* * The physical address space is densely populated. */ +#ifdef __powerpc64__ +#define VM_PHYSSEG_SPARSE +#else #define VM_PHYSSEG_DENSE +#endif /* * Create three free page pools: VM_FREEPOOL_DEFAULT is the default pool * from which physical pages are allocated and VM_FREEPOOL_DIRECT is * the pool from which physical pages for small UMA objects are * allocated. */ #define VM_NFREEPOOL 3 #define VM_FREEPOOL_CACHE 2 #define VM_FREEPOOL_DEFAULT 0 #define VM_FREEPOOL_DIRECT 1 /* * Create one free page list. */ #define VM_NFREELIST 1 #define VM_FREELIST_DEFAULT 0 /* * The largest allocation size is 4MB. */ #define VM_NFREEORDER 11 /* * Only one memory domain. */ #ifndef VM_NDOMAIN #define VM_NDOMAIN 1 #endif /* * Disable superpage reservations. */ #ifndef VM_NRESERVLEVEL #define VM_NRESERVLEVEL 0 #endif #ifndef VM_INITIAL_PAGEIN #define VM_INITIAL_PAGEIN 16 #endif #ifndef SGROWSIZ #define SGROWSIZ (128UL*1024) /* amount to grow stack */ #endif #ifndef VM_KMEM_SIZE #define VM_KMEM_SIZE (12 * 1024 * 1024) #endif #ifdef __powerpc64__ #ifndef VM_KMEM_SIZE_SCALE #define VM_KMEM_SIZE_SCALE (3) #endif #ifndef VM_KMEM_SIZE_MAX #define VM_KMEM_SIZE_MAX 0x1c0000000 /* 7 GB */ #endif #endif #endif /* _MACHINE_VMPARAM_H_ */ Index: user/nwhitehorn/ps3/powerpc/powerpc/platform.c =================================================================== --- user/nwhitehorn/ps3/powerpc/powerpc/platform.c (revision 212100) +++ user/nwhitehorn/ps3/powerpc/powerpc/platform.c (revision 212101) @@ -1,188 +1,194 @@ /*- * Copyright (c) 2005 Peter Grehan * Copyright (c) 2009 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); /* * Dispatch platform calls to the appropriate platform implementation * through a previously registered kernel object. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "platform_if.h" static platform_def_t *plat_def_impl; static platform_t plat_obj; static struct kobj_ops plat_kernel_kops; static struct platform_kobj plat_kernel_obj; static char plat_name[64] = ""; SYSCTL_STRING(_hw, OID_AUTO, platform, CTLFLAG_RD | CTLFLAG_TUN, plat_name, 0, "Platform currently in use"); void mem_regions(struct mem_region **phys, int *physsz, struct mem_region **avail, int *availsz) { PLATFORM_MEM_REGIONS(plat_obj, phys, physsz, avail, availsz); } +vm_offset_t +platform_real_maxaddr(void) +{ + return (PLATFORM_REAL_MAXADDR(plat_obj)); +} + const char * installed_platform() { return (plat_def_impl->name); } u_long platform_timebase_freq(struct cpuref *cpu) { return (PLATFORM_TIMEBASE_FREQ(plat_obj, cpu)); } int platform_smp_first_cpu(struct cpuref *cpu) { return (PLATFORM_SMP_FIRST_CPU(plat_obj, cpu)); } int platform_smp_next_cpu(struct cpuref *cpu) { return (PLATFORM_SMP_NEXT_CPU(plat_obj, cpu)); } int platform_smp_get_bsp(struct cpuref *cpu) { return (PLATFORM_SMP_GET_BSP(plat_obj, cpu)); } int platform_smp_start_cpu(struct pcpu *cpu) { return (PLATFORM_SMP_START_CPU(plat_obj, cpu)); } /* * Reset back to firmware. */ void cpu_reset() { PLATFORM_RESET(plat_obj); } /* * Platform install routines. Highest priority wins, using the same * algorithm as bus attachment. */ SET_DECLARE(platform_set, platform_def_t); void platform_probe_and_attach() { platform_def_t **platpp, *platp; int prio, best_prio; plat_obj = &plat_kernel_obj; best_prio = 0; /* * Try to locate the best platform kobj */ SET_FOREACH(platpp, platform_set) { platp = *platpp; /* * Take care of compiling the selected class, and * then statically initialise the MMU object */ kobj_class_compile_static(platp, &plat_kernel_kops); kobj_init((kobj_t)plat_obj, platp); prio = PLATFORM_PROBE(plat_obj); /* Check for errors */ if (prio > 0) continue; /* * Check if this module was specifically requested through * the loader tunable we provide. */ if (strcmp(platp->name,plat_name) == 0) { plat_def_impl = platp; break; } /* Otherwise, see if it is better than our current best */ if (plat_def_impl == NULL || prio > best_prio) { best_prio = prio; plat_def_impl = platp; } /* * We can't free the KOBJ, since it is static. Reset the ops * member of this class so that we can come back later. */ platp->ops = NULL; } if (plat_def_impl == NULL) panic("No platform module found!"); /* * Recompile to make sure we ended with the * correct one, and then attach. */ kobj_class_compile_static(plat_def_impl, &plat_kernel_kops); kobj_init((kobj_t)plat_obj, plat_def_impl); strlcpy(plat_name,plat_def_impl->name,sizeof(plat_name)); PLATFORM_ATTACH(plat_obj); } Index: user/nwhitehorn/ps3/powerpc/powerpc/platform_if.m =================================================================== --- user/nwhitehorn/ps3/powerpc/powerpc/platform_if.m (revision 212100) +++ user/nwhitehorn/ps3/powerpc/powerpc/platform_if.m (revision 212101) @@ -1,170 +1,184 @@ #- # Copyright (c) 2009 Nathan Whitehorn # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # #include #include #include #include #include #include #include +#include /** * @defgroup PLATFORM platform - KObj methods for PowerPC platform * implementations * @brief A set of methods required by all platform implementations. * These are used to bring up secondary CPUs, supply the physical memory * map, etc. *@{ */ INTERFACE platform; # # Default implementations # CODE { static void platform_null_attach(platform_t plat) { return; } static int platform_null_smp_first_cpu(platform_t plat, struct cpuref *cpuref) { cpuref->cr_hwref = -1; cpuref->cr_cpuid = 0; return (0); } static int platform_null_smp_next_cpu(platform_t plat, struct cpuref *_cpuref) { return (ENOENT); } + static vm_offset_t platform_null_real_maxaddr(platform_t plat) + { + return (VM_MAX_ADDRESS); + } }; /** * @brief Probe for whether we are on this platform, returning the standard * newbus probe codes. If we have Open Firmware or a flattened device tree, * it is guaranteed to be available at this point. */ METHOD int probe { platform_t _plat; }; /** * @brief Attach this platform module. This happens before the MMU is online, * so the platform module can install its own high-priority MMU module at * this point. */ METHOD int attach { platform_t _plat; } DEFAULT platform_null_attach; /** * @brief Return the system's physical memory map. * * It shall provide the total and the available regions of RAM. * The available regions need not take the kernel into account. * * @param _memp Array of physical memory chunks * @param _memsz Number of physical memory chunks * @param _availp Array of available physical memory chunks * @param _availsz Number of available physical memory chunks */ METHOD void mem_regions { platform_t _plat; struct mem_region **_memp; int *_memsz; struct mem_region **_availp; int *_availsz; }; + +/** + * @brief Return the maximum address accessible in real mode + * (for use with hypervisors) + */ +METHOD vm_offset_t real_maxaddr { + platform_t _plat; +} DEFAULT platform_null_real_maxaddr; + /** * @brief Get the CPU's timebase frequency, in ticks per second. * * @param _cpu CPU whose timebase to query */ METHOD u_long timebase_freq { platform_t _plat; struct cpuref *_cpu; }; # SMP bits /** * @brief Fill the first CPU's cpuref * * @param _cpuref CPU */ METHOD int smp_first_cpu { platform_t _plat; struct cpuref *_cpuref; } DEFAULT platform_null_smp_first_cpu; /** * @brief Fill the next CPU's cpuref * * @param _cpuref CPU */ METHOD int smp_next_cpu { platform_t _plat; struct cpuref *_cpuref; } DEFAULT platform_null_smp_next_cpu; /** * @brief Find the boot processor * * @param _cpuref CPU */ METHOD int smp_get_bsp { platform_t _plat; struct cpuref *_cpuref; } DEFAULT platform_null_smp_first_cpu; /** * @brief Start a CPU * * @param _cpuref CPU */ METHOD int smp_start_cpu { platform_t _plat; struct pcpu *_cpu; }; /** * @brief Reset system */ METHOD void reset { platform_t _plat; }; Index: user/nwhitehorn/ps3/powerpc/ps3/if_glc.c =================================================================== --- user/nwhitehorn/ps3/powerpc/ps3/if_glc.c (revision 212100) +++ user/nwhitehorn/ps3/powerpc/ps3/if_glc.c (revision 212101) @@ -1,710 +1,739 @@ /*- * Copyright (C) 2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: user/nwhitehorn/ps3/powerpc/ofw/ofw_cpu.c 193156 2009-05-31 09:01:23Z nwhitehorn $ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #include "ps3bus.h" #include "ps3-hvcall.h" #include "if_glcreg.h" static int glc_probe(device_t); static int glc_attach(device_t); static void glc_init(void *xsc); static void glc_start(struct ifnet *ifp); static int glc_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data); static int glc_add_rxbuf(struct glc_softc *sc, int idx); static int glc_add_rxbuf_dma(struct glc_softc *sc, int idx); static int glc_encap(struct glc_softc *sc, struct mbuf **m_head, bus_addr_t *pktdesc); static void glc_intr(void *xsc); static MALLOC_DEFINE(M_GLC, "gelic", "PS3 GELIC ethernet"); static device_method_t glc_methods[] = { /* Device interface */ DEVMETHOD(device_probe, glc_probe), DEVMETHOD(device_attach, glc_attach), { 0, 0 } }; static driver_t glc_driver = { "glc", glc_methods, sizeof(struct glc_softc) }; static devclass_t glc_devclass; DRIVER_MODULE(glc, ps3bus, glc_driver, glc_devclass, 0, 0); static int glc_probe(device_t dev) { if (ps3bus_get_bustype(dev) != PS3_BUSTYPE_SYSBUS || ps3bus_get_devtype(dev) != PS3_DEVTYPE_GELIC) return (ENXIO); device_set_desc(dev, "Playstation 3 GELIC Network Controller"); return (BUS_PROBE_SPECIFIC); } static void glc_getphys(void *xaddr, bus_dma_segment_t *segs, int nsegs, int error) { if (error != 0) return; *(bus_addr_t *)xaddr = segs[0].ds_addr; } +static bus_addr_t +glc_map_addr(struct glc_softc *sc, bus_addr_t lpar_addr) +{ + static struct mem_region *regions = NULL; + static int rcount; + int i; + + if (regions == NULL) + mem_regions(®ions, &rcount, ®ions, &rcount); + + for (i = 0; i < rcount; i++) { + if (lpar_addr >= regions[i].mr_start && + lpar_addr < regions[i].mr_start + regions[i].mr_size) + break; + } + + if (i == rcount) + panic("glc: unable to map address %#lx", lpar_addr); + + return (sc->sc_dma_base[i] + (lpar_addr - regions[i].mr_start)); +} + static int glc_attach(device_t dev) { struct glc_softc *sc; struct glc_txsoft *txs; + struct mem_region *regions; uint64_t mac64, val, junk; - int i, err; + int i, err, rcount; sc = device_get_softc(dev); sc->sc_bus = ps3bus_get_bus(dev); sc->sc_dev = ps3bus_get_device(dev); sc->sc_self = dev; mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, MTX_DEF); sc->next_txdma_slot = 0; sc->first_used_txdma_slot = -1; /* * Shut down existing tasks. */ lv1_net_stop_tx_dma(sc->sc_bus, sc->sc_dev, 0); lv1_net_stop_rx_dma(sc->sc_bus, sc->sc_dev, 0); sc->sc_ifp = if_alloc(IFT_ETHER); sc->sc_ifp->if_softc = sc; /* * Get MAC address and VLAN id */ lv1_net_control(sc->sc_bus, sc->sc_dev, GELIC_GET_MAC_ADDRESS, 0, 0, 0, &mac64, &junk); memcpy(sc->sc_enaddr, &((uint8_t *)&mac64)[2], sizeof(sc->sc_enaddr)); sc->sc_tx_vlan = sc->sc_rx_vlan = -1; err = lv1_net_control(sc->sc_bus, sc->sc_dev, GELIC_GET_VLAN_ID, GELIC_VLAN_TX_ETHERNET, 0, 0, &val, &junk); if (err == 0) sc->sc_tx_vlan = val; err = lv1_net_control(sc->sc_bus, sc->sc_dev, GELIC_GET_VLAN_ID, GELIC_VLAN_RX_ETHERNET, 0, 0, &val, &junk); if (err == 0) sc->sc_rx_vlan = val; /* * Set up interrupt handler */ sc->sc_irqid = 0; sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irqid, RF_ACTIVE); if (sc->sc_irq == NULL) { device_printf(dev, "Could not allocate IRQ!\n"); mtx_destroy(&sc->sc_mtx); return (ENXIO); } bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_MISC | INTR_MPSAFE | INTR_ENTROPY, NULL, glc_intr, sc, &sc->sc_irqctx); sc->sc_interrupt_status = (uint64_t *)contigmalloc(8, M_GLC, M_ZERO, 0, BUS_SPACE_MAXADDR_32BIT, 8, PAGE_SIZE); lv1_net_set_interrupt_status_indicator(sc->sc_bus, sc->sc_dev, vtophys(sc->sc_interrupt_status), 0); lv1_net_set_interrupt_mask(sc->sc_bus, sc->sc_dev, GELIC_INT_RXDONE | GELIC_INT_TXDONE | GELIC_INT_RXFRAME | GELIC_INT_PHY, 0); /* * Set up DMA. */ /* XXX: following should be integrated to busdma */ - err = lv1_allocate_device_dma_region(sc->sc_bus, sc->sc_dev, - 0x8000000 /* 128 MB */, 24 /* log_2(16 MB) */, - 0 /* 32-bit transfers */, &sc->sc_dma_base); - if (err != 0) { - device_printf(dev, "could not allocate DMA region: %d\n", err); - goto fail; - } + mem_regions(®ions, &rcount, ®ions, &rcount); + for (i = 0; i < rcount; i++) { + err = lv1_allocate_device_dma_region(sc->sc_bus, sc->sc_dev, + regions[i].mr_size, 24 /* log_2(16 MB) */, + 0 /* 32-bit transfers */, &sc->sc_dma_base[i]); + if (err != 0) { + device_printf(dev, + "could not allocate DMA region %d: %d\n", i, err); + goto fail; + } - err = lv1_map_device_dma_region(sc->sc_bus, sc->sc_dev, 0 /* physmem */, - sc->sc_dma_base, 0x8000000 /* 128 MB */, - 0xf800000000000000UL /* see Cell IO/MMU docs */); - if (err != 0) { - device_printf(dev, "could not map DMA region: %d\n", err); - goto fail; + err = lv1_map_device_dma_region(sc->sc_bus, sc->sc_dev, + regions[i].mr_start, sc->sc_dma_base[i], regions[i].mr_size, + 0xf800000000000000UL /* see Cell IO/MMU docs */); + if (err != 0) { + device_printf(dev, + "could not map DMA region %d: %d\n", i, err); + goto fail; + } } err = bus_dma_tag_create(bus_get_dma_tag(dev), 32, 0, - BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 129*sizeof(struct glc_dmadesc), 1, 128*sizeof(struct glc_dmadesc), 0, NULL,NULL, &sc->sc_dmadesc_tag); err = bus_dmamem_alloc(sc->sc_dmadesc_tag, (void **)&sc->sc_txdmadesc, BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->sc_txdmadesc_map); err = bus_dmamap_load(sc->sc_dmadesc_tag, sc->sc_txdmadesc_map, sc->sc_txdmadesc, 128*sizeof(struct glc_dmadesc), glc_getphys, &sc->sc_txdmadesc_phys, 0); err = bus_dmamem_alloc(sc->sc_dmadesc_tag, (void **)&sc->sc_rxdmadesc, BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->sc_rxdmadesc_map); err = bus_dmamap_load(sc->sc_dmadesc_tag, sc->sc_rxdmadesc_map, sc->sc_rxdmadesc, 128*sizeof(struct glc_dmadesc), glc_getphys, &sc->sc_rxdmadesc_phys, 0); err = bus_dma_tag_create(bus_get_dma_tag(dev), 128, 0, - BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0, NULL,NULL, &sc->sc_rxdma_tag); err = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, - BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, + BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE_32BIT, 16, BUS_SPACE_MAXSIZE_32BIT, 0, NULL,NULL, &sc->sc_txdma_tag); /* init transmit descriptors */ STAILQ_INIT(&sc->sc_txfreeq); STAILQ_INIT(&sc->sc_txdirtyq); /* create TX DMA maps */ err = ENOMEM; for (i = 0; i < GLC_MAX_TX_PACKETS; i++) { txs = &sc->sc_txsoft[i]; txs->txs_mbuf = NULL; err = bus_dmamap_create(sc->sc_txdma_tag, 0, &txs->txs_dmamap); if (err) { device_printf(dev, "unable to create TX DMA map %d, error = %d\n", i, err); } STAILQ_INSERT_TAIL(&sc->sc_txfreeq, txs, txs_q); } /* Create the receive buffer DMA maps. */ for (i = 0; i < GLC_MAX_RX_PACKETS; i++) { err = bus_dmamap_create(sc->sc_rxdma_tag, 0, &sc->sc_rxsoft[i].rxs_dmamap); if (err) { device_printf(dev, "unable to create RX DMA map %d, error = %d\n", i, err); } sc->sc_rxsoft[i].rxs_mbuf = NULL; } /* * Attach to network stack */ if_initname(sc->sc_ifp, device_get_name(dev), device_get_unit(dev)); sc->sc_ifp->if_mtu = ETHERMTU; sc->sc_ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; sc->sc_ifp->if_start = glc_start; sc->sc_ifp->if_ioctl = glc_ioctl; sc->sc_ifp->if_init = glc_init; IFQ_SET_MAXLEN(&sc->sc_ifp->if_snd, GLC_MAX_TX_PACKETS); sc->sc_ifp->if_snd.ifq_drv_maxlen = GLC_MAX_TX_PACKETS; IFQ_SET_READY(&sc->sc_ifp->if_snd); ether_ifattach(sc->sc_ifp, sc->sc_enaddr); sc->sc_ifp->if_hwassist = 0; return (0); fail: mtx_destroy(&sc->sc_mtx); if_free(sc->sc_ifp); return (ENXIO); } static void glc_init_locked(struct glc_softc *sc) { int i; struct glc_rxsoft *rxs; mtx_assert(&sc->sc_mtx, MA_OWNED); lv1_net_stop_tx_dma(sc->sc_bus, sc->sc_dev, 0); lv1_net_stop_rx_dma(sc->sc_bus, sc->sc_dev, 0); for (i = 0; i < GLC_MAX_RX_PACKETS; i++) { rxs = &sc->sc_rxsoft[i]; rxs->rxs_desc_slot = i; if (rxs->rxs_mbuf == NULL) { glc_add_rxbuf(sc, i); if (rxs->rxs_mbuf == NULL) { rxs->rxs_desc_slot = -1; break; } } glc_add_rxbuf_dma(sc, i); bus_dmamap_sync(sc->sc_dmadesc_tag, sc->sc_rxdmadesc_map, BUS_DMASYNC_PREREAD); } lv1_net_start_rx_dma(sc->sc_bus, sc->sc_dev, sc->sc_rxsoft[0].rxs_desc, 0); sc->sc_ifp->if_drv_flags |= IFF_DRV_RUNNING; sc->sc_ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; sc->sc_ifpflags = sc->sc_ifp->if_flags; } static void glc_init(void *xsc) { struct glc_softc *sc = xsc; mtx_lock(&sc->sc_mtx); glc_init_locked(sc); mtx_unlock(&sc->sc_mtx); } static void glc_start_locked(struct ifnet *ifp) { struct glc_softc *sc = ifp->if_softc; bus_addr_t first, pktdesc; int kickstart = 0; struct mbuf *mb_head; mtx_assert(&sc->sc_mtx, MA_OWNED); first = 0; if (STAILQ_EMPTY(&sc->sc_txdirtyq)) kickstart = 1; while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { IFQ_DRV_DEQUEUE(&ifp->if_snd, mb_head); if (mb_head == NULL) break; BPF_MTAP(ifp, mb_head); if (sc->sc_tx_vlan >= 0) mb_head = ether_vlanencap(mb_head, sc->sc_tx_vlan); if (glc_encap(sc, &mb_head, &pktdesc)) { /* Put the packet back and stop */ ifp->if_drv_flags |= IFF_DRV_OACTIVE; IFQ_DRV_PREPEND(&ifp->if_snd, mb_head); break; } if (first == 0) first = pktdesc; } bus_dmamap_sync(sc->sc_dmadesc_tag, sc->sc_txdmadesc_map, BUS_DMASYNC_PREREAD); /* XXX: kickstart always until problems are sorted out */ kickstart = 1; if (kickstart && first != 0) { lv1_net_stop_tx_dma(sc->sc_bus, sc->sc_dev, 0); lv1_net_start_tx_dma(sc->sc_bus, sc->sc_dev, first, 0); } } static void glc_start(struct ifnet *ifp) { struct glc_softc *sc = ifp->if_softc; mtx_lock(&sc->sc_mtx); glc_start_locked(ifp); mtx_unlock(&sc->sc_mtx); } static int glc_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct glc_softc *sc = ifp->if_softc; #if 0 struct ifreq *ifr = (struct ifreq *)data; #endif int err = 0; switch (cmd) { case SIOCSIFFLAGS: mtx_lock(&sc->sc_mtx); if ((ifp->if_flags & IFF_UP) != 0) { #if 0 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0 && ((ifp->if_flags ^ sc->sc_ifpflags) & (IFF_ALLMULTI | IFF_PROMISC)) != 0) bm_setladrf(sc); else #endif glc_init_locked(sc); } #if 0 else if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) bm_stop(sc); #endif sc->sc_ifpflags = ifp->if_flags; mtx_unlock(&sc->sc_mtx); break; default: err = ether_ioctl(ifp, cmd, data); break; } return (err); } static int glc_add_rxbuf(struct glc_softc *sc, int idx) { struct glc_rxsoft *rxs = &sc->sc_rxsoft[idx]; struct mbuf *m; bus_dma_segment_t segs[1]; int error, nsegs; m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); if (m == NULL) return (ENOBUFS); m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; if (rxs->rxs_mbuf != NULL) { bus_dmamap_sync(sc->sc_rxdma_tag, rxs->rxs_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->sc_rxdma_tag, rxs->rxs_dmamap); } error = bus_dmamap_load_mbuf_sg(sc->sc_rxdma_tag, rxs->rxs_dmamap, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_self, "cannot load RS DMA map %d, error = %d\n", idx, error); m_freem(m); return (error); } /* If nsegs is wrong then the stack is corrupt. */ KASSERT(nsegs == 1, ("%s: too many DMA segments (%d)", __func__, nsegs)); rxs->rxs_mbuf = m; rxs->segment = segs[0]; bus_dmamap_sync(sc->sc_rxdma_tag, rxs->rxs_dmamap, BUS_DMASYNC_PREREAD); return (0); } static int glc_add_rxbuf_dma(struct glc_softc *sc, int idx) { struct glc_rxsoft *rxs = &sc->sc_rxsoft[idx]; bzero(&sc->sc_rxdmadesc[idx], sizeof(sc->sc_rxdmadesc[idx])); - sc->sc_rxdmadesc[idx].paddr = sc->sc_dma_base + rxs->segment.ds_addr; + sc->sc_rxdmadesc[idx].paddr = glc_map_addr(sc, rxs->segment.ds_addr); sc->sc_rxdmadesc[idx].len = rxs->segment.ds_len; - sc->sc_rxdmadesc[idx].next = sc->sc_dma_base + sc->sc_rxdmadesc_phys + - ((idx + 1) % GLC_MAX_RX_PACKETS)*sizeof(sc->sc_rxdmadesc[idx]); + sc->sc_rxdmadesc[idx].next = glc_map_addr(sc, sc->sc_rxdmadesc_phys + + ((idx + 1) % GLC_MAX_RX_PACKETS)*sizeof(sc->sc_rxdmadesc[idx])); sc->sc_rxdmadesc[idx].cmd_stat = GELIC_DESCR_OWNED; rxs->rxs_desc_slot = idx; - rxs->rxs_desc = sc->sc_dma_base + sc->sc_rxdmadesc_phys + - idx*sizeof(struct glc_dmadesc); + rxs->rxs_desc = glc_map_addr(sc, sc->sc_rxdmadesc_phys + + idx*sizeof(struct glc_dmadesc)); return (0); } static int glc_encap(struct glc_softc *sc, struct mbuf **m_head, bus_addr_t *pktdesc) { bus_dma_segment_t segs[16]; struct glc_txsoft *txs; struct mbuf *m; bus_addr_t firstslotphys; int i, idx; int nsegs = 16; int err = 0; /* Check if the ring buffer is full */ if (sc->next_txdma_slot == sc->first_used_txdma_slot) return (-1); /* Max number of segments is the number of free DMA slots */ if (sc->next_txdma_slot >= sc->first_used_txdma_slot) nsegs = 128 - sc->next_txdma_slot + sc->first_used_txdma_slot; else nsegs = sc->first_used_txdma_slot - sc->next_txdma_slot; if (nsegs > 16) nsegs = 16; /* Get a work queue entry. */ if ((txs = STAILQ_FIRST(&sc->sc_txfreeq)) == NULL) { /* Ran out of descriptors. */ return (ENOBUFS); } err = bus_dmamap_load_mbuf_sg(sc->sc_txdma_tag, txs->txs_dmamap, *m_head, segs, &nsegs, BUS_DMA_NOWAIT); if (err == EFBIG) { m = m_collapse(*m_head, M_DONTWAIT, nsegs); if (m == NULL) { m_freem(*m_head); *m_head = NULL; return (ENOBUFS); } *m_head = m; err = bus_dmamap_load_mbuf_sg(sc->sc_txdma_tag, txs->txs_dmamap, *m_head, segs, &nsegs, BUS_DMA_NOWAIT); if (err != 0) { m_freem(*m_head); *m_head = NULL; return (err); } } else if (err != 0) return (err); if (nsegs == 0) { m_freem(*m_head); *m_head = NULL; return (EIO); } txs->txs_ndescs = nsegs; txs->txs_firstdesc = sc->next_txdma_slot; idx = txs->txs_firstdesc; - firstslotphys = sc->sc_dma_base + sc->sc_txdmadesc_phys + - txs->txs_firstdesc*sizeof(struct glc_dmadesc); + firstslotphys = glc_map_addr(sc, sc->sc_txdmadesc_phys + + txs->txs_firstdesc*sizeof(struct glc_dmadesc)); for (i = 0; i < nsegs; i++) { if (i+1 == nsegs) txs->txs_lastdesc = sc->next_txdma_slot; bzero(&sc->sc_txdmadesc[idx], sizeof(sc->sc_txdmadesc[idx])); - sc->sc_txdmadesc[idx].paddr = sc->sc_dma_base + segs[i].ds_addr; + sc->sc_txdmadesc[idx].paddr = glc_map_addr(sc, segs[i].ds_addr); sc->sc_txdmadesc[idx].len = segs[i].ds_len; - sc->sc_txdmadesc[idx].next = sc->sc_dma_base + + sc->sc_txdmadesc[idx].next = glc_map_addr(sc, sc->sc_txdmadesc_phys + - ((idx + 1) % GLC_MAX_TX_PACKETS)*sizeof(struct glc_dmadesc); + ((idx + 1) % GLC_MAX_TX_PACKETS)*sizeof(struct glc_dmadesc)); sc->sc_txdmadesc[idx].cmd_stat |= GELIC_CMDSTAT_NOIPSEC; if (i+1 == nsegs) { txs->txs_lastdesc = sc->next_txdma_slot; sc->sc_txdmadesc[idx].next = 0; sc->sc_txdmadesc[idx].cmd_stat |= GELIC_CMDSTAT_LAST; } sc->sc_txdmadesc[idx].cmd_stat |= GELIC_DESCR_OWNED; idx = (idx + 1) % GLC_MAX_TX_PACKETS; } sc->next_txdma_slot = idx; idx = (txs->txs_firstdesc - 1) % GLC_MAX_TX_PACKETS; sc->sc_txdmadesc[idx].next = firstslotphys; if (sc->first_used_txdma_slot < 0) sc->first_used_txdma_slot = txs->txs_firstdesc; bus_dmamap_sync(sc->sc_txdma_tag, txs->txs_dmamap, BUS_DMASYNC_PREWRITE); STAILQ_REMOVE_HEAD(&sc->sc_txfreeq, txs_q); STAILQ_INSERT_TAIL(&sc->sc_txdirtyq, txs, txs_q); txs->txs_mbuf = *m_head; if (pktdesc != NULL) *pktdesc = firstslotphys; return (0); } static void glc_rxintr(struct glc_softc *sc) { int i, restart_rxdma; struct mbuf *m; struct ifnet *ifp = sc->sc_ifp; bus_dmamap_sync(sc->sc_dmadesc_tag, sc->sc_rxdmadesc_map, BUS_DMASYNC_PREWRITE); restart_rxdma = 0; while ((sc->sc_rxdmadesc[sc->sc_next_rxdma_slot].cmd_stat & GELIC_DESCR_OWNED) == 0) { i = sc->sc_next_rxdma_slot; if (sc->sc_rxdmadesc[i].rxerror & GELIC_RXERRORS) { ifp->if_ierrors++; goto requeue; } m = sc->sc_rxsoft[i].rxs_mbuf; if (glc_add_rxbuf(sc, i)) { ifp->if_ierrors++; goto requeue; } ifp->if_ipackets++; m->m_pkthdr.rcvif = ifp; m->m_len = sc->sc_rxdmadesc[i].valid_size; m->m_pkthdr.len = m->m_len; sc->sc_next_rxdma_slot++; if (sc->sc_next_rxdma_slot >= GLC_MAX_RX_PACKETS) sc->sc_next_rxdma_slot = 0; if (sc->sc_rx_vlan >= 0) m_adj(m, 2); mtx_unlock(&sc->sc_mtx); (*ifp->if_input)(ifp, m); mtx_lock(&sc->sc_mtx); requeue: if (sc->sc_rxdmadesc[i].cmd_stat & GELIC_CMDSTAT_RX_END) restart_rxdma = 1; glc_add_rxbuf_dma(sc, i); if (restart_rxdma) lv1_net_start_rx_dma(sc->sc_bus, sc->sc_dev, sc->sc_rxsoft[i].rxs_desc, 0); } } static void glc_txintr(struct glc_softc *sc) { struct ifnet *ifp = sc->sc_ifp; struct glc_txsoft *txs; int progress = 0; while ((txs = STAILQ_FIRST(&sc->sc_txdirtyq)) != NULL) { if (sc->sc_txdmadesc[txs->txs_lastdesc].cmd_stat & GELIC_DESCR_OWNED) break; STAILQ_REMOVE_HEAD(&sc->sc_txdirtyq, txs_q); bus_dmamap_unload(sc->sc_txdma_tag, txs->txs_dmamap); if (txs->txs_mbuf != NULL) { m_freem(txs->txs_mbuf); txs->txs_mbuf = NULL; } STAILQ_INSERT_TAIL(&sc->sc_txfreeq, txs, txs_q); ifp->if_opackets++; progress = 1; } if (txs != NULL) sc->first_used_txdma_slot = txs->txs_firstdesc; else sc->first_used_txdma_slot = -1; if (progress) { /* * We freed some descriptors, so reset IFF_DRV_OACTIVE * and restart. */ ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; #if 0 sc->sc_wdog_timer = STAILQ_EMPTY(&sc->sc_txdirtyq) ? 0 : 5; #endif if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) glc_start_locked(ifp); } } static void glc_intr(void *xsc) { struct glc_softc *sc = xsc; mtx_lock(&sc->sc_mtx); powerpc_sync(); if (*sc->sc_interrupt_status == 0) { device_printf(sc->sc_self, "stray interrupt!\n"); mtx_unlock(&sc->sc_mtx); return; } if (*sc->sc_interrupt_status & (GELIC_INT_RXDONE | GELIC_INT_RXFRAME)) glc_rxintr(sc); if (*sc->sc_interrupt_status & GELIC_INT_TXDONE) glc_txintr(sc); mtx_unlock(&sc->sc_mtx); } Index: user/nwhitehorn/ps3/powerpc/ps3/if_glcreg.h =================================================================== --- user/nwhitehorn/ps3/powerpc/ps3/if_glcreg.h (revision 212100) +++ user/nwhitehorn/ps3/powerpc/ps3/if_glcreg.h (revision 212101) @@ -1,147 +1,147 @@ /*- * Copyright (C) 2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD: user/nwhitehorn/ps3/powerpc/ofw/ofw_cpu.c 193156 2009-05-31 09:01:23Z nwhitehorn $ */ #ifndef _POWERPC_PS3_IF_GLCREG_H #define _POWERPC_PS3_IF_GLCREG_H #define GLC_MAX_TX_PACKETS 128 #define GLC_MAX_RX_PACKETS 128 struct glc_dmadesc; /* * software state for transmit job mbufs (may be elements of mbuf chains) */ struct glc_txsoft { struct mbuf *txs_mbuf; /* head of our mbuf chain */ bus_dmamap_t txs_dmamap; /* our DMA map */ int txs_firstdesc; /* first descriptor in packet */ int txs_lastdesc; /* last descriptor in packet */ int txs_ndescs; /* number of descriptors */ STAILQ_ENTRY(glc_txsoft) txs_q; }; STAILQ_HEAD(glc_txsq, glc_txsoft); /* * software state for receive jobs */ struct glc_rxsoft { struct mbuf *rxs_mbuf; /* head of our mbuf chain */ bus_dmamap_t rxs_dmamap; /* our DMA map */ int rxs_desc_slot; /* DMA descriptor for this packet */ bus_addr_t rxs_desc; bus_dma_segment_t segment; }; struct glc_softc { struct ifnet *sc_ifp; device_t sc_self; struct mtx sc_mtx; u_char sc_enaddr[ETHER_ADDR_LEN]; int sc_tx_vlan, sc_rx_vlan; int sc_ifpflags; - uint64_t sc_dma_base; + uint64_t sc_dma_base[5]; bus_dma_tag_t sc_dmadesc_tag; int sc_irqid; struct resource *sc_irq; void *sc_irqctx; uint64_t *sc_interrupt_status; /* Transmission */ bus_dma_tag_t sc_txdma_tag; struct glc_txsoft sc_txsoft[GLC_MAX_TX_PACKETS]; struct glc_dmadesc *sc_txdmadesc; int next_txdma_slot, first_used_txdma_slot; bus_dmamap_t sc_txdmadesc_map; bus_addr_t sc_txdmadesc_phys; struct glc_txsq sc_txfreeq; struct glc_txsq sc_txdirtyq; /* Reception */ bus_dma_tag_t sc_rxdma_tag; struct glc_rxsoft sc_rxsoft[GLC_MAX_RX_PACKETS]; struct glc_dmadesc *sc_rxdmadesc; int sc_next_rxdma_slot; bus_dmamap_t sc_rxdmadesc_map; bus_addr_t sc_rxdmadesc_phys; int sc_bus, sc_dev; }; #define GELIC_GET_MAC_ADDRESS 0x0001 #define GELIC_GET_LINK_STATUS 0x0002 #define GELIC_LINK_UP 0x0001 #define GELIC_FULL_DUPLEX 0x0002 #define GELIC_AUTO_NEG 0x0004 #define GELIC_SPEED_10 0x0010 #define GELIC_SPEED_100 0x0020 #define GELIC_SPEED_1000 0x0040 #define GELIC_GET_VLAN_ID 0x0004 #define GELIC_VLAN_TX_ETHERNET 0x0002 #define GELIC_VLAN_RX_ETHERNET 0x0012 #define GELIC_VLAN_TX_WIRELESS 0x0003 #define GELIC_VLAN_RX_WIRELESS 0x0013 /* Command status code */ #define GELIC_DESCR_OWNED 0xa0000000 #define GELIC_CMDSTAT_DMA_DONE 0x00000000 #define GELIC_CMDSTAT_RX_END 0x00000002 #define GELIC_CMDSTAT_NOIPSEC 0x00080000 #define GELIC_CMDSTAT_LAST 0x00040000 #define GELIC_RXERRORS 0x7def8000 /* Interrupt options */ #define GELIC_INT_RXDONE 0x0000000000004000UL #define GELIC_INT_RXFRAME 0x1000000000000000UL #define GELIC_INT_TXDONE 0x0080000000000000UL #define GELIC_INT_PHY 0x0000000020000000UL /* Hardware DMA descriptor. Must be 32-byte aligned */ struct glc_dmadesc { uint32_t paddr; /* Must be 128 byte aligned for receive */ uint32_t len; uint32_t next; uint32_t cmd_stat; uint32_t result_size; uint32_t valid_size; uint32_t data_stat; uint32_t rxerror; }; #endif /* _POWERPC_PS3_IF_GLCREG_H */ Index: user/nwhitehorn/ps3/powerpc/ps3/mmu_ps3.c =================================================================== --- user/nwhitehorn/ps3/powerpc/ps3/mmu_ps3.c (revision 212100) +++ user/nwhitehorn/ps3/powerpc/ps3/mmu_ps3.c (revision 212101) @@ -1,346 +1,346 @@ /*- * Copyright (C) 2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "mmu_if.h" #include "ps3-hvcall.h" #define VSID_HASH_MASK 0x0000007fffffffffUL extern int ps3fb_remap(void); static uint64_t mps3_vas_id; /* * Kernel MMU interface */ static void mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend); static void mps3_cpu_bootstrap(mmu_t mmup, int ap); static void mps3_pte_synch(struct lpte *pt, struct lpte *pvo_pt); static void mps3_pte_clear(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn, u_int64_t ptebit); static void mps3_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn); static void mps3_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn); static int mps3_pte_insert(u_int ptegidx, struct lpte *pvo_pt); static struct lpte *mps3_pvo_to_pte(const struct pvo_entry *pvo); static mmu_method_t mps3_methods[] = { MMUMETHOD(mmu_change_wiring, moea64_change_wiring), MMUMETHOD(mmu_clear_modify, moea64_clear_modify), MMUMETHOD(mmu_clear_reference, moea64_clear_reference), MMUMETHOD(mmu_copy_page, moea64_copy_page), MMUMETHOD(mmu_enter, moea64_enter), MMUMETHOD(mmu_enter_object, moea64_enter_object), MMUMETHOD(mmu_enter_quick, moea64_enter_quick), MMUMETHOD(mmu_extract, moea64_extract), MMUMETHOD(mmu_extract_and_hold, moea64_extract_and_hold), MMUMETHOD(mmu_init, moea64_init), MMUMETHOD(mmu_is_modified, moea64_is_modified), MMUMETHOD(mmu_is_referenced, moea64_is_referenced), MMUMETHOD(mmu_ts_referenced, moea64_ts_referenced), MMUMETHOD(mmu_map, moea64_map), MMUMETHOD(mmu_page_exists_quick,moea64_page_exists_quick), MMUMETHOD(mmu_page_wired_mappings,moea64_page_wired_mappings), MMUMETHOD(mmu_pinit, moea64_pinit), MMUMETHOD(mmu_pinit0, moea64_pinit0), MMUMETHOD(mmu_protect, moea64_protect), MMUMETHOD(mmu_qenter, moea64_qenter), MMUMETHOD(mmu_qremove, moea64_qremove), MMUMETHOD(mmu_release, moea64_release), MMUMETHOD(mmu_remove, moea64_remove), MMUMETHOD(mmu_remove_all, moea64_remove_all), MMUMETHOD(mmu_remove_write, moea64_remove_write), MMUMETHOD(mmu_sync_icache, moea64_sync_icache), MMUMETHOD(mmu_zero_page, moea64_zero_page), MMUMETHOD(mmu_zero_page_area, moea64_zero_page_area), MMUMETHOD(mmu_zero_page_idle, moea64_zero_page_idle), MMUMETHOD(mmu_activate, moea64_activate), MMUMETHOD(mmu_deactivate, moea64_deactivate), /* Internal interfaces */ MMUMETHOD(mmu_bootstrap, mps3_bootstrap), MMUMETHOD(mmu_cpu_bootstrap, mps3_cpu_bootstrap), MMUMETHOD(mmu_mapdev, moea64_mapdev), MMUMETHOD(mmu_unmapdev, moea64_unmapdev), MMUMETHOD(mmu_kextract, moea64_kextract), MMUMETHOD(mmu_kenter, moea64_kenter), MMUMETHOD(mmu_dev_direct_mapped,moea64_dev_direct_mapped), { 0, 0 } }; static mmu_def_t ps3_mmu = { "mmu_ps3", mps3_methods, 0 }; MMU_DEF(ps3_mmu); static void mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { uint64_t final_pteg_count; /* * Set our page table override functions */ moea64_pte_synch_hook = mps3_pte_synch; moea64_pte_clear_hook = mps3_pte_clear; moea64_pte_unset_hook = mps3_pte_unset; moea64_pte_change_hook = mps3_pte_change; moea64_pte_insert_hook = mps3_pte_insert; moea64_pvo_to_pte_hook = mps3_pvo_to_pte; moea64_early_bootstrap(mmup, kernelstart, kernelend); lv1_construct_virtual_address_space( 20 /* log_2(moea64_pteg_count) */, 2 /* n page sizes */, (24UL << 56) | (16UL << 48) /* page sizes 16 MB + 64 KB */, &mps3_vas_id, &final_pteg_count ); moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); moea64_late_bootstrap(mmup, kernelstart, kernelend); } static void mps3_cpu_bootstrap(mmu_t mmup, int ap) { struct slb *slb = PCPU_GET(slb); register_t seg0; int i; mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); /* * Destroy the loader's address space if we are coming up for * the first time, and redo the FB mapping so we can continue * having a console. */ if (!ap) lv1_destruct_virtual_address_space(0); lv1_select_virtual_address_space(mps3_vas_id); if (!ap) ps3fb_remap(); /* * Install kernel SLB entries */ __asm __volatile ("slbia"); __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); for (i = 0; i < 64; i++) { if (!(slb[i].slbe & SLBE_VALID)) continue; __asm __volatile ("slbmte %0, %1" :: "r"(slb[i].slbv), "r"(slb[i].slbe)); } } static void mps3_pte_synch(struct lpte *pt, struct lpte *pvo_pt) { uint64_t halfbucket[4], rcbits; uint64_t slot = (uint64_t)(pt)-1; lv1_read_htab_entries(mps3_vas_id, slot & ~0x3UL, &halfbucket[0], &halfbucket[1], &halfbucket[2], &halfbucket[3], &rcbits); /* * rcbits contains the low 12 bits of each PTEs 2nd part, * spaced at 16-bit intervals */ KASSERT((halfbucket[slot & 0x3] & LPTE_AVPN_MASK) == (pvo_pt->pte_hi & LPTE_AVPN_MASK), ("PTE upper word %#lx != %#lx\n", halfbucket[slot & 0x3], pvo_pt->pte_hi)); pvo_pt->pte_lo &= ~(LPTE_CHG | LPTE_REF); pvo_pt->pte_lo |= (rcbits >> ((3 - (slot & 0x3))*16)) & (LPTE_CHG | LPTE_REF); } static void mps3_pte_clear(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn, u_int64_t ptebit) { uint64_t slot = (uint64_t)(pt)-1; lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi, pvo_pt->pte_lo & ~ptebit); } static void mps3_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { uint64_t slot = (uint64_t)(pt)-1; mps3_pte_synch(pt, pvo_pt); pvo_pt->pte_hi &= ~LPTE_VALID; lv1_write_htab_entry(mps3_vas_id, slot, 0, 0); moea64_pte_valid--; } static void mps3_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { uint64_t slot = (uint64_t)(pt)-1; mps3_pte_synch(pt, pvo_pt); lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi, pvo_pt->pte_lo); } static int mps3_pte_insert(u_int ptegidx, struct lpte *pvo_pt) { int result; struct lpte evicted; struct pvo_entry *pvo; uint64_t index; pvo_pt->pte_hi |= LPTE_VALID; pvo_pt->pte_hi &= ~LPTE_HID; evicted.pte_hi = 0; result = lv1_insert_htab_entry(mps3_vas_id, ptegidx << 3, pvo_pt->pte_hi, pvo_pt->pte_lo, LPTE_LOCKED | LPTE_WIRED, 0, &index, &evicted.pte_hi, &evicted.pte_lo); if (result != 0) { /* No freeable slots in either PTEG? We're hosed. */ - mtmsr(mfmsr() | PSL_DR); panic("moea64_pte_insert: overflow"); + panic("mps3_pte_insert: overflow (%d)", result); return (-1); } /* * See where we ended up. */ if (index >> 3 != ptegidx) pvo_pt->pte_hi |= LPTE_HID; moea64_pte_valid++; if (!evicted.pte_hi) return (index & 0x7); /* * Synchronize the sacrifice PTE with its PVO, then mark both * invalid. The PVO will be reused when/if the VM system comes * here after a fault. */ if (evicted.pte_hi & LPTE_HID) ptegidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { if ((pvo->pvo_pte.lpte.pte_hi & LPTE_AVPN_MASK) == (evicted.pte_hi & LPTE_AVPN_MASK)) { KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, ("Invalid PVO for valid PTE!")); pvo->pvo_pte.lpte.pte_hi &= ~LPTE_VALID; pvo->pvo_pte.lpte.pte_lo |= evicted.pte_lo & (LPTE_REF | LPTE_CHG); PVO_PTEGIDX_CLR(pvo); moea64_pte_valid--; moea64_pte_overflow++; break; } } return (index & 0x7); } static __inline u_int va_to_pteg(uint64_t vsid, vm_offset_t addr, int large) { uint64_t hash; int shift; shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> shift); return (hash & moea64_pteg_mask); } static struct lpte * mps3_pvo_to_pte(const struct pvo_entry *pvo) { uint64_t slot, vsid; u_int ptegidx; /* If the PTEG index is not set, then there is no page table entry */ if (!PVO_PTEGIDX_ISSET(pvo)) return (NULL); vsid = PVO_VSID(pvo); ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); /* * We can find the actual pte entry without searching by grabbing * the PTEG index from 3 unused bits in pvo_vaddr and by * noticing the HID bit. */ if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) ptegidx ^= moea64_pteg_mask; slot = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo); return ((struct lpte *)(slot + 1)); } Index: user/nwhitehorn/ps3/powerpc/ps3/platform_ps3.c =================================================================== --- user/nwhitehorn/ps3/powerpc/ps3/platform_ps3.c (revision 212100) +++ user/nwhitehorn/ps3/powerpc/ps3/platform_ps3.c (revision 212101) @@ -1,225 +1,248 @@ /*- * Copyright (c) 2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD: user/nwhitehorn/ps3/powerpc/booke/platform_ps3.c 193492 2009-06-05 09:46:00Z raj $"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "platform_if.h" #include "ps3-hvcall.h" #ifdef SMP extern void *ap_pcpu; #endif static int ps3_probe(platform_t); static int ps3_attach(platform_t); static void ps3_mem_regions(platform_t, struct mem_region **phys, int *physsz, struct mem_region **avail, int *availsz); +static vm_offset_t ps3_real_maxaddr(platform_t); static u_long ps3_timebase_freq(platform_t, struct cpuref *cpuref); static int ps3_smp_first_cpu(platform_t, struct cpuref *cpuref); static int ps3_smp_next_cpu(platform_t, struct cpuref *cpuref); static int ps3_smp_get_bsp(platform_t, struct cpuref *cpuref); static int ps3_smp_start_cpu(platform_t, struct pcpu *cpu); static void ps3_reset(platform_t); static platform_method_t ps3_methods[] = { PLATFORMMETHOD(platform_probe, ps3_probe), PLATFORMMETHOD(platform_attach, ps3_attach), PLATFORMMETHOD(platform_mem_regions, ps3_mem_regions), + PLATFORMMETHOD(platform_real_maxaddr, ps3_real_maxaddr), PLATFORMMETHOD(platform_timebase_freq, ps3_timebase_freq), PLATFORMMETHOD(platform_smp_first_cpu, ps3_smp_first_cpu), PLATFORMMETHOD(platform_smp_next_cpu, ps3_smp_next_cpu), PLATFORMMETHOD(platform_smp_get_bsp, ps3_smp_get_bsp), PLATFORMMETHOD(platform_smp_start_cpu, ps3_smp_start_cpu), PLATFORMMETHOD(platform_reset, ps3_reset), { 0, 0 } }; static platform_def_t ps3_platform = { "ps3", ps3_methods, 0 }; PLATFORM_DEF(ps3_platform); static int ps3_probe(platform_t plat) { #if 0 phandle_t root; char compatible[64]; root = OF_finddevice("/"); if (OF_getprop(root, "compatible", compatible, sizeof(compatible)) <= 0) return (ENXIO); if (strncmp(compatible, "sony,ps3", sizeof(compatible)) == 0) return (BUS_PROBE_SPECIFIC); return (ENXIO); #else return (BUS_PROBE_NOWILDCARD); #endif } +#define MEM_REGIONS 2 +static struct mem_region avail_regions[MEM_REGIONS]; + static int ps3_attach(platform_t plat) { + uint64_t lpar_id, junk, ppe_id; + /* Get real mode memory region */ + avail_regions[0].mr_start = 0; + lv1_get_logical_partition_id(&lpar_id); + lv1_get_logical_ppe_id(&ppe_id); + lv1_get_repository_node_value(lpar_id, + lv1_repository_string("bi") >> 32, lv1_repository_string("pu"), + ppe_id, lv1_repository_string("rm_size"), + &avail_regions[0].mr_size, &junk); + + /* Now get extended memory region */ + lv1_get_repository_node_value(lpar_id, + lv1_repository_string("bi") >> 32, + lv1_repository_string("rgntotal"), 0, 0, + &avail_regions[1].mr_size, &junk); + + /* Convert to maximum amount we can allocate in 16 MB pages */ + avail_regions[1].mr_size -= avail_regions[0].mr_size; + avail_regions[1].mr_size -= avail_regions[1].mr_size % (16*1024*1024); + + lv1_allocate_memory(avail_regions[1].mr_size, 24 /* 16 MB pages */, + 0, 0x04 /* any address */, &avail_regions[1].mr_start, &junk); + pmap_mmu_install("mmu_ps3", BUS_PROBE_SPECIFIC); return (0); } -#define MEM_REGIONS 8 -static struct mem_region avail_regions[MEM_REGIONS]; - void ps3_mem_regions(platform_t plat, struct mem_region **phys, int *physsz, struct mem_region **avail, int *availsz) { - uint64_t lpar_id, junk, ppe_id; - avail_regions[0].mr_start = 0; - lv1_get_logical_partition_id(&lpar_id); - lv1_get_logical_ppe_id(&ppe_id); - lv1_get_repository_node_value(lpar_id, - lv1_repository_string("bi") >> 32, lv1_repository_string("pu"), - ppe_id, lv1_repository_string("rm_size"), - &avail_regions[0].mr_size, &junk); - *phys = *avail = avail_regions; - *physsz = *availsz = 1; + *physsz = *availsz = MEM_REGIONS; } static u_long ps3_timebase_freq(platform_t plat, struct cpuref *cpuref) { uint64_t ticks, node_id, junk; lv1_get_repository_node_value(PS3_LPAR_ID_PME, lv1_repository_string("be") >> 32, 0, 0, 0, &node_id, &junk); lv1_get_repository_node_value(PS3_LPAR_ID_PME, lv1_repository_string("be") >> 32, node_id, lv1_repository_string("clock"), 0, &ticks, &junk); return (ticks); } static int ps3_smp_first_cpu(platform_t plat, struct cpuref *cpuref) { cpuref->cr_cpuid = 0; cpuref->cr_hwref = cpuref->cr_cpuid; return (0); } static int ps3_smp_next_cpu(platform_t plat, struct cpuref *cpuref) { if (cpuref->cr_cpuid >= 1) return (ENOENT); cpuref->cr_cpuid++; cpuref->cr_hwref = cpuref->cr_cpuid; return (0); } static int ps3_smp_get_bsp(platform_t plat, struct cpuref *cpuref) { cpuref->cr_cpuid = 0; cpuref->cr_hwref = cpuref->cr_cpuid; return (0); } static int ps3_smp_start_cpu(platform_t plat, struct pcpu *pc) { #ifdef SMP /* loader(8) is spinning on 0x40 == 0 right now */ uint32_t *secondary_spin_sem = (uint32_t *)(0x40); int timeout; if (pc->pc_hwref != 1) return (ENXIO); ap_pcpu = pc; *secondary_spin_sem = 1; powerpc_sync(); DELAY(1); timeout = 10000; while (!pc->pc_awake && timeout--) DELAY(100); return ((pc->pc_awake) ? 0 : EBUSY); #else /* No SMP support */ return (ENXIO); #endif } static void ps3_reset(platform_t plat) { lv1_panic(1); } + +static vm_offset_t +ps3_real_maxaddr(platform_t plat) +{ + return (avail_regions[0].mr_start + avail_regions[0].mr_size); +} +