Index: head/sys/sparc64/include/tlb.h =================================================================== --- head/sys/sparc64/include/tlb.h (revision 205257) +++ head/sys/sparc64/include/tlb.h (revision 205258) @@ -1,150 +1,162 @@ /*- * Copyright (c) 2001 Jake Burkholder. + * Copyright (c) 2008, 2010 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_TLB_H_ #define _MACHINE_TLB_H_ #define TLB_DIRECT_ADDRESS_BITS (43) #define TLB_DIRECT_PAGE_BITS (PAGE_SHIFT_4M) #define TLB_DIRECT_ADDRESS_MASK ((1UL << TLB_DIRECT_ADDRESS_BITS) - 1) #define TLB_DIRECT_PAGE_MASK ((1UL << TLB_DIRECT_PAGE_BITS) - 1) -#define TLB_PHYS_TO_DIRECT(pa) \ +#define TLB_PHYS_TO_DIRECT(pa) \ ((pa) | VM_MIN_DIRECT_ADDRESS) -#define TLB_DIRECT_TO_PHYS(va) \ +#define TLB_DIRECT_TO_PHYS(va) \ ((va) & TLB_DIRECT_ADDRESS_MASK) -#define TLB_DIRECT_TO_TTE_MASK \ +#define TLB_DIRECT_TO_TTE_MASK \ (TD_V | TD_4M | (TLB_DIRECT_ADDRESS_MASK - TLB_DIRECT_PAGE_MASK)) #define TLB_DAR_SLOT_SHIFT (3) #define TLB_DAR_SLOT(slot) ((slot) << TLB_DAR_SLOT_SHIFT) #define TAR_VPN_SHIFT (13) #define TAR_CTX_MASK ((1 << TAR_VPN_SHIFT) - 1) #define TLB_TAR_VA(va) ((va) & ~TAR_CTX_MASK) #define TLB_TAR_CTX(ctx) ((ctx) & TAR_CTX_MASK) #define TLB_CXR_CTX_BITS (13) #define TLB_CXR_CTX_MASK \ (((1UL << TLB_CXR_CTX_BITS) - 1) << TLB_CXR_CTX_SHIFT) #define TLB_CXR_CTX_SHIFT (0) #define TLB_CXR_PGSZ_BITS (3) -#define TLB_PCXR_PGSZ_MASK \ - ((((1UL << TLB_CXR_PGSZ_BITS) - 1) << TLB_PCXR_N_PGSZ0_SHIFT) | \ - (((1UL << TLB_CXR_PGSZ_BITS) - 1) << TLB_PCXR_N_PGSZ1_SHIFT) | \ - (((1UL << TLB_CXR_PGSZ_BITS) - 1) << TLB_PCXR_P_PGSZ0_SHIFT) | \ - (((1UL << TLB_CXR_PGSZ_BITS) - 1) << TLB_PCXR_P_PGSZ1_SHIFT)) +#define TLB_CXR_PGSZ_MASK (~TLB_CXR_CTX_MASK) +#define TLB_PCXR_N_IPGSZ0_SHIFT (53) /* SPARC64 VI, VII, VIIIfx */ +#define TLB_PCXR_N_IPGSZ1_SHIFT (50) /* SPARC64 VI, VII, VIIIfx */ #define TLB_PCXR_N_PGSZ0_SHIFT (61) #define TLB_PCXR_N_PGSZ1_SHIFT (58) +#define TLB_PCXR_N_PGSZ_I_SHIFT (55) /* US-IV+ */ +#define TLB_PCXR_P_IPGSZ0_SHIFT (24) /* SPARC64 VI, VII, VIIIfx */ +#define TLB_PCXR_P_IPGSZ1_SHIFT (27) /* SPARC64 VI, VII, VIIIfx */ #define TLB_PCXR_P_PGSZ0_SHIFT (16) #define TLB_PCXR_P_PGSZ1_SHIFT (19) -#define TLB_SCXR_PGSZ_MASK \ - ((((1UL << TLB_CXR_PGSZ_BITS) - 1) << TLB_SCXR_S_PGSZ0_SHIFT) | \ - (((1UL << TLB_CXR_PGSZ_BITS) - 1) << TLB_SCXR_S_PGSZ1_SHIFT)) +/* + * Note that the US-IV+ documentation appears to have TLB_PCXR_P_PGSZ_I_SHIFT + * and TLB_PCXR_P_PGSZ0_SHIFT erroneously inverted. + */ +#define TLB_PCXR_P_PGSZ_I_SHIFT (22) /* US-IV+ */ #define TLB_SCXR_S_PGSZ1_SHIFT (19) #define TLB_SCXR_S_PGSZ0_SHIFT (16) #define TLB_TAE_PGSZ_BITS (3) #define TLB_TAE_PGSZ0_MASK \ (((1UL << TLB_TAE_PGSZ_BITS) - 1) << TLB_TAE_PGSZ0_SHIFT) #define TLB_TAE_PGSZ1_MASK \ (((1UL << TLB_TAE_PGSZ_BITS) - 1) << TLB_TAE_PGSZ1_SHIFT) #define TLB_TAE_PGSZ0_SHIFT (16) #define TLB_TAE_PGSZ1_SHIFT (19) #define TLB_DEMAP_ID_SHIFT (4) #define TLB_DEMAP_ID_PRIMARY (0) #define TLB_DEMAP_ID_SECONDARY (1) #define TLB_DEMAP_ID_NUCLEUS (2) #define TLB_DEMAP_TYPE_SHIFT (6) #define TLB_DEMAP_TYPE_PAGE (0) #define TLB_DEMAP_TYPE_CONTEXT (1) -#define TLB_DEMAP_TYPE_ALL (2) /* USIII and beyond only */ +#define TLB_DEMAP_TYPE_ALL (2) /* US-III and beyond only */ #define TLB_DEMAP_VA(va) ((va) & ~PAGE_MASK) #define TLB_DEMAP_ID(id) ((id) << TLB_DEMAP_ID_SHIFT) #define TLB_DEMAP_TYPE(type) ((type) << TLB_DEMAP_TYPE_SHIFT) #define TLB_DEMAP_PAGE (TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_PAGE)) #define TLB_DEMAP_CONTEXT (TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_CONTEXT)) #define TLB_DEMAP_ALL (TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_ALL)) #define TLB_DEMAP_PRIMARY (TLB_DEMAP_ID(TLB_DEMAP_ID_PRIMARY)) #define TLB_DEMAP_SECONDARY (TLB_DEMAP_ID(TLB_DEMAP_ID_SECONDARY)) #define TLB_DEMAP_NUCLEUS (TLB_DEMAP_ID(TLB_DEMAP_ID_NUCLEUS)) #define TLB_CTX_KERNEL (0) #define TLB_CTX_USER_MIN (1) #define TLB_CTX_USER_MAX (8192) #define MMU_SFSR_ASI_SHIFT (16) #define MMU_SFSR_FT_SHIFT (7) #define MMU_SFSR_E_SHIFT (6) #define MMU_SFSR_CT_SHIFT (4) #define MMU_SFSR_PR_SHIFT (3) #define MMU_SFSR_W_SHIFT (2) #define MMU_SFSR_OW_SHIFT (1) #define MMU_SFSR_FV_SHIFT (0) #define MMU_SFSR_ASI_SIZE (8) #define MMU_SFSR_FT_SIZE (6) #define MMU_SFSR_CT_SIZE (2) -#define MMU_SFSR_GET_ASI(sfsr) \ +#define MMU_SFSR_GET_ASI(sfsr) \ (((sfsr) >> MMU_SFSR_ASI_SHIFT) & ((1UL << MMU_SFSR_ASI_SIZE) - 1)) +#define MMU_SFSR_GET_FT(sfsr) \ + (((sfsr) >> MMU_SFSR_FT_SHIFT) & ((1UL << MMU_SFSR_FT_SIZE) - 1)) +#define MMU_SFSR_GET_CT(sfsr) \ + (((sfsr) >> MMU_SFSR_CT_SHIFT) & ((1UL << MMU_SFSR_CT_SIZE) - 1)) + +#define MMU_SFSR_E (1UL << MMU_SFSR_E_SHIFT) +#define MMU_SFSR_PR (1UL << MMU_SFSR_PR_SHIFT) #define MMU_SFSR_W (1UL << MMU_SFSR_W_SHIFT) +#define MMU_SFSR_OW (1UL << MMU_SFSR_OW_SHIFT) #define MMU_SFSR_FV (1UL << MMU_SFSR_FV_SHIFT) typedef void tlb_flush_nonlocked_t(void); typedef void tlb_flush_user_t(void); struct pmap; struct tlb_entry; extern int dtlb_slots; extern int itlb_slots; extern int kernel_tlb_slots; extern struct tlb_entry *kernel_tlbs; void tlb_context_demap(struct pmap *pm); void tlb_page_demap(struct pmap *pm, vm_offset_t va); void tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end); tlb_flush_nonlocked_t cheetah_tlb_flush_nonlocked; tlb_flush_user_t cheetah_tlb_flush_user; tlb_flush_nonlocked_t spitfire_tlb_flush_nonlocked; tlb_flush_user_t spitfire_tlb_flush_user; extern tlb_flush_nonlocked_t *tlb_flush_nonlocked; extern tlb_flush_user_t *tlb_flush_user; #endif /* !_MACHINE_TLB_H_ */ Index: head/sys/sparc64/include/tte.h =================================================================== --- head/sys/sparc64/include/tte.h (revision 205257) +++ head/sys/sparc64/include/tte.h (revision 205258) @@ -1,137 +1,166 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: BSDI: pmap.v9.h,v 1.10.2.6 1999/08/23 22:18:44 cp Exp * $FreeBSD$ */ #ifndef _MACHINE_TTE_H_ #define _MACHINE_TTE_H_ #define TTE_SHIFT (5) #define TD_SIZE_SHIFT (61) #define TD_SOFT2_SHIFT (50) +#define TD_RSVD2_SHIFT (49) +#define TD_SIZE2_SHIFT (48) #define TD_DIAG_SF_SHIFT (41) #define TD_RSVD_CH_SHIFT (43) +#define TD_RSVD_OC_SHIFT (47) +#define TD_RSVD_PT_SHIFT TD_RSVD_CH_SHIFT +#define TD_RSVD_VE_SHIFT (41) #define TD_PA_SHIFT (13) #define TD_SOFT_SHIFT (7) #define TD_SIZE_BITS (2) #define TD_SOFT2_BITS (9) -#define TD_DIAG_SF_BITS (9) -#define TD_RSVD_CH_BITS (7) -#define TD_PA_CH_BITS (30) -#define TD_PA_SF_BITS (28) +#define TD_RSVD2_BITS (1) /* US-IV+, SPARC64 VI, VII, VIIIfx */ +#define TD_SIZE2_BITS (1) /* US-IV+, SPARC64 VI, VII, VIIIfx */ +#define TD_DIAG_SF_BITS (9) /* US-I, II{,e,i} */ +#define TD_RSVD_CH_BITS (7) /* US-III{,i,+}, US-IV, SPARC64 V */ +#define TD_RSVD_OC_BITS (1) /* SPARC64 VI, VII */ +#define TD_RSVD_PT_BITS (5) /* US-IV+, SPARC64 VI, VII */ +#define TD_RSVD_VE_BITS (7) /* SPARC64 VIIIfx */ +#define TD_PA_CH_BITS (30) /* US-III{,i,+}, US-IV{,+}, SPARC64 V */ +#define TD_PA_OC_BITS (34) /* SPARC64 VI, VII */ +#define TD_PA_SF_BITS (28) /* US-I, II{,e,i}, SPARC64 VIIIfx */ #define TD_PA_BITS TD_PA_CH_BITS #define TD_SOFT_BITS (6) #define TD_SIZE_MASK ((1UL << TD_SIZE_BITS) - 1) #define TD_SOFT2_MASK ((1UL << TD_SOFT2_BITS) - 1) +#define TD_RSVD2_MASK ((1UL << TD_RSVD2_BITS) - 1) +#define TD_SIZE2_MASK ((1UL << TD_SIZE2_BITS) - 1) #define TD_DIAG_SF_MASK ((1UL << TD_DIAG_SF_BITS) - 1) #define TD_RSVD_CH_MASK ((1UL << TD_RSVD_CH_BITS) - 1) +#define TD_RSVD_OC_MASK ((1UL << TD_RSVD_OC_BITS) - 1) +#define TD_RSVD_PT_MASK ((1UL << TD_RSVD_PT_BITS) - 1) +#define TD_RSVD_VE_MASK ((1UL << TD_RSVD_VE_BITS) - 1) #define TD_PA_CH_MASK ((1UL << TD_PA_CH_BITS) - 1) +#define TD_PA_OC_MASK ((1UL << TD_PA_OC_BITS) - 1) #define TD_PA_SF_MASK ((1UL << TD_PA_SF_BITS) - 1) #define TD_PA_MASK ((1UL << TD_PA_BITS) - 1) #define TD_SOFT_MASK ((1UL << TD_SOFT_BITS) - 1) #define TS_8K (0UL) #define TS_64K (1UL) #define TS_512K (2UL) #define TS_4M (3UL) +#define TS_32M (4UL) /* US-IV+, SPARC64 VI, VII only */ +#define TS_256M (5UL) /* US-IV+, SPARC64 VI, VII only */ +#define TS_2G (6UL) /* SPARC64 VIIIfx only */ #define TS_MIN TS_8K #define TS_MAX TS_4M #define TD_V (1UL << 63) #define TD_8K (TS_8K << TD_SIZE_SHIFT) #define TD_64K (TS_64K << TD_SIZE_SHIFT) #define TD_512K (TS_512K << TD_SIZE_SHIFT) #define TD_4M (TS_4M << TD_SIZE_SHIFT) +#define TD_32M \ + (((TS_32M & TD_SIZE_MASK) << TD_SIZE_SHIFT) | \ + (TD_SIZE2_MASK << TD_SIZE2_SHIFT)) +#define TD_256M \ + (((TS_256M & TD_SIZE_MASK) << TD_SIZE_SHIFT) | \ + (TD_SIZE2_MASK << TD_SIZE2_SHIFT)) +#define TD_2G \ + (((TS_2G & TD_SIZE_MASK) << TD_SIZE_SHIFT) | \ + (TD_SIZE2_MASK << TD_SIZE2_SHIFT)) #define TD_NFO (1UL << 60) #define TD_IE (1UL << 59) #define TD_PA(pa) ((pa) & (TD_PA_MASK << TD_PA_SHIFT)) /* NOTE: bit 6 of TD_SOFT will be sign-extended if used as an immediate. */ #define TD_FAKE ((1UL << 5) << TD_SOFT_SHIFT) #define TD_EXEC ((1UL << 4) << TD_SOFT_SHIFT) #define TD_REF ((1UL << 3) << TD_SOFT_SHIFT) #define TD_PV ((1UL << 2) << TD_SOFT_SHIFT) #define TD_SW ((1UL << 1) << TD_SOFT_SHIFT) #define TD_WIRED ((1UL << 0) << TD_SOFT_SHIFT) #define TD_L (1UL << 6) #define TD_CP (1UL << 5) #define TD_CV (1UL << 4) #define TD_E (1UL << 3) #define TD_P (1UL << 2) #define TD_W (1UL << 1) #define TD_G (1UL << 0) #define TV_SIZE_BITS (TD_SIZE_BITS) #define TV_VPN(va, sz) ((((va) >> TTE_PAGE_SHIFT(sz)) << TV_SIZE_BITS) | sz) #define TTE_SIZE_SPREAD (3) -#define TTE_PAGE_SHIFT(sz) \ +#define TTE_PAGE_SHIFT(sz) \ (PAGE_SHIFT + ((sz) * TTE_SIZE_SPREAD)) -#define TTE_GET_SIZE(tp) \ +#define TTE_GET_SIZE(tp) \ (((tp)->tte_data >> TD_SIZE_SHIFT) & TD_SIZE_MASK) -#define TTE_GET_PAGE_SHIFT(tp) \ +#define TTE_GET_PAGE_SHIFT(tp) \ TTE_PAGE_SHIFT(TTE_GET_SIZE(tp)) -#define TTE_GET_PAGE_SIZE(tp) \ +#define TTE_GET_PAGE_SIZE(tp) \ (1 << TTE_GET_PAGE_SHIFT(tp)) -#define TTE_GET_PAGE_MASK(tp) \ +#define TTE_GET_PAGE_MASK(tp) \ (TTE_GET_PAGE_SIZE(tp) - 1) -#define TTE_GET_PA(tp) \ +#define TTE_GET_PA(tp) \ ((tp)->tte_data & (TD_PA_MASK << TD_PA_SHIFT)) -#define TTE_GET_VPN(tp) \ +#define TTE_GET_VPN(tp) \ ((tp)->tte_vpn >> TV_SIZE_BITS) -#define TTE_GET_VA(tp) \ +#define TTE_GET_VA(tp) \ (TTE_GET_VPN(tp) << TTE_GET_PAGE_SHIFT(tp)) -#define TTE_GET_PMAP(tp) \ - (((tp)->tte_data & TD_P) != 0 ? \ - (kernel_pmap) : \ - (PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)(tp)))->md.pmap)) -#define TTE_ZERO(tp) \ +#define TTE_GET_PMAP(tp) \ + (((tp)->tte_data & TD_P) != 0 ? (kernel_pmap) : \ + (PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)(tp)))->md.pmap)) +#define TTE_ZERO(tp) \ memset(tp, 0, sizeof(*tp)) struct pmap; struct tte { u_long tte_vpn; u_long tte_data; TAILQ_ENTRY(tte) tte_link; }; static __inline int tte_match(struct tte *tp, vm_offset_t va) { + return (((tp->tte_data & TD_V) != 0) && (tp->tte_vpn == TV_VPN(va, TTE_GET_SIZE(tp)))); } #endif /* !_MACHINE_TTE_H_ */ Index: head/sys/sparc64/sparc64/genassym.c =================================================================== --- head/sys/sparc64/sparc64/genassym.c (revision 205257) +++ head/sys/sparc64/sparc64/genassym.c (revision 205258) @@ -1,309 +1,309 @@ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)genassym.c 5.11 (Berkeley) 5/10/91 */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #ifdef SUN4U #include #endif #ifdef SUN4V #include #endif #include #include #include #include #include #ifdef SUN4V #include #include #endif #include ASSYM(KERNBASE, KERNBASE); ASSYM(KSTACK_PAGES, KSTACK_PAGES); ASSYM(PCPU_PAGES, PCPU_PAGES); ASSYM(TAR_VPN_SHIFT, TAR_VPN_SHIFT); #ifdef SUN4U ASSYM(TLB_DEMAP_ALL, TLB_DEMAP_ALL); #endif ASSYM(TLB_DEMAP_CONTEXT, TLB_DEMAP_CONTEXT); ASSYM(TLB_DEMAP_NUCLEUS, TLB_DEMAP_NUCLEUS); ASSYM(TLB_DEMAP_PAGE, TLB_DEMAP_PAGE); ASSYM(TLB_DEMAP_PRIMARY, TLB_DEMAP_PRIMARY); ASSYM(INT_SHIFT, INT_SHIFT); ASSYM(PTR_SHIFT, PTR_SHIFT); ASSYM(PAGE_SHIFT, PAGE_SHIFT); ASSYM(PAGE_SHIFT_8K, PAGE_SHIFT_8K); ASSYM(PAGE_SHIFT_4M, PAGE_SHIFT_4M); ASSYM(PAGE_SIZE, PAGE_SIZE); ASSYM(PAGE_SIZE_4M, PAGE_SIZE_4M); #ifdef SMP ASSYM(CSA_PCPU, offsetof(struct cpu_start_args, csa_pcpu)); ASSYM(CSA_STATE, offsetof(struct cpu_start_args, csa_state)); #ifdef SUN4U ASSYM(CSA_MID, offsetof(struct cpu_start_args, csa_mid)); ASSYM(CSA_STICK, offsetof(struct cpu_start_args, csa_stick)); ASSYM(CSA_TICK, offsetof(struct cpu_start_args, csa_tick)); ASSYM(CSA_TTES, offsetof(struct cpu_start_args, csa_ttes)); ASSYM(CSA_VER, offsetof(struct cpu_start_args, csa_ver)); #endif #ifdef SUN4V ASSYM(CSA_CPUID, offsetof(struct cpu_start_args, csa_cpuid)); #endif #endif #ifdef SUN4U ASSYM(DC_SIZE, offsetof(struct cacheinfo, dc_size)); ASSYM(DC_LINESIZE, offsetof(struct cacheinfo, dc_linesize)); ASSYM(IC_SIZE, offsetof(struct cacheinfo, ic_size)); ASSYM(IC_LINESIZE, offsetof(struct cacheinfo, ic_linesize)); #endif #ifdef SMP ASSYM(ICA_PA, offsetof(struct ipi_cache_args, ica_pa)); #endif ASSYM(KTR_SIZEOF, sizeof(struct ktr_entry)); ASSYM(KTR_LINE, offsetof(struct ktr_entry, ktr_line)); ASSYM(KTR_FILE, offsetof(struct ktr_entry, ktr_file)); ASSYM(KTR_DESC, offsetof(struct ktr_entry, ktr_desc)); ASSYM(KTR_CPU, offsetof(struct ktr_entry, ktr_cpu)); ASSYM(KTR_TIMESTAMP, offsetof(struct ktr_entry, ktr_timestamp)); ASSYM(KTR_PARM1, offsetof(struct ktr_entry, ktr_parms[0])); ASSYM(KTR_PARM2, offsetof(struct ktr_entry, ktr_parms[1])); ASSYM(KTR_PARM3, offsetof(struct ktr_entry, ktr_parms[2])); ASSYM(KTR_PARM4, offsetof(struct ktr_entry, ktr_parms[3])); ASSYM(KTR_PARM5, offsetof(struct ktr_entry, ktr_parms[4])); ASSYM(KTR_PARM6, offsetof(struct ktr_entry, ktr_parms[5])); ASSYM(TTE_SHIFT, TTE_SHIFT); #ifdef SUN4U ASSYM(TTE_VPN, offsetof(struct tte, tte_vpn)); ASSYM(TTE_DATA, offsetof(struct tte, tte_data)); ASSYM(TD_V, TD_V); ASSYM(TD_EXEC, TD_EXEC); ASSYM(TD_REF, TD_REF); ASSYM(TD_SW, TD_SW); ASSYM(TD_L, TD_L); ASSYM(TD_CP, TD_CP); ASSYM(TD_CV, TD_CV); ASSYM(TD_W, TD_W); ASSYM(TS_MIN, TS_MIN); ASSYM(TS_MAX, TS_MAX); ASSYM(TLB_DAR_SLOT_SHIFT, TLB_DAR_SLOT_SHIFT); -ASSYM(TLB_PCXR_PGSZ_MASK, TLB_PCXR_PGSZ_MASK); +ASSYM(TLB_CXR_PGSZ_MASK, TLB_CXR_PGSZ_MASK); ASSYM(TLB_DIRECT_TO_TTE_MASK, TLB_DIRECT_TO_TTE_MASK); ASSYM(TV_SIZE_BITS, TV_SIZE_BITS); #endif #ifdef SUN4V ASSYM(VTD_REF, VTD_REF); ASSYM(VTD_W, VTD_W); ASSYM(VTD_SW_W, VTD_SW_W); ASSYM(VTD_LOCK, VTD_LOCK); ASSYM(THE_SHIFT, THE_SHIFT); ASSYM(PM_HASHSCRATCH, offsetof(struct pmap, pm_hashscratch)); ASSYM(PM_TSBSCRATCH, offsetof(struct pmap, pm_tsbscratch)); ASSYM(PM_TSB_RA, offsetof(struct pmap, pm_tsb_ra)); ASSYM(PM_TLBACTIVE, offsetof(struct pmap, pm_tlbactive)); ASSYM(HASH_ENTRY_SHIFT, HASH_ENTRY_SHIFT); #endif ASSYM(V_INTR, offsetof(struct vmmeter, v_intr)); ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(PC_CPUMASK, offsetof(struct pcpu, pc_cpumask)); ASSYM(PC_IRHEAD, offsetof(struct pcpu, pc_irhead)); ASSYM(PC_IRTAIL, offsetof(struct pcpu, pc_irtail)); ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree)); ASSYM(PC_CNT, offsetof(struct pcpu, pc_cnt)); ASSYM(PC_SIZEOF, sizeof(struct pcpu)); #ifdef SUN4V ASSYM(PC_CPU_Q_RA, offsetof(struct pcpu, pc_cpu_q_ra)); ASSYM(PC_CPU_Q_SIZE, offsetof(struct pcpu, pc_cpu_q_size)); ASSYM(PC_DEV_Q_RA, offsetof(struct pcpu, pc_dev_q_ra)); ASSYM(PC_DEV_Q_SIZE, offsetof(struct pcpu, pc_dev_q_size)); ASSYM(PC_RQ_BASE, offsetof(struct pcpu, pc_rq_ra)); ASSYM(PC_RQ_SIZE, offsetof(struct pcpu, pc_rq_size)); ASSYM(PC_NRQ_BASE, offsetof(struct pcpu, pc_nrq_ra)); ASSYM(PC_NRQ_SIZE, offsetof(struct pcpu, pc_nrq_size)); ASSYM(PC_MONDO_DATA, offsetof(struct pcpu, pc_mondo_data)); ASSYM(PC_MONDO_DATA_RA, offsetof(struct pcpu, pc_mondo_data_ra)); ASSYM(PC_KWBUF_FULL, offsetof(struct pcpu, pc_kwbuf_full)); ASSYM(PC_KWBUF_SP, offsetof(struct pcpu, pc_kwbuf_sp)); ASSYM(PC_KWBUF, offsetof(struct pcpu, pc_kwbuf)); ASSYM(PC_PAD, offsetof(struct pcpu, pad)); ASSYM(PC_PMAP, offsetof(struct pcpu, pc_curpmap)); ASSYM(PC_TSBWBUF, offsetof(struct pcpu, pc_tsbwbuf)); ASSYM(PCB_KSTACK, offsetof(struct pcb, pcb_kstack)); ASSYM(PCB_TSTATE, offsetof(struct pcb, pcb_tstate)); ASSYM(PCB_TPC, offsetof(struct pcb, pcb_tpc)); ASSYM(PCB_TNPC, offsetof(struct pcb, pcb_tnpc)); ASSYM(PCB_TT, offsetof(struct pcb, pcb_tt)); ASSYM(PCB_SFAR, offsetof(struct pcb, pcb_sfar)); ASSYM(PM_TSB_MISS_COUNT, offsetof(struct pmap, pm_tsb_miss_count)); ASSYM(PM_TSB_CAP_MISS_COUNT, offsetof(struct pmap, pm_tsb_cap_miss_count)); #endif #ifdef SUN4U ASSYM(PC_CACHE, offsetof(struct pcpu, pc_cache)); ASSYM(PC_MID, offsetof(struct pcpu, pc_mid)); ASSYM(PC_PMAP, offsetof(struct pcpu, pc_pmap)); ASSYM(PC_TLB_CTX, offsetof(struct pcpu, pc_tlb_ctx)); ASSYM(PC_TLB_CTX_MAX, offsetof(struct pcpu, pc_tlb_ctx_max)); ASSYM(PC_TLB_CTX_MIN, offsetof(struct pcpu, pc_tlb_ctx_min)); #endif ASSYM(IR_NEXT, offsetof(struct intr_request, ir_next)); ASSYM(IR_FUNC, offsetof(struct intr_request, ir_func)); ASSYM(IR_ARG, offsetof(struct intr_request, ir_arg)); ASSYM(IR_PRI, offsetof(struct intr_request, ir_pri)); ASSYM(IR_VEC, offsetof(struct intr_request, ir_vec)); #ifdef SMP ASSYM(ITA_MASK, offsetof(struct ipi_tlb_args, ita_mask)); ASSYM(ITA_PMAP, offsetof(struct ipi_tlb_args, ita_pmap)); ASSYM(ITA_START, offsetof(struct ipi_tlb_args, ita_start)); ASSYM(ITA_END, offsetof(struct ipi_tlb_args, ita_end)); ASSYM(ITA_VA, offsetof(struct ipi_tlb_args, ita_va)); #endif ASSYM(IV_FUNC, offsetof(struct intr_vector, iv_func)); ASSYM(IV_ARG, offsetof(struct intr_vector, iv_arg)); ASSYM(IV_PRI, offsetof(struct intr_vector, iv_pri)); ASSYM(TDF_ASTPENDING, TDF_ASTPENDING); ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED); ASSYM(MD_UTRAP, offsetof(struct mdproc, md_utrap)); ASSYM(P_COMM, offsetof(struct proc, p_comm)); ASSYM(P_MD, offsetof(struct proc, p_md)); ASSYM(P_PID, offsetof(struct proc, p_pid)); ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); ASSYM(TD_KSTACK, offsetof(struct thread, td_kstack)); ASSYM(TD_LOCK, offsetof(struct thread, td_lock)); ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); ASSYM(TD_PROC, offsetof(struct thread, td_proc)); ASSYM(TD_MD, offsetof(struct thread, td_md)); ASSYM(MD_SAVED_PIL, offsetof(struct mdthread, md_saved_pil)); ASSYM(PCB_SIZEOF, sizeof(struct pcb)); ASSYM(PCB_RW, offsetof(struct pcb, pcb_rw)); ASSYM(PCB_KFP, offsetof(struct pcb, pcb_kfp)); ASSYM(PCB_UFP, offsetof(struct pcb, pcb_ufp)); ASSYM(PCB_RWSP, offsetof(struct pcb, pcb_rwsp)); ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags)); ASSYM(PCB_NSAVED, offsetof(struct pcb, pcb_nsaved)); ASSYM(PCB_PC, offsetof(struct pcb, pcb_pc)); ASSYM(PCB_SP, offsetof(struct pcb, pcb_sp)); ASSYM(PCB_PAD, offsetof(struct pcb, pcb_pad)); ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); ASSYM(PM_CONTEXT, offsetof(struct pmap, pm_context)); ASSYM(PM_TSB, offsetof(struct pmap, pm_tsb)); ASSYM(_JB_FP, offsetof(struct _jmp_buf, _jb[_JB_FP])); ASSYM(_JB_PC, offsetof(struct _jmp_buf, _jb[_JB_PC])); ASSYM(_JB_SP, offsetof(struct _jmp_buf, _jb[_JB_SP])); ASSYM(_JB_SIGFLAG, offsetof(struct _jmp_buf, _jb[_JB_SIGFLAG])); ASSYM(_JB_SIGMASK, offsetof(struct _jmp_buf, _jb[_JB_SIGMASK])); ASSYM(TF_G0, offsetof(struct trapframe, tf_global[0])); ASSYM(TF_G1, offsetof(struct trapframe, tf_global[1])); ASSYM(TF_G2, offsetof(struct trapframe, tf_global[2])); ASSYM(TF_G3, offsetof(struct trapframe, tf_global[3])); ASSYM(TF_G4, offsetof(struct trapframe, tf_global[4])); ASSYM(TF_G5, offsetof(struct trapframe, tf_global[5])); ASSYM(TF_G6, offsetof(struct trapframe, tf_global[6])); ASSYM(TF_G7, offsetof(struct trapframe, tf_global[7])); ASSYM(TF_O0, offsetof(struct trapframe, tf_out[0])); ASSYM(TF_O1, offsetof(struct trapframe, tf_out[1])); ASSYM(TF_O2, offsetof(struct trapframe, tf_out[2])); ASSYM(TF_O3, offsetof(struct trapframe, tf_out[3])); ASSYM(TF_O4, offsetof(struct trapframe, tf_out[4])); ASSYM(TF_O5, offsetof(struct trapframe, tf_out[5])); ASSYM(TF_O6, offsetof(struct trapframe, tf_out[6])); ASSYM(TF_O7, offsetof(struct trapframe, tf_out[7])); ASSYM(TF_FPRS, offsetof(struct trapframe, tf_fprs)); ASSYM(TF_FSR, offsetof(struct trapframe, tf_fsr)); ASSYM(TF_GSR, offsetof(struct trapframe, tf_gsr)); ASSYM(TF_PIL, offsetof(struct trapframe, tf_pil)); #ifdef SUN4U ASSYM(TF_LEVEL, offsetof(struct trapframe, tf_level)); ASSYM(TF_SFAR, offsetof(struct trapframe, tf_sfar)); ASSYM(TF_SFSR, offsetof(struct trapframe, tf_sfsr)); ASSYM(TF_TAR, offsetof(struct trapframe, tf_tar)); ASSYM(TF_TYPE, offsetof(struct trapframe, tf_type)); ASSYM(TF_Y, offsetof(struct trapframe, tf_y)); #endif #ifdef SUN4V ASSYM(TF_ASI, offsetof(struct trapframe, tf_asi)); #endif ASSYM(TF_TNPC, offsetof(struct trapframe, tf_tnpc)); ASSYM(TF_TPC, offsetof(struct trapframe, tf_tpc)); ASSYM(TF_TSTATE, offsetof(struct trapframe, tf_tstate)); ASSYM(TF_WSTATE, offsetof(struct trapframe, tf_wstate)); ASSYM(TF_SIZEOF, sizeof(struct trapframe)); ASSYM(VM_MIN_DIRECT_ADDRESS, VM_MIN_DIRECT_ADDRESS); ASSYM(VM_MIN_PROM_ADDRESS, VM_MIN_PROM_ADDRESS); ASSYM(VM_MAX_PROM_ADDRESS, VM_MAX_PROM_ADDRESS); Index: head/sys/sparc64/sparc64/pmap.c =================================================================== --- head/sys/sparc64/sparc64/pmap.c (revision 205257) +++ head/sys/sparc64/sparc64/pmap.c (revision 205258) @@ -1,2013 +1,2013 @@ /*- * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * In addition to hardware address maps, this module is called upon to * provide software-use-only maps which may or may not be stored in the * same form as hardware maps. These pseudo-maps are used to store * intermediate results from copy operations to and from address spaces. * * Since the information managed by this module is also stored by the * logical address mapping module, this module may throw away valid virtual * to physical mappings at almost any time. However, invalidations of * mappings must be done as requested. * * In order to cope with hardware architectures which make virtual to * physical map invalidates expensive, this module may delay invalidate * reduced protection operations until such time as they are actually * necessary. This module is given full information as to which processors * are currently using which maps, and to when physical maps must be made * correct. */ #include "opt_kstack_pages.h" #include "opt_msgbuf.h" #include "opt_pmap.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define PMAP_DEBUG #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif /* XXX */ #include "opt_sched.h" #ifndef SCHED_4BSD #error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock." #endif extern struct mtx sched_lock; /* * Virtual address of message buffer */ struct msgbuf *msgbufp; /* * Map of physical memory reagions */ vm_paddr_t phys_avail[128]; static struct ofw_mem_region mra[128]; struct ofw_mem_region sparc64_memreg[128]; int sparc64_nmemreg; static struct ofw_map translations[128]; static int translations_size; static vm_offset_t pmap_idle_map; static vm_offset_t pmap_temp_map_1; static vm_offset_t pmap_temp_map_2; /* * First and last available kernel virtual addresses */ vm_offset_t virtual_avail; vm_offset_t virtual_end; vm_offset_t kernel_vm_end; vm_offset_t vm_max_kernel_address; /* * Kernel pmap */ struct pmap kernel_pmap_store; /* * Allocate physical memory for use in pmap_bootstrap. */ static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. * * The page queues and pmap must be locked. */ static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired); extern int tl1_immu_miss_patch_1[]; extern int tl1_immu_miss_patch_2[]; extern int tl1_dmmu_miss_patch_1[]; extern int tl1_dmmu_miss_patch_2[]; extern int tl1_dmmu_prot_patch_1[]; extern int tl1_dmmu_prot_patch_2[]; /* * If user pmap is processed with pmap_remove and with pmap_remove and the * resident count drops to 0, there are no more pages to remove, so we * need not continue. */ #define PMAP_REMOVE_DONE(pm) \ ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) /* * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() * and pmap_protect() instead of trying each virtual address. */ #define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); PMAP_STATS_VAR(pmap_nenter); PMAP_STATS_VAR(pmap_nenter_update); PMAP_STATS_VAR(pmap_nenter_replace); PMAP_STATS_VAR(pmap_nenter_new); PMAP_STATS_VAR(pmap_nkenter); PMAP_STATS_VAR(pmap_nkenter_oc); PMAP_STATS_VAR(pmap_nkenter_stupid); PMAP_STATS_VAR(pmap_nkremove); PMAP_STATS_VAR(pmap_nqenter); PMAP_STATS_VAR(pmap_nqremove); PMAP_STATS_VAR(pmap_ncache_enter); PMAP_STATS_VAR(pmap_ncache_enter_c); PMAP_STATS_VAR(pmap_ncache_enter_oc); PMAP_STATS_VAR(pmap_ncache_enter_cc); PMAP_STATS_VAR(pmap_ncache_enter_coc); PMAP_STATS_VAR(pmap_ncache_enter_nc); PMAP_STATS_VAR(pmap_ncache_enter_cnc); PMAP_STATS_VAR(pmap_ncache_remove); PMAP_STATS_VAR(pmap_ncache_remove_c); PMAP_STATS_VAR(pmap_ncache_remove_oc); PMAP_STATS_VAR(pmap_ncache_remove_cc); PMAP_STATS_VAR(pmap_ncache_remove_coc); PMAP_STATS_VAR(pmap_ncache_remove_nc); PMAP_STATS_VAR(pmap_nzero_page); PMAP_STATS_VAR(pmap_nzero_page_c); PMAP_STATS_VAR(pmap_nzero_page_oc); PMAP_STATS_VAR(pmap_nzero_page_nc); PMAP_STATS_VAR(pmap_nzero_page_area); PMAP_STATS_VAR(pmap_nzero_page_area_c); PMAP_STATS_VAR(pmap_nzero_page_area_oc); PMAP_STATS_VAR(pmap_nzero_page_area_nc); PMAP_STATS_VAR(pmap_nzero_page_idle); PMAP_STATS_VAR(pmap_nzero_page_idle_c); PMAP_STATS_VAR(pmap_nzero_page_idle_oc); PMAP_STATS_VAR(pmap_nzero_page_idle_nc); PMAP_STATS_VAR(pmap_ncopy_page); PMAP_STATS_VAR(pmap_ncopy_page_c); PMAP_STATS_VAR(pmap_ncopy_page_oc); PMAP_STATS_VAR(pmap_ncopy_page_nc); PMAP_STATS_VAR(pmap_ncopy_page_dc); PMAP_STATS_VAR(pmap_ncopy_page_doc); PMAP_STATS_VAR(pmap_ncopy_page_sc); PMAP_STATS_VAR(pmap_ncopy_page_soc); PMAP_STATS_VAR(pmap_nnew_thread); PMAP_STATS_VAR(pmap_nnew_thread_oc); /* * Quick sort callout for comparing memory regions */ static int mr_cmp(const void *a, const void *b); static int om_cmp(const void *a, const void *b); static int mr_cmp(const void *a, const void *b) { const struct ofw_mem_region *mra; const struct ofw_mem_region *mrb; mra = a; mrb = b; if (mra->mr_start < mrb->mr_start) return (-1); else if (mra->mr_start > mrb->mr_start) return (1); else return (0); } static int om_cmp(const void *a, const void *b) { const struct ofw_map *oma; const struct ofw_map *omb; oma = a; omb = b; if (oma->om_start < omb->om_start) return (-1); else if (oma->om_start > omb->om_start) return (1); else return (0); } /* * Bootstrap the system enough to run with virtual memory. */ void pmap_bootstrap(u_int cpu_impl) { struct pmap *pm; struct tte *tp; vm_offset_t off; vm_offset_t va; vm_paddr_t pa; vm_size_t physsz; vm_size_t virtsz; phandle_t pmem; phandle_t vmem; int sz; int i; int j; /* * Find out what physical memory is available from the PROM and * initialize the phys_avail array. This must be done before * pmap_bootstrap_alloc is called. */ if ((pmem = OF_finddevice("/memory")) == -1) panic("pmap_bootstrap: finddevice /memory"); if ((sz = OF_getproplen(pmem, "available")) == -1) panic("pmap_bootstrap: getproplen /memory/available"); if (sizeof(phys_avail) < sz) panic("pmap_bootstrap: phys_avail too small"); if (sizeof(mra) < sz) panic("pmap_bootstrap: mra too small"); bzero(mra, sz); if (OF_getprop(pmem, "available", mra, sz) == -1) panic("pmap_bootstrap: getprop /memory/available"); sz /= sizeof(*mra); CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); qsort(mra, sz, sizeof (*mra), mr_cmp); physsz = 0; getenv_quad("hw.physmem", &physmem); physmem = btoc(physmem); for (i = 0, j = 0; i < sz; i++, j += 2) { CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, mra[i].mr_size); if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { if (btoc(physsz) < physmem) { phys_avail[j] = mra[i].mr_start; phys_avail[j + 1] = mra[i].mr_start + (ctob(physmem) - physsz); physsz = ctob(physmem); } break; } phys_avail[j] = mra[i].mr_start; phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; physsz += mra[i].mr_size; } physmem = btoc(physsz); /* * Calculate the size of kernel virtual memory, and the size and mask * for the kernel TSB based on the phsyical memory size but limited * by letting the kernel TSB take up no more than half of the dTLB * slots available for locked entries. */ virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); virtsz = MIN(virtsz, (dtlb_slots / 2 * PAGE_SIZE_4M) << (PAGE_SHIFT - TTE_SHIFT)); vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; if (kernel_tlb_slots + PCPU_PAGES + tsb_kernel_size / PAGE_SIZE_4M + 1 /* PROM page */ + 1 /* spare */ > dtlb_slots) panic("pmap_bootstrap: insufficient dTLB entries"); if (kernel_tlb_slots + 1 /* PROM page */ + 1 /* spare */ > itlb_slots) panic("pmap_bootstrap: insufficient iTLB entries"); /* * Allocate the kernel TSB and lock it in the TLB. */ pa = pmap_bootstrap_alloc(tsb_kernel_size); if (pa & PAGE_MASK_4M) panic("pmap_bootstrap: tsb unaligned\n"); tsb_kernel_phys = pa; tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); pmap_map_tsb(); bzero(tsb_kernel, tsb_kernel_size); /* * Allocate and map the dynamic per-CPU area for the BSP. */ pa = pmap_bootstrap_alloc(DPCPU_SIZE); dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa); /* * Allocate and map the message buffer. */ pa = pmap_bootstrap_alloc(MSGBUF_SIZE); msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa); /* * Patch the virtual address and the tsb mask into the trap table. */ #define SETHI(rd, imm22) \ (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ EIF_IMM((imm22) >> 10, 22)) #define OR_R_I_R(rd, imm13, rs1) \ (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) #define PATCH(addr) do { \ if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ panic("pmap_boostrap: patched instructions have changed"); \ addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ flush(addr); \ flush(addr + 1); \ flush(addr + 2); \ } while (0) PATCH(tl1_immu_miss_patch_1); PATCH(tl1_immu_miss_patch_2); PATCH(tl1_dmmu_miss_patch_1); PATCH(tl1_dmmu_miss_patch_2); PATCH(tl1_dmmu_prot_patch_1); PATCH(tl1_dmmu_prot_patch_2); /* * Enter fake 8k pages for the 4MB kernel pages, so that * pmap_kextract() will work for them. */ for (i = 0; i < kernel_tlb_slots; i++) { pa = kernel_tlbs[i].te_pa; va = kernel_tlbs[i].te_va; for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { tp = tsb_kvtotte(va + off); tp->tte_vpn = TV_VPN(va + off, TS_8K); tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; } } /* * Set the start and end of KVA. The kernel is loaded starting * at the first available 4MB super page, so we advance to the * end of the last one used for it. */ virtual_avail = KERNBASE + kernel_tlb_slots * PAGE_SIZE_4M; virtual_end = vm_max_kernel_address; kernel_vm_end = vm_max_kernel_address; /* * Allocate kva space for temporary mappings. */ pmap_idle_map = virtual_avail; virtual_avail += PAGE_SIZE * DCACHE_COLORS; pmap_temp_map_1 = virtual_avail; virtual_avail += PAGE_SIZE * DCACHE_COLORS; pmap_temp_map_2 = virtual_avail; virtual_avail += PAGE_SIZE * DCACHE_COLORS; /* * Allocate a kernel stack with guard page for thread0 and map it * into the kernel TSB. We must ensure that the virtual address is * coloured properly, since we're allocating from phys_avail so the * memory won't have an associated vm_page_t. */ pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); kstack0_phys = pa; virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * PAGE_SIZE; kstack0 = virtual_avail; virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), ("pmap_bootstrap: kstack0 miscoloured")); for (i = 0; i < KSTACK_PAGES; i++) { pa = kstack0_phys + i * PAGE_SIZE; va = kstack0 + i * PAGE_SIZE; tp = tsb_kvtotte(va); tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; } /* * Calculate the last available physical address. */ for (i = 0; phys_avail[i + 2] != 0; i += 2) ; Maxmem = sparc64_btop(phys_avail[i + 1]); /* * Add the PROM mappings to the kernel TSB. */ if ((vmem = OF_finddevice("/virtual-memory")) == -1) panic("pmap_bootstrap: finddevice /virtual-memory"); if ((sz = OF_getproplen(vmem, "translations")) == -1) panic("pmap_bootstrap: getproplen translations"); if (sizeof(translations) < sz) panic("pmap_bootstrap: translations too small"); bzero(translations, sz); if (OF_getprop(vmem, "translations", translations, sz) == -1) panic("pmap_bootstrap: getprop /virtual-memory/translations"); sz /= sizeof(*translations); translations_size = sz; CTR0(KTR_PMAP, "pmap_bootstrap: translations"); qsort(translations, sz, sizeof (*translations), om_cmp); for (i = 0; i < sz; i++) { CTR3(KTR_PMAP, "translation: start=%#lx size=%#lx tte=%#lx", translations[i].om_start, translations[i].om_size, translations[i].om_tte); if ((translations[i].om_tte & TD_V) == 0) continue; if (translations[i].om_start < VM_MIN_PROM_ADDRESS || translations[i].om_start > VM_MAX_PROM_ADDRESS) continue; for (off = 0; off < translations[i].om_size; off += PAGE_SIZE) { va = translations[i].om_start + off; tp = tsb_kvtotte(va); tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = ((translations[i].om_tte & ~((TD_SOFT2_MASK << TD_SOFT2_SHIFT) | (cpu_impl < CPU_IMPL_ULTRASPARCIII ? (TD_DIAG_SF_MASK << TD_DIAG_SF_SHIFT) : (TD_RSVD_CH_MASK << TD_RSVD_CH_SHIFT)) | (TD_SOFT_MASK << TD_SOFT_SHIFT))) | TD_EXEC) + off; } } /* * Get the available physical memory ranges from /memory/reg. These * are only used for kernel dumps, but it may not be wise to do PROM * calls in that situation. */ if ((sz = OF_getproplen(pmem, "reg")) == -1) panic("pmap_bootstrap: getproplen /memory/reg"); if (sizeof(sparc64_memreg) < sz) panic("pmap_bootstrap: sparc64_memreg too small"); if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) panic("pmap_bootstrap: getprop /memory/reg"); sparc64_nmemreg = sz / sizeof(*sparc64_memreg); /* * Initialize the kernel pmap (which is statically allocated). * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization * but sparc64 start up is not ready to initialize mutexes yet. * It is called in machdep.c. */ pm = kernel_pmap; for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = TLB_CTX_KERNEL; pm->pm_active = ~0; /* * Flush all non-locked TLB entries possibly left over by the * firmware. */ tlb_flush_nonlocked(); } void pmap_map_tsb(void) { vm_offset_t va; vm_paddr_t pa; u_long data; register_t s; int i; s = intr_disable(); /* * Map the 4MB TSB pages. */ for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { va = (vm_offset_t)tsb_kernel + i; pa = tsb_kernel_phys + i; data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | TLB_TAR_CTX(TLB_CTX_KERNEL)); stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); } /* * Set the secondary context to be the kernel context (needed for * FP block operations in the kernel). */ stxa(AA_DMMU_SCXR, ASI_DMMU, (ldxa(AA_DMMU_SCXR, ASI_DMMU) & - TLB_SCXR_PGSZ_MASK) | TLB_CTX_KERNEL); + TLB_CXR_PGSZ_MASK) | TLB_CTX_KERNEL); flush(KERNBASE); intr_restore(s); } /* * Allocate a physical page of memory directly from the phys_avail map. * Can only be called from pmap_bootstrap before avail start and end are * calculated. */ static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size) { vm_paddr_t pa; int i; size = roundup(size, PAGE_SIZE * DCACHE_COLORS); for (i = 0; phys_avail[i + 1] != 0; i += 2) { if (phys_avail[i + 1] - phys_avail[i] < size) continue; pa = phys_avail[i]; phys_avail[i] += size; return (pa); } panic("pmap_bootstrap_alloc"); } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.tte_list); m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); m->md.flags = 0; m->md.pmap = NULL; } /* * Initialize the pmap module. */ void pmap_init(void) { vm_offset_t addr; vm_size_t size; int result; int i; for (i = 0; i < translations_size; i++) { addr = translations[i].om_start; size = translations[i].om_size; if ((translations[i].om_tte & TD_V) == 0) continue; if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) continue; result = vm_map_find(kernel_map, NULL, 0, &addr, size, VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); if (result != KERN_SUCCESS || addr != translations[i].om_start) panic("pmap_init: vm_map_find"); } } /* * Extract the physical page address associated with the given * map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pm, vm_offset_t va) { struct tte *tp; vm_paddr_t pa; if (pm == kernel_pmap) return (pmap_kextract(va)); PMAP_LOCK(pm); tp = tsb_tte_lookup(pm, va); if (tp == NULL) pa = 0; else pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); PMAP_UNLOCK(pm); return (pa); } /* * Atomically extract and hold the physical page with the given * pmap and virtual address pair if that mapping permits the given * protection. */ vm_page_t pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) { struct tte *tp; vm_page_t m; m = NULL; vm_page_lock_queues(); if (pm == kernel_pmap) { if (va >= VM_MIN_DIRECT_ADDRESS) { tp = NULL; m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); vm_page_hold(m); } else { tp = tsb_kvtotte(va); if ((tp->tte_data & TD_V) == 0) tp = NULL; } } else { PMAP_LOCK(pm); tp = tsb_tte_lookup(pm, va); } if (tp != NULL && ((tp->tte_data & TD_SW) || (prot & VM_PROT_WRITE) == 0)) { m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); vm_page_hold(m); } vm_page_unlock_queues(); if (pm != kernel_pmap) PMAP_UNLOCK(pm); return (m); } /* * Extract the physical page address associated with the given kernel virtual * address. */ vm_paddr_t pmap_kextract(vm_offset_t va) { struct tte *tp; if (va >= VM_MIN_DIRECT_ADDRESS) return (TLB_DIRECT_TO_PHYS(va)); tp = tsb_kvtotte(va); if ((tp->tte_data & TD_V) == 0) return (0); return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); } int pmap_cache_enter(vm_page_t m, vm_offset_t va) { struct tte *tp; int color; mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_cache_enter: fake page")); PMAP_STATS_INC(pmap_ncache_enter); /* * Find the color for this virtual address and note the added mapping. */ color = DCACHE_COLOR(va); m->md.colors[color]++; /* * If all existing mappings have the same color, the mapping is * cacheable. */ if (m->md.color == color) { KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, ("pmap_cache_enter: cacheable, mappings of other color")); if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_enter_c); else PMAP_STATS_INC(pmap_ncache_enter_oc); return (1); } /* * If there are no mappings of the other color, and the page still has * the wrong color, this must be a new mapping. Change the color to * match the new mapping, which is cacheable. We must flush the page * from the cache now. */ if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { KASSERT(m->md.colors[color] == 1, ("pmap_cache_enter: changing color, not new mapping")); dcache_page_inval(VM_PAGE_TO_PHYS(m)); m->md.color = color; if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_enter_cc); else PMAP_STATS_INC(pmap_ncache_enter_coc); return (1); } /* * If the mapping is already non-cacheable, just return. */ if (m->md.color == -1) { PMAP_STATS_INC(pmap_ncache_enter_nc); return (0); } PMAP_STATS_INC(pmap_ncache_enter_cnc); /* * Mark all mappings as uncacheable, flush any lines with the other * color out of the dcache, and set the color to none (-1). */ TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { atomic_clear_long(&tp->tte_data, TD_CV); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } dcache_page_inval(VM_PAGE_TO_PHYS(m)); m->md.color = -1; return (0); } void pmap_cache_remove(vm_page_t m, vm_offset_t va) { struct tte *tp; int color; mtx_assert(&vm_page_queue_mtx, MA_OWNED); CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, m->md.colors[DCACHE_COLOR(va)]); KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_cache_remove: fake page")); KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, ("pmap_cache_remove: no mappings %d <= 0", m->md.colors[DCACHE_COLOR(va)])); PMAP_STATS_INC(pmap_ncache_remove); /* * Find the color for this virtual address and note the removal of * the mapping. */ color = DCACHE_COLOR(va); m->md.colors[color]--; /* * If the page is cacheable, just return and keep the same color, even * if there are no longer any mappings. */ if (m->md.color != -1) { if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_remove_c); else PMAP_STATS_INC(pmap_ncache_remove_oc); return; } KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, ("pmap_cache_remove: uncacheable, no mappings of other color")); /* * If the page is not cacheable (color is -1), and the number of * mappings for this color is not zero, just return. There are * mappings of the other color still, so remain non-cacheable. */ if (m->md.colors[color] != 0) { PMAP_STATS_INC(pmap_ncache_remove_nc); return; } /* * The number of mappings for this color is now zero. Recache the * other colored mappings, and change the page color to the other * color. There should be no lines in the data cache for this page, * so flushing should not be needed. */ TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { atomic_set_long(&tp->tte_data, TD_CV); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } m->md.color = DCACHE_OTHER_COLOR(color); if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_remove_cc); else PMAP_STATS_INC(pmap_ncache_remove_coc); } /* * Map a wired page into kernel virtual address space. */ void pmap_kenter(vm_offset_t va, vm_page_t m) { vm_offset_t ova; struct tte *tp; vm_page_t om; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_STATS_INC(pmap_nkenter); tp = tsb_kvtotte(va); CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { CTR5(KTR_CT2, "pmap_kenter: off colour va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", va, VM_PAGE_TO_PHYS(m), m->object, m->object ? m->object->type : -1, m->pindex); PMAP_STATS_INC(pmap_nkenter_oc); } if ((tp->tte_data & TD_V) != 0) { om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); ova = TTE_GET_VA(tp); if (m == om && va == ova) { PMAP_STATS_INC(pmap_nkenter_stupid); return; } TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); pmap_cache_remove(om, ova); if (va != ova) tlb_page_demap(kernel_pmap, ova); } data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | TD_P | TD_W; if (pmap_cache_enter(m, va) != 0) data |= TD_CV; tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = data; TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); } /* * Map a wired page into kernel virtual address space. This additionally * takes a flag argument wich is or'ed to the TTE data. This is used by * sparc64_bus_mem_map(). * NOTE: if the mapping is non-cacheable, it's the caller's responsibility * to flush entries that might still be in the cache, if applicable. */ void pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) { struct tte *tp; tp = tsb_kvtotte(va); CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", va, pa, tp, tp->tte_data); tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; } /* * Remove a wired page from kernel virtual address space. */ void pmap_kremove(vm_offset_t va) { struct tte *tp; vm_page_t m; mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_STATS_INC(pmap_nkremove); tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); if ((tp->tte_data & TD_V) == 0) return; m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); pmap_cache_remove(m, va); TTE_ZERO(tp); } /* * Inverse of pmap_kenter_flags, used by bus_space_unmap(). */ void pmap_kremove_flags(vm_offset_t va) { struct tte *tp; tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove_flags: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); TTE_ZERO(tp); } /* * Map a range of physical addresses into kernel virtual address space. * * The value passed in *virt is a suggested virtual address for the mapping. * Architectures which can support a direct-mapped physical to virtual region * can return the appropriate address within that region, leaving '*virt' * unchanged. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { return (TLB_PHYS_TO_DIRECT(start)); } /* * Map a list of wired pages into kernel virtual address space. This is * intended for temporary mappings which do not need page modification or * references recorded. Existing mappings in the region are overwritten. */ void pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) { vm_offset_t va; int locked; PMAP_STATS_INC(pmap_nqenter); va = sva; if (!(locked = mtx_owned(&vm_page_queue_mtx))) vm_page_lock_queues(); while (count-- > 0) { pmap_kenter(va, *m); va += PAGE_SIZE; m++; } if (!locked) vm_page_unlock_queues(); tlb_range_demap(kernel_pmap, sva, va); } /* * Remove page mappings from kernel virtual address space. Intended for * temporary mappings entered by pmap_qenter. */ void pmap_qremove(vm_offset_t sva, int count) { vm_offset_t va; int locked; PMAP_STATS_INC(pmap_nqremove); va = sva; if (!(locked = mtx_owned(&vm_page_queue_mtx))) vm_page_lock_queues(); while (count-- > 0) { pmap_kremove(va); va += PAGE_SIZE; } if (!locked) vm_page_unlock_queues(); tlb_range_demap(kernel_pmap, sva, va); } /* * Initialize the pmap associated with process 0. */ void pmap_pinit0(pmap_t pm) { int i; PMAP_LOCK_INIT(pm); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = 0; pm->pm_active = 0; pm->pm_tsb = NULL; pm->pm_tsb_obj = NULL; bzero(&pm->pm_stats, sizeof(pm->pm_stats)); } /* * Initialize a preallocated and zeroed pmap structure, such as one in a * vmspace structure. */ int pmap_pinit(pmap_t pm) { vm_page_t ma[TSB_PAGES]; vm_page_t m; int i; PMAP_LOCK_INIT(pm); /* * Allocate KVA space for the TSB. */ if (pm->pm_tsb == NULL) { pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, TSB_BSIZE); if (pm->pm_tsb == NULL) { PMAP_LOCK_DESTROY(pm); return (0); } } /* * Allocate an object for it. */ if (pm->pm_tsb_obj == NULL) pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); VM_OBJECT_LOCK(pm->pm_tsb_obj); for (i = 0; i < TSB_PAGES; i++) { m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); m->valid = VM_PAGE_BITS_ALL; m->md.pmap = pm; ma[i] = m; } VM_OBJECT_UNLOCK(pm->pm_tsb_obj); pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = -1; pm->pm_active = 0; bzero(&pm->pm_stats, sizeof(pm->pm_stats)); return (1); } /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pm) { vm_object_t obj; vm_page_t m; struct pcpu *pc; CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", pm->pm_context[curcpu], pm->pm_tsb); KASSERT(pmap_resident_count(pm) == 0, ("pmap_release: resident pages %ld != 0", pmap_resident_count(pm))); /* * After the pmap was freed, it might be reallocated to a new process. * When switching, this might lead us to wrongly assume that we need * not switch contexts because old and new pmap pointer are equal. * Therefore, make sure that this pmap is not referenced by any PCPU * pointer any more. This could happen in two cases: * - A process that referenced the pmap is currently exiting on a CPU. * However, it is guaranteed to not switch in any more after setting * its state to PRS_ZOMBIE. * - A process that referenced this pmap ran on a CPU, but we switched * to a kernel thread, leaving the pmap pointer unchanged. */ mtx_lock_spin(&sched_lock); SLIST_FOREACH(pc, &cpuhead, pc_allcpu) if (pc->pc_pmap == pm) pc->pc_pmap = NULL; mtx_unlock_spin(&sched_lock); obj = pm->pm_tsb_obj; VM_OBJECT_LOCK(obj); KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); while (!TAILQ_EMPTY(&obj->memq)) { m = TAILQ_FIRST(&obj->memq); vm_page_lock_queues(); if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) continue; KASSERT(m->hold_count == 0, ("pmap_release: freeing held tsb page")); m->md.pmap = NULL; m->wire_count--; atomic_subtract_int(&cnt.v_wire_count, 1); vm_page_free_zero(m); vm_page_unlock_queues(); } VM_OBJECT_UNLOCK(obj); pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); PMAP_LOCK_DESTROY(pm); } /* * Grow the number of kernel page table entries. Unneeded. */ void pmap_growkernel(vm_offset_t addr) { panic("pmap_growkernel: can't grow kernel"); } int pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, vm_offset_t va) { vm_page_t m; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); data = atomic_readandclear_long(&tp->tte_data); if ((data & TD_FAKE) == 0) { m = PHYS_TO_VM_PAGE(TD_PA(data)); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); if ((data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((data & TD_PV) != 0) { if ((data & TD_W) != 0) vm_page_dirty(m); if ((data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if (TAILQ_EMPTY(&m->md.tte_list)) vm_page_flag_clear(m, PG_WRITEABLE); pm->pm_stats.resident_count--; } pmap_cache_remove(m, va); } TTE_ZERO(tp); if (PMAP_REMOVE_DONE(pm)) return (0); return (1); } /* * Remove the given range of addresses from the specified map. */ void pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) { struct tte *tp; vm_offset_t va; CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", pm->pm_context[curcpu], start, end); if (PMAP_REMOVE_DONE(pm)) return; vm_page_lock_queues(); PMAP_LOCK(pm); if (end - start > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, start, end, pmap_remove_tte); tlb_context_demap(pm); } else { for (va = start; va < end; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(pm, va)) != NULL && !pmap_remove_tte(pm, NULL, tp, va)) break; tlb_range_demap(pm, start, end - 1); } PMAP_UNLOCK(pm); vm_page_unlock_queues(); } void pmap_remove_all(vm_page_t m) { struct pmap *pm; struct tte *tpn; struct tte *tp; vm_offset_t va; mtx_assert(&vm_page_queue_mtx, MA_OWNED); for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { tpn = TAILQ_NEXT(tp, tte_link); if ((tp->tte_data & TD_PV) == 0) continue; pm = TTE_GET_PMAP(tp); va = TTE_GET_VA(tp); PMAP_LOCK(pm); if ((tp->tte_data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((tp->tte_data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if ((tp->tte_data & TD_W) != 0) vm_page_dirty(m); tp->tte_data &= ~TD_V; tlb_page_demap(pm, va); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); pm->pm_stats.resident_count--; pmap_cache_remove(m, va); TTE_ZERO(tp); PMAP_UNLOCK(pm); } vm_page_flag_clear(m, PG_WRITEABLE); } int pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, vm_offset_t va) { u_long data; vm_page_t m; data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); if ((data & TD_PV) != 0) { m = PHYS_TO_VM_PAGE(TD_PA(data)); if ((data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if ((data & TD_W) != 0) vm_page_dirty(m); } return (1); } /* * Set the physical protection on the specified range of this map as requested. */ void pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va; struct tte *tp; CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", pm->pm_context[curcpu], sva, eva, prot); if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pm, sva, eva); return; } if (prot & VM_PROT_WRITE) return; vm_page_lock_queues(); PMAP_LOCK(pm); if (eva - sva > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); tlb_context_demap(pm); } else { for (va = sva; va < eva; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(pm, va)) != NULL) pmap_protect_tte(pm, NULL, tp, va); tlb_range_demap(pm, sva, eva - 1); } PMAP_UNLOCK(pm); vm_page_unlock_queues(); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. */ void pmap_enter(pmap_t pm, vm_offset_t va, vm_prot_t access, vm_page_t m, vm_prot_t prot, boolean_t wired) { vm_page_lock_queues(); PMAP_LOCK(pm); pmap_enter_locked(pm, va, m, prot, wired); vm_page_unlock_queues(); PMAP_UNLOCK(pm); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. * * The page queues and pmap must be locked. */ static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { struct tte *tp; vm_paddr_t pa; u_long data; int i; mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_LOCK_ASSERT(pm, MA_OWNED); PMAP_STATS_INC(pmap_nenter); pa = VM_PAGE_TO_PHYS(m); /* * If this is a fake page from the device_pager, but it covers actual * physical memory, convert to the real backing page. */ if ((m->flags & PG_FICTITIOUS) != 0) { for (i = 0; phys_avail[i + 1] != 0; i += 2) { if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { m = PHYS_TO_VM_PAGE(pa); break; } } } CTR6(KTR_PMAP, "pmap_enter_locked: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", pm->pm_context[curcpu], m, va, pa, prot, wired); /* * If there is an existing mapping, and the physical address has not * changed, must be protection or wiring change. */ if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { CTR0(KTR_PMAP, "pmap_enter_locked: update"); PMAP_STATS_INC(pmap_nenter_update); /* * Wiring change, just update stats. */ if (wired) { if ((tp->tte_data & TD_WIRED) == 0) { tp->tte_data |= TD_WIRED; pm->pm_stats.wired_count++; } } else { if ((tp->tte_data & TD_WIRED) != 0) { tp->tte_data &= ~TD_WIRED; pm->pm_stats.wired_count--; } } /* * Save the old bits and clear the ones we're interested in. */ data = tp->tte_data; tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); /* * If we're turning off write permissions, sense modify status. */ if ((prot & VM_PROT_WRITE) != 0) { tp->tte_data |= TD_SW; if (wired) tp->tte_data |= TD_W; vm_page_flag_set(m, PG_WRITEABLE); } else if ((data & TD_W) != 0) vm_page_dirty(m); /* * If we're turning on execute permissions, flush the icache. */ if ((prot & VM_PROT_EXECUTE) != 0) { if ((data & TD_EXEC) == 0) icache_page_inval(pa); tp->tte_data |= TD_EXEC; } /* * Delete the old mapping. */ tlb_page_demap(pm, TTE_GET_VA(tp)); } else { /* * If there is an existing mapping, but its for a different * phsyical address, delete the old mapping. */ if (tp != NULL) { CTR0(KTR_PMAP, "pmap_enter_locked: replace"); PMAP_STATS_INC(pmap_nenter_replace); pmap_remove_tte(pm, NULL, tp, va); tlb_page_demap(pm, va); } else { CTR0(KTR_PMAP, "pmap_enter_locked: new"); PMAP_STATS_INC(pmap_nenter_new); } /* * Now set up the data and install the new mapping. */ data = TD_V | TD_8K | TD_PA(pa); if (pm == kernel_pmap) data |= TD_P; if ((prot & VM_PROT_WRITE) != 0) { data |= TD_SW; vm_page_flag_set(m, PG_WRITEABLE); } if (prot & VM_PROT_EXECUTE) { data |= TD_EXEC; icache_page_inval(pa); } /* * If its wired update stats. We also don't need reference or * modify tracking for wired mappings, so set the bits now. */ if (wired) { pm->pm_stats.wired_count++; data |= TD_REF | TD_WIRED; if ((prot & VM_PROT_WRITE) != 0) data |= TD_W; } tsb_tte_enter(pm, m, va, TS_8K, data); } } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_page_t m; vm_pindex_t diff, psize; psize = atop(end - start); m = m_start; PMAP_LOCK(pm); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { pmap_enter_locked(pm, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); m = TAILQ_NEXT(m, listq); } PMAP_UNLOCK(pm); } void pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) { PMAP_LOCK(pm); pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); PMAP_UNLOCK(pm); } void pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } /* * Change the wiring attribute for a map/virtual-address pair. * The mapping must already exist in the pmap. */ void pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) { struct tte *tp; u_long data; PMAP_LOCK(pm); if ((tp = tsb_tte_lookup(pm, va)) != NULL) { if (wired) { data = atomic_set_long(&tp->tte_data, TD_WIRED); if ((data & TD_WIRED) == 0) pm->pm_stats.wired_count++; } else { data = atomic_clear_long(&tp->tte_data, TD_WIRED); if ((data & TD_WIRED) != 0) pm->pm_stats.wired_count--; } } PMAP_UNLOCK(pm); } static int pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) { vm_page_t m; u_long data; if ((tp->tte_data & TD_FAKE) != 0) return (1); if (tsb_tte_lookup(dst_pmap, va) == NULL) { data = tp->tte_data & ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); tsb_tte_enter(dst_pmap, m, va, TS_8K, data); } return (1); } void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { struct tte *tp; vm_offset_t va; if (dst_addr != src_addr) return; vm_page_lock_queues(); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); } else { PMAP_LOCK(src_pmap); PMAP_LOCK(dst_pmap); } if (len > PMAP_TSB_THRESH) { tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, pmap_copy_tte); tlb_context_demap(dst_pmap); } else { for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) pmap_copy_tte(src_pmap, dst_pmap, tp, va); tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); } vm_page_unlock_queues(); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } void pmap_zero_page(vm_page_t m) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page: fake page")); PMAP_STATS_INC(pmap_nzero_page); pa = VM_PAGE_TO_PHYS(m); if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_nc); aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); } else if (m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_c); va = TLB_PHYS_TO_DIRECT(pa); cpu_block_zero((void *)va, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_nzero_page_oc); PMAP_LOCK(kernel_pmap); va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); cpu_block_zero((void *)va, PAGE_SIZE); tlb_page_demap(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } } void pmap_zero_page_area(vm_page_t m, int off, int size) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page_area: fake page")); KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); PMAP_STATS_INC(pmap_nzero_page_area); pa = VM_PAGE_TO_PHYS(m); if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_area_nc); aszero(ASI_PHYS_USE_EC, pa + off, size); } else if (m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_area_c); va = TLB_PHYS_TO_DIRECT(pa); bzero((void *)(va + off), size); } else { PMAP_STATS_INC(pmap_nzero_page_area_oc); PMAP_LOCK(kernel_pmap); va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); bzero((void *)(va + off), size); tlb_page_demap(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } } void pmap_zero_page_idle(vm_page_t m) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page_idle: fake page")); PMAP_STATS_INC(pmap_nzero_page_idle); pa = VM_PAGE_TO_PHYS(m); if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_idle_nc); aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); } else if (m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_idle_c); va = TLB_PHYS_TO_DIRECT(pa); cpu_block_zero((void *)va, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_nzero_page_idle_oc); va = pmap_idle_map + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); cpu_block_zero((void *)va, PAGE_SIZE); tlb_page_demap(kernel_pmap, va); } } void pmap_copy_page(vm_page_t msrc, vm_page_t mdst) { vm_offset_t vdst; vm_offset_t vsrc; vm_paddr_t pdst; vm_paddr_t psrc; struct tte *tp; KASSERT((mdst->flags & PG_FICTITIOUS) == 0, ("pmap_copy_page: fake dst page")); KASSERT((msrc->flags & PG_FICTITIOUS) == 0, ("pmap_copy_page: fake src page")); PMAP_STATS_INC(pmap_ncopy_page); pdst = VM_PAGE_TO_PHYS(mdst); psrc = VM_PAGE_TO_PHYS(msrc); if (msrc->md.color == -1 && mdst->md.color == -1) { PMAP_STATS_INC(pmap_ncopy_page_nc); ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); } else if (msrc->md.color == DCACHE_COLOR(psrc) && mdst->md.color == DCACHE_COLOR(pdst)) { PMAP_STATS_INC(pmap_ncopy_page_c); vdst = TLB_PHYS_TO_DIRECT(pdst); vsrc = TLB_PHYS_TO_DIRECT(psrc); cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); } else if (msrc->md.color == -1) { if (mdst->md.color == DCACHE_COLOR(pdst)) { PMAP_STATS_INC(pmap_ncopy_page_dc); vdst = TLB_PHYS_TO_DIRECT(pdst); ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_ncopy_page_doc); PMAP_LOCK(kernel_pmap); vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); tp = tsb_kvtotte(vdst); tp->tte_data = TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vdst, TS_8K); ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vdst); PMAP_UNLOCK(kernel_pmap); } } else if (mdst->md.color == -1) { if (msrc->md.color == DCACHE_COLOR(psrc)) { PMAP_STATS_INC(pmap_ncopy_page_sc); vsrc = TLB_PHYS_TO_DIRECT(psrc); ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_ncopy_page_soc); PMAP_LOCK(kernel_pmap); vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); tp = tsb_kvtotte(vsrc); tp->tte_data = TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vsrc, TS_8K); ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vsrc); PMAP_UNLOCK(kernel_pmap); } } else { PMAP_STATS_INC(pmap_ncopy_page_oc); PMAP_LOCK(kernel_pmap); vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); tp = tsb_kvtotte(vdst); tp->tte_data = TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vdst, TS_8K); vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); tp = tsb_kvtotte(vsrc); tp->tte_data = TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vsrc, TS_8K); cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vdst); tlb_page_demap(kernel_pmap, vsrc); PMAP_UNLOCK(kernel_pmap); } } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pm, vm_page_t m) { struct tte *tp; int loops; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); loops = 0; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if (TTE_GET_PMAP(tp) == pm) return (TRUE); if (++loops >= 16) break; } return (FALSE); } /* * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { struct tte *tp; int count; count = 0; if ((m->flags & PG_FICTITIOUS) != 0) return (count); mtx_assert(&vm_page_queue_mtx, MA_OWNED); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) if ((tp->tte_data & (TD_PV | TD_WIRED)) == (TD_PV | TD_WIRED)) count++; return (count); } /* * Remove all pages from specified address space, this aids process exit * speeds. This is much faster than pmap_remove n the case of running down * an entire address space. Only works for the current pmap. */ void pmap_remove_pages(pmap_t pm) { } /* * Returns TRUE if the given page has a managed mapping. */ boolean_t pmap_page_is_mapped(vm_page_t m) { struct tte *tp; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); mtx_assert(&vm_page_queue_mtx, MA_OWNED); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) if ((tp->tte_data & TD_PV) != 0) return (TRUE); return (FALSE); } /* * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct tte *tpf; struct tte *tpn; struct tte *tp; u_long data; int count; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (0); count = 0; if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { tpf = tp; do { tpn = TAILQ_NEXT(tp, tte_link); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_REF); if ((data & TD_REF) != 0 && ++count > 4) break; } while ((tp = tpn) != NULL && tp != tpf); } return (count); } boolean_t pmap_is_modified(vm_page_t m) { struct tte *tp; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if ((tp->tte_data & TD_W) != 0) return (TRUE); } return (FALSE); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is elgible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { return (FALSE); } void pmap_clear_modify(vm_page_t m) { struct tte *tp; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_W); if ((data & TD_W) != 0) tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } void pmap_clear_reference(vm_page_t m) { struct tte *tp; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_REF); if ((data & TD_REF) != 0) tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } void pmap_remove_write(vm_page_t m) { struct tte *tp; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || (m->flags & PG_WRITEABLE) == 0) return; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); if ((data & TD_W) != 0) { vm_page_dirty(m); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } vm_page_flag_clear(m, PG_WRITEABLE); } int pmap_mincore(pmap_t pm, vm_offset_t addr) { /* TODO; */ return (0); } /* * Activate a user pmap. The pmap must be activated before its address space * can be accessed in any way. */ void pmap_activate(struct thread *td) { struct vmspace *vm; struct pmap *pm; int context; vm = td->td_proc->p_vmspace; pm = vmspace_pmap(vm); mtx_lock_spin(&sched_lock); context = PCPU_GET(tlb_ctx); if (context == PCPU_GET(tlb_ctx_max)) { tlb_flush_user(); context = PCPU_GET(tlb_ctx_min); } PCPU_SET(tlb_ctx, context + 1); pm->pm_context[curcpu] = context; pm->pm_active |= PCPU_GET(cpumask); PCPU_SET(pmap, pm); stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); stxa(AA_DMMU_PCXR, ASI_DMMU, (ldxa(AA_DMMU_PCXR, ASI_DMMU) & - TLB_PCXR_PGSZ_MASK) | context); + TLB_CXR_PGSZ_MASK) | context); flush(KERNBASE); mtx_unlock_spin(&sched_lock); } void pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) { } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { } Index: head/sys/sparc64/sparc64/swtch.S =================================================================== --- head/sys/sparc64/sparc64/swtch.S (revision 205257) +++ head/sys/sparc64/sparc64/swtch.S (revision 205258) @@ -1,288 +1,288 @@ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include "assym.s" .register %g2, #ignore .register %g3, #ignore /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) save %sp, -CCFSZ, %sp flushw ba %xcc, .Lsw1 mov %g0, %i2 END(cpu_throw) /* * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) */ ENTRY(cpu_switch) save %sp, -CCFSZ, %sp /* * If the current thread was using floating point in the kernel, save * its context. The userland floating point context has already been * saved in that case. */ rd %fprs, %l2 andcc %l2, FPRS_FEF, %g0 bz,a,pt %xcc, 1f nop call savefpctx add PCB_REG, PCB_KFP, %o0 ba,a %xcc, 2f nop /* * If the current thread was using floating point in userland, save * its context. */ 1: sub PCB_REG, TF_SIZEOF, %l2 ldx [%l2 + TF_FPRS], %l3 andcc %l3, FPRS_FEF, %g0 bz,a,pt %xcc, 2f nop call savefpctx add PCB_REG, PCB_UFP, %o0 andn %l3, FPRS_FEF, %l3 stx %l3, [%l2 + TF_FPRS] ldx [PCB_REG + PCB_FLAGS], %l3 or %l3, PCB_FEF, %l3 stx %l3, [PCB_REG + PCB_FLAGS] /* * Flush the windows out to the stack and save the current frame * pointer and program counter. */ 2: flushw wrpr %g0, 0, %cleanwin stx %fp, [PCB_REG + PCB_SP] stx %i7, [PCB_REG + PCB_PC] /* * Load the new thread's frame pointer and program counter, and set * the current thread and pcb. */ .Lsw1: #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "cpu_switch: new td=%p pc=%#lx fp=%#lx" , %g1, %g2, %g3, 8, 9, 10) stx %i1, [%g1 + KTR_PARM1] ldx [%i1 + TD_PCB], %g2 ldx [%g2 + PCB_PC], %g3 stx %g3, [%g1 + KTR_PARM2] ldx [%g2 + PCB_SP], %g3 stx %g3, [%g1 + KTR_PARM3] 10: #endif ldx [%i1 + TD_PCB], %l0 stx %i1, [PCPU(CURTHREAD)] stx %l0, [PCPU(CURPCB)] wrpr %g0, PSTATE_NORMAL, %pstate mov %l0, PCB_REG wrpr %g0, PSTATE_ALT, %pstate mov %l0, PCB_REG wrpr %g0, PSTATE_KERNEL, %pstate ldx [PCB_REG + PCB_SP], %fp ldx [PCB_REG + PCB_PC], %i7 sub %fp, CCFSZ, %sp /* * Point to the pmaps of the new process, and of the last non-kernel * process to run. */ ldx [%i1 + TD_PROC], %l1 ldx [PCPU(PMAP)], %l2 ldx [%l1 + P_VMSPACE], %i5 add %i5, VM_PMAP, %l1 #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "cpu_switch: new pmap=%p old pmap=%p" , %g1, %g2, %g3, 8, 9, 10) stx %l1, [%g1 + KTR_PARM1] stx %l2, [%g1 + KTR_PARM2] 10: #endif /* * If they are the same we are done. */ cmp %l2, %l1 be,a,pn %xcc, 7f nop /* * If the new process is a kernel thread we can just leave the old * context active and avoid recycling its context number. */ SET(vmspace0, %i4, %i3) cmp %i5, %i3 be,a,pn %xcc, 7f nop /* * If there was no non-kernel pmap, don't try to deactivate it. */ brz,pn %l2, 3f lduw [PCPU(CPUMASK)], %l4 /* * Mark the pmap of the last non-kernel vmspace to run as no longer * active on this CPU. */ lduw [%l2 + PM_ACTIVE], %l3 andn %l3, %l4, %l3 stw %l3, [%l2 + PM_ACTIVE] /* * Take away its context number. */ lduw [PCPU(CPUID)], %l3 sllx %l3, INT_SHIFT, %l3 add %l2, PM_CONTEXT, %l4 mov -1, %l5 stw %l5, [%l3 + %l4] 3: cmp %i2, %g0 be,pn %xcc, 4f lduw [PCPU(TLB_CTX_MAX)], %i4 stx %i2, [%i0 + TD_LOCK] /* * Find a new TLB context. If we've run out we have to flush all * user mappings from the TLB and reset the context numbers. */ 4: lduw [PCPU(TLB_CTX)], %i3 cmp %i3, %i4 bne,a,pt %xcc, 5f nop SET(tlb_flush_user, %i5, %i4) ldx [%i4], %i5 call %i5 lduw [PCPU(TLB_CTX_MIN)], %i3 /* * Advance next free context. */ 5: add %i3, 1, %i4 stw %i4, [PCPU(TLB_CTX)] /* * Set the new context number in the pmap. */ lduw [PCPU(CPUID)], %i4 sllx %i4, INT_SHIFT, %i4 add %l1, PM_CONTEXT, %i5 stw %i3, [%i4 + %i5] /* * Mark the pmap as active on this CPU. */ lduw [%l1 + PM_ACTIVE], %i4 lduw [PCPU(CPUMASK)], %i5 or %i4, %i5, %i4 stw %i4, [%l1 + PM_ACTIVE] /* * Make note of the change in pmap. */ stx %l1, [PCPU(PMAP)] /* * Fiddle the hardware bits. Set the TSB registers and install the * new context number in the CPU. */ ldx [%l1 + PM_TSB], %i4 mov AA_DMMU_TSB, %i5 stxa %i4, [%i5] ASI_DMMU mov AA_IMMU_TSB, %i5 stxa %i4, [%i5] ASI_IMMU - setx TLB_PCXR_PGSZ_MASK, %i5, %i4 + setx TLB_CXR_PGSZ_MASK, %i5, %i4 mov AA_DMMU_PCXR, %i5 ldxa [%i5] ASI_DMMU, %l1 and %l1, %i4, %l1 or %i3, %l1, %i3 sethi %hi(KERNBASE), %i4 stxa %i3, [%i5] ASI_DMMU flush %i4 /* * Done, return and load the new process's window from the stack. */ 6: ret restore 7: cmp %i2, %g0 be,a,pn %xcc, 6b nop stx %i2, [%i0 + TD_LOCK] ret restore END(cpu_switch) ENTRY(savectx) save %sp, -CCFSZ, %sp flushw call savefpctx add %i0, PCB_UFP, %o0 stx %fp, [%i0 + PCB_SP] stx %i7, [%i0 + PCB_PC] ret restore %g0, 0, %o0 END(savectx) /* * void savefpctx(uint32_t *); */ ENTRY(savefpctx) wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi stda %f0, [%o0 + (0 * 64)] %asi stda %f16, [%o0 + (1 * 64)] %asi stda %f32, [%o0 + (2 * 64)] %asi stda %f48, [%o0 + (3 * 64)] %asi membar #Sync retl wr %g0, 0, %fprs END(savefpctx)