diff --git a/sys/sparc64/include/pmap.h b/sys/sparc64/include/pmap.h index 1a8803ad4b4d..2c8fe9f9074c 100644 --- a/sys/sparc64/include/pmap.h +++ b/sys/sparc64/include/pmap.h @@ -1,120 +1,121 @@ /* * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 * from: FreeBSD: src/sys/i386/include/pmap.h,v 1.70 2000/11/30 * $FreeBSD$ */ #ifndef _MACHINE_PMAP_H_ #define _MACHINE_PMAP_H_ #include #define DCACHE_COLOR_BITS (1) #define DCACHE_COLORS (1 << DCACHE_COLOR_BITS) #define DCACHE_COLOR_MASK (DCACHE_COLORS - 1) #define DCACHE_COLOR(va) (((va) >> PAGE_SHIFT) & DCACHE_COLOR_MASK) #define PMAP_CONTEXT_MAX 8192 #define PG_UNCACHEABLE (1<<0) #define pmap_resident_count(pm) (pm->pm_stats.resident_count) typedef struct pmap *pmap_t; struct md_page { STAILQ_HEAD(, tte) tte_list; int colors[DCACHE_COLORS]; + int color; int flags; }; struct pmap { struct tte *pm_tsb; vm_object_t pm_tsb_obj; u_int pm_active; u_int pm_context[MAXCPU]; struct pmap_statistics pm_stats; }; void pmap_bootstrap(vm_offset_t ekva); void pmap_context_rollover(void); vm_offset_t pmap_kextract(vm_offset_t va); void pmap_kenter_flags(vm_offset_t va, vm_offset_t pa, u_long flags); void pmap_kremove_flags(vm_offset_t va); void pmap_qenter_flags(vm_offset_t va, vm_page_t *m, int count, u_long fl); int pmap_cache_enter(vm_page_t m, vm_offset_t va); void pmap_cache_remove(vm_page_t m, vm_offset_t va); int pmap_remove_tte(struct pmap *pm1, struct pmap *pm2, struct tte *tp, vm_offset_t va); int pmap_protect_tte(struct pmap *pm1, struct pmap *pm2, struct tte *tp, vm_offset_t va); void pmap_map_tsb(void); void pmap_remove_all(vm_page_t m); void pmap_clear_write(vm_page_t m); #define vtophys(va) pmap_kextract(((vm_offset_t) (va))) extern vm_offset_t avail_start; extern vm_offset_t avail_end; extern struct pmap kernel_pmap_store; #define kernel_pmap (&kernel_pmap_store) extern vm_offset_t phys_avail[]; extern vm_offset_t virtual_avail; extern vm_offset_t virtual_end; extern vm_offset_t kernel_page; extern int pmap_pagedaemon_waken; extern vm_offset_t msgbuf_phys; static __inline int pmap_track_modified(pmap_t pm, vm_offset_t va) { if (pm == kernel_pmap) return ((va < kmi.clean_sva) || (va >= kmi.clean_eva)); else return (1); } #endif /* !_MACHINE_PMAP_H_ */ diff --git a/sys/sparc64/include/tlb.h b/sys/sparc64/include/tlb.h index 65e08f2f16d5..da5f532ac6bc 100644 --- a/sys/sparc64/include/tlb.h +++ b/sys/sparc64/include/tlb.h @@ -1,92 +1,98 @@ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_TLB_H_ #define _MACHINE_TLB_H_ +#define TLB_DIRECT_MASK (((1UL << (64 - 38)) - 1) << 38) +#define TLB_DIRECT_SHIFT (3) +#define TLB_DIRECT_UNCACHEABLE_SHIFT (11) +#define TLB_DIRECT_COLOR_SHIFT (10) +#define TLB_DIRECT_UNCACHEABLE (1 << TLB_DIRECT_UNCACHEABLE_SHIFT) + #define TLB_DAR_SLOT_SHIFT (3) #define TLB_DAR_SLOT(slot) ((slot) << TLB_DAR_SLOT_SHIFT) #define TAR_VPN_SHIFT (13) #define TAR_CTX_MASK ((1 << TAR_VPN_SHIFT) - 1) #define TLB_TAR_VA(va) ((va) & ~TAR_CTX_MASK) #define TLB_TAR_CTX(ctx) ((ctx) & TAR_CTX_MASK) #define TLB_DEMAP_ID_SHIFT (4) #define TLB_DEMAP_ID_PRIMARY (0) #define TLB_DEMAP_ID_SECONDARY (1) #define TLB_DEMAP_ID_NUCLEUS (2) #define TLB_DEMAP_TYPE_SHIFT (6) #define TLB_DEMAP_TYPE_PAGE (0) #define TLB_DEMAP_TYPE_CONTEXT (1) #define TLB_DEMAP_VA(va) ((va) & ~PAGE_MASK) #define TLB_DEMAP_ID(id) ((id) << TLB_DEMAP_ID_SHIFT) #define TLB_DEMAP_TYPE(type) ((type) << TLB_DEMAP_TYPE_SHIFT) #define TLB_DEMAP_PAGE (TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_PAGE)) #define TLB_DEMAP_CONTEXT (TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_CONTEXT)) #define TLB_DEMAP_PRIMARY (TLB_DEMAP_ID(TLB_DEMAP_ID_PRIMARY)) #define TLB_DEMAP_SECONDARY (TLB_DEMAP_ID(TLB_DEMAP_ID_SECONDARY)) #define TLB_DEMAP_NUCLEUS (TLB_DEMAP_ID(TLB_DEMAP_ID_NUCLEUS)) #define TLB_CTX_KERNEL (0) #define TLB_CTX_USER_MIN (1) #define TLB_CTX_USER_MAX (8192) #define MMU_SFSR_ASI_SHIFT (16) #define MMU_SFSR_FT_SHIFT (7) #define MMU_SFSR_E_SHIFT (6) #define MMU_SFSR_CT_SHIFT (4) #define MMU_SFSR_PR_SHIFT (3) #define MMU_SFSR_W_SHIFT (2) #define MMU_SFSR_OW_SHIFT (1) #define MMU_SFSR_FV_SHIFT (0) #define MMU_SFSR_ASI_SIZE (8) #define MMU_SFSR_FT_SIZE (6) #define MMU_SFSR_CT_SIZE (2) #define MMU_SFSR_W (1L << MMU_SFSR_W_SHIFT) struct tlb_entry; extern int kernel_tlb_slots; extern struct tlb_entry *kernel_tlbs; extern int tlb_slot_count; void tlb_context_demap(struct pmap *pm); void tlb_page_demap(struct pmap *pm, vm_offset_t va); void tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end); void tlb_dump(void); #endif /* !_MACHINE_TLB_H_ */ diff --git a/sys/sparc64/sparc64/exception.S b/sys/sparc64/sparc64/exception.S index 1822aa421cc5..14ed0ce8cd16 100644 --- a/sys/sparc64/sparc64/exception.S +++ b/sys/sparc64/sparc64/exception.S @@ -1,2776 +1,2824 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp */ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_ddb.h" #include #include #include #include #include #include #include #include "assym.s" .register %g2,#ignore .register %g3,#ignore .register %g6,#ignore .register %g7,#ignore /* * Atomically set the reference bit in a tte. */ #define TTE_SET_BIT(r1, r2, r3, bit) \ add r1, TTE_DATA, r1 ; \ ldx [r1], r2 ; \ 9: or r2, bit, r3 ; \ casxa [r1] ASI_N, r2, r3 ; \ cmp r2, r3 ; \ bne,pn %xcc, 9b ; \ mov r3, r2 #define TTE_SET_REF(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_REF) #define TTE_SET_W(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_W) /* * Macros for spilling and filling live windows. * * NOTE: These macros use exactly 16 instructions, and it is assumed that the * handler will not use more than 24 instructions total, to leave room for * resume vectors which occupy the last 8 instructions. */ #define SPILL(storer, base, size, asi) \ storer %l0, [base + (0 * size)] asi ; \ storer %l1, [base + (1 * size)] asi ; \ storer %l2, [base + (2 * size)] asi ; \ storer %l3, [base + (3 * size)] asi ; \ storer %l4, [base + (4 * size)] asi ; \ storer %l5, [base + (5 * size)] asi ; \ storer %l6, [base + (6 * size)] asi ; \ storer %l7, [base + (7 * size)] asi ; \ storer %i0, [base + (8 * size)] asi ; \ storer %i1, [base + (9 * size)] asi ; \ storer %i2, [base + (10 * size)] asi ; \ storer %i3, [base + (11 * size)] asi ; \ storer %i4, [base + (12 * size)] asi ; \ storer %i5, [base + (13 * size)] asi ; \ storer %i6, [base + (14 * size)] asi ; \ storer %i7, [base + (15 * size)] asi #define FILL(loader, base, size, asi) \ loader [base + (0 * size)] asi, %l0 ; \ loader [base + (1 * size)] asi, %l1 ; \ loader [base + (2 * size)] asi, %l2 ; \ loader [base + (3 * size)] asi, %l3 ; \ loader [base + (4 * size)] asi, %l4 ; \ loader [base + (5 * size)] asi, %l5 ; \ loader [base + (6 * size)] asi, %l6 ; \ loader [base + (7 * size)] asi, %l7 ; \ loader [base + (8 * size)] asi, %i0 ; \ loader [base + (9 * size)] asi, %i1 ; \ loader [base + (10 * size)] asi, %i2 ; \ loader [base + (11 * size)] asi, %i3 ; \ loader [base + (12 * size)] asi, %i4 ; \ loader [base + (13 * size)] asi, %i5 ; \ loader [base + (14 * size)] asi, %i6 ; \ loader [base + (15 * size)] asi, %i7 #define ERRATUM50(reg) mov reg, reg #define KSTACK_SLOP 1024 /* * Sanity check the kernel stack and bail out if its wrong. * XXX: doesn't handle being on the panic stack. */ #define KSTACK_CHECK \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ add %sp, SPOFF, %g1 ; \ andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ bnz,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [PCPU(CURTHREAD)], %g2 ; \ ldx [%g2 + TD_KSTACK], %g2 ; \ add %g2, KSTACK_SLOP, %g2 ; \ subcc %g1, %g2, %g1 ; \ ble,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ cmp %g1, %g2 ; \ bgt,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG ENTRY(tl1_kstack_fault) rdpr %tl, %g1 1: cmp %g1, 2 be,a 2f nop #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" , %g2, %g3, %g4, 7, 8, 9) rdpr %tl, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tnpc, %g3 stx %g3, [%g2 + KTR_PARM1] 9: #endif sub %g1, 1, %g1 wrpr %g1, 0, %tl ba,a %xcc, 1b nop 2: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" , %g1, %g2, %g3, 7, 8, 9) add %sp, SPOFF, %g2 stx %g2, [%g1 + KTR_PARM1] ldx [PCPU(CURTHREAD)], %g2 ldx [%g2 + TD_KSTACK], %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %canrestore, %g2 stx %g2, [%g1 + KTR_PARM3] rdpr %cansave, %g2 stx %g2, [%g1 + KTR_PARM4] rdpr %otherwin, %g2 stx %g2, [%g1 + KTR_PARM5] rdpr %wstate, %g2 stx %g2, [%g1 + KTR_PARM6] 9: #endif wrpr %g0, 0, %canrestore wrpr %g0, 6, %cansave wrpr %g0, 0, %otherwin wrpr %g0, WSTATE_KERNEL, %wstate sub ASP_REG, SPOFF + CCFSZ, %sp clr %fp b %xcc, tl1_trap mov T_KSTACK_FAULT | T_KERNEL, %o0 END(tl1_kstack_fault) /* * Magic to resume from a spill or fill trap. If we get an alignment or an * mmu fault during a spill or a fill, this macro will detect the fault and * resume at a set instruction offset in the trap handler. * * To check if the previous trap was a spill/fill we convert the trapped pc * to a trap type and verify that it is in the range of spill/fill vectors. * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the * tl bit allows us to detect both ranges with one test. * * This is: * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 * * To calculate the new pc we take advantage of the xor feature of wrpr. * Forcing all the low bits of the trapped pc on we can produce any offset * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. * * 0x7f ^ 0x1f == 0x60 * 0x1f == (0x80 - 0x60) - 1 * * Which are the offset and xor value used to resume from alignment faults. */ /* * Determine if we have trapped inside of a spill/fill vector, and if so resume * at a fixed instruction offset in the trap vector. Must be called on * alternate globals. */ #define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ rdpr %tpc, %g1 ; \ ERRATUM50(%g1) ; \ rdpr %tba, %g2 ; \ sub %g1, %g2, %g2 ; \ srlx %g2, 5, %g2 ; \ andn %g2, 0x200, %g2 ; \ cmp %g2, 0x80 ; \ blu,pt %xcc, 9f ; \ cmp %g2, 0x100 ; \ bgeu,pt %xcc, 9f ; \ or %g1, 0x7f, %g1 ; \ wrpr %g1, xor, %tnpc ; \ stxa_g0_sfsr ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG ; \ done ; \ 9: ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG /* * For certain faults we need to clear the sfsr mmu register before returning. */ #define RSF_CLR_SFSR \ wr %g0, ASI_DMMU, %asi ; \ stxa %g0, [%g0 + AA_DMMU_SFSR] %asi #define RSF_XOR(off) ((0x80 - off) - 1) /* * Instruction offsets in spill and fill trap handlers for handling certain * nested traps, and corresponding xor constants for wrpr. */ #define RSF_OFF_ALIGN 0x60 #define RSF_OFF_MMU 0x70 #define RESUME_SPILLFILL_ALIGN \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) #define RESUME_SPILLFILL_MMU \ RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) #define RESUME_SPILLFILL_MMU_CLR_SFSR \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) /* * Constant to add to %tnpc when taking a fill trap just before returning to * user mode. */ #define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill /* * Retry a spill or fill with a different wstate due to an alignment fault. * We may just be using the wrong stack offset. */ #define RSF_ALIGN_RETRY(ws) \ wrpr %g0, (ws), %wstate ; \ retry ; \ .align 16 /* * Generate a T_SPILL or T_FILL trap if the window operation fails. */ #define RSF_TRAP(type) \ b %xcc, tl0_sftrap ; \ mov type, %g2 ; \ .align 16 /* * Game over if the window operation fails. */ #define RSF_FATAL(type) \ b %xcc, rsf_fatal ; \ mov type, %g2 ; \ .align 16 /* * Magic to resume from a failed fill a few instructions after the corrsponding * restore. This is used on return from the kernel to usermode. */ #define RSF_FILL_MAGIC \ rdpr %tnpc, %g1 ; \ add %g1, RSF_FILL_INC, %g1 ; \ wrpr %g1, 0, %tnpc ; \ done ; \ .align 16 /* * Spill to the pcb if a spill to the user stack in kernel mode fails. */ #define RSF_SPILL_TOPCB \ b,a %xcc, tl1_spill_topcb ; \ nop ; \ .align 16 ENTRY(rsf_fatal) #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" , %g1, %g3, %g4, 7, 8, 9) rdpr %tt, %g3 stx %g3, [%g1 + KTR_PARM1] stx %g2, [%g1 + KTR_PARM2] 9: #endif KSTACK_CHECK sir END(rsf_fatal) .comm intrnames, IV_MAX * 8 .comm eintrnames, 0 .comm intrcnt, IV_MAX * 8 .comm eintrcnt, 0 /* * Trap table and associated macros * * Due to its size a trap table is an inherently hard thing to represent in * code in a clean way. There are approximately 1024 vectors, of 8 or 32 * instructions each, many of which are identical. The way that this is * layed out is the instructions (8 or 32) for the actual trap vector appear * as an AS macro. In general this code branches to tl0_trap or tl1_trap, * but if not supporting code can be placed just after the definition of the * macro. The macros are then instantiated in a different section (.trap), * which is setup to be placed by the linker at the beginning of .text, and the * code around the macros is moved to the end of trap table. In this way the * code that must be sequential in memory can be split up, and located near * its supporting code so that it is easier to follow. */ /* * Clean window traps occur when %cleanwin is zero to ensure that data * is not leaked between address spaces in registers. */ .macro clean_window clr %o0 clr %o1 clr %o2 clr %o3 clr %o4 clr %o5 clr %o6 clr %o7 clr %l0 clr %l1 clr %l2 clr %l3 clr %l4 clr %l5 clr %l6 rdpr %cleanwin, %l7 inc %l7 wrpr %l7, 0, %cleanwin clr %l7 retry .align 128 .endm /* * Stack fixups for entry from user mode. We are still running on the * user stack, and with its live registers, so we must save soon. We * are on alternate globals so we do have some registers. Set the * transitional window state, and do the save. If this traps we * we attempt to spill a window to the user stack. If this fails, * we spill the window to the pcb and continue. Spilling to the pcb * must not fail. * * NOTE: Must be called with alternate globals and clobbers %g1. */ .macro tl0_split rdpr %wstate, %g1 wrpr %g1, WSTATE_TRANSITION, %wstate save .endm .macro tl0_setup type tl0_split b %xcc, tl0_trap mov \type, %o0 .endm /* * Generic trap type. Call trap() with the specified type. */ .macro tl0_gen type tl0_setup \type .align 32 .endm /* * This is used to suck up the massive swaths of reserved trap types. * Generates count "reserved" trap vectors. */ .macro tl0_reserved count .rept \count tl0_gen T_RESERVED .endr .endm .macro tl0_fp_restore wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi ldda [PCB_REG + PCB_FPSTATE + FP_FB0] %asi, %f0 ldda [PCB_REG + PCB_FPSTATE + FP_FB1] %asi, %f16 ldda [PCB_REG + PCB_FPSTATE + FP_FB2] %asi, %f32 ldda [PCB_REG + PCB_FPSTATE + FP_FB3] %asi, %f48 membar #Sync done .align 32 .endm .macro tl0_insn_excptn wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync b %xcc, tl0_sfsr_trap mov T_INSTRUCTION_EXCEPTION, %g2 .align 32 .endm .macro tl0_data_excptn wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync b %xcc, tl0_sfsr_trap mov T_DATA_EXCEPTION, %g2 .align 32 .endm .macro tl0_align wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync b %xcc, tl0_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 .align 32 .endm ENTRY(tl0_sfsr_trap) tl0_split mov %g3, %o4 mov %g4, %o5 b %xcc, tl0_trap mov %g2, %o0 END(tl0_sfsr_trap) .macro tl0_intr level, mask tl0_split set \mask, %o1 b %xcc, tl0_intr mov \level, %o0 .align 32 .endm #define INTR(level, traplvl) \ tl ## traplvl ## _intr level, 1 << level #define TICK(traplvl) \ tl ## traplvl ## _intr PIL_TICK, 1 #define INTR_LEVEL(tl) \ INTR(1, tl) ; \ INTR(2, tl) ; \ INTR(3, tl) ; \ INTR(4, tl) ; \ INTR(5, tl) ; \ INTR(6, tl) ; \ INTR(7, tl) ; \ INTR(8, tl) ; \ INTR(9, tl) ; \ INTR(10, tl) ; \ INTR(11, tl) ; \ INTR(12, tl) ; \ INTR(13, tl) ; \ TICK(tl) ; \ INTR(15, tl) ; .macro tl0_intr_level INTR_LEVEL(0) .endm .macro intr_vector ldxa [%g0] ASI_INTR_RECEIVE, %g1 andcc %g1, IRSR_BUSY, %g0 bnz,a,pt %xcc, intr_enqueue nop sir .align 32 .endm .macro immu_miss_user /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Compute the tte bucket address. */ set (1 << TSB_BUCKET_ADDRESS_BITS) - 1, %g1 and %g1, %g3, %g1 sllx %g1, TSB_BUCKET_SHIFT + TTE_SHIFT, %g1 add %g1, TSB_REG, %g1 /* * Loop over the ttes in this bucket */ /* * Load the tte. Note that this instruction may fault, clobbering * the context of the tag access register (at least), and the contents * of %g3, %g4, %g5, and %g6. Luckily we can recover %g3, and we do * not use %g4 or %g5 until this instruction completes successfully. */ 1: ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Recover the virtual page number, which may have been clobbered. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Check that its valid and executable and that the virtual page * numbers match. */ brgez,pn %g5, 2f andcc %g5, TD_EXEC, %g0 bz,pn %xcc, 2f cmp %g3, %g4 bne,pn %xcc, 2f EMPTY /* * We matched a tte, load the tlb. */ /* * Set the reference bit, if it's currently clear. */ andcc %g5, TD_REF, %g0 bz,a,pn %xcc, tl0_immu_miss_set_ref nop /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_IMMU_TAR] %asi stxa %g5, [%g0] ASI_ITLB_DATA_IN_REG retry /* * Check the low bits to see if we've finished the bucket. */ 2: add %g1, 1 << TTE_SHIFT, %g1 andcc %g1, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,a,pt %xcc, 1b nop .endm .macro tl0_immu_miss /* * Force kernel store order. */ wrpr %g0, PSTATE_MMU, %pstate /* * Load the virtual page number and context from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 immu_miss_user b,a %xcc, tl0_immu_miss_trap nop .align 128 .endm ENTRY(tl0_immu_miss_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g1, %g4, %g5) /* * May have become invalid, in which case start over. */ brgez,pn %g4, 1f or %g4, TD_REF, %g4 /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_IMMU_TAR] %asi stxa %g4, [%g0] ASI_ITLB_DATA_IN_REG 1: retry END(tl0_immu_miss_set_ref) ENTRY(tl0_immu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_IMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Reload the tag access register. */ ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 /* * Save the tag access register, and call common trap code. */ tl0_split mov %g2, %o3 b %xcc, tl0_trap mov T_INSTRUCTION_MISS, %o0 END(tl0_immu_miss_trap) .macro dmmu_miss_user /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Compute the tte bucket address. */ set (1 << TSB_BUCKET_ADDRESS_BITS) - 1, %g1 and %g1, %g3, %g1 sllx %g1, TSB_BUCKET_SHIFT + TTE_SHIFT, %g1 add %g1, TSB_REG, %g1 /* * Loop over the ttes in this bucket */ /* * Load the tte. Note that this instruction may fault, clobbering * the contents of the tag access register (at least), and the contents * of %g3, %g4, %g5, and %g6. Luckily we can recover %g3, and we do * not use %g4 or %g5 until this instruction completes successfully. */ 1: ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Recover the virtual page number, which may have been clobbered. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Check that its valid and that the virtual page numbers match. */ brgez,pn %g5, 2f cmp %g3, %g4 bne,pn %xcc, 2f EMPTY /* * We matched a tte, load the tlb. */ /* * Set the reference bit, if it's currently clear. */ andcc %g5, TD_REF, %g0 bz,a,pn %xcc, dmmu_miss_user_set_ref nop /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG retry /* * Check the low bits to see if we've finished the bucket. */ 2: add %g1, 1 << TTE_SHIFT, %g1 andcc %g1, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,a,pt %xcc, 1b nop .endm ENTRY(dmmu_miss_user_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g1, %g4, %g5) /* * May have become invalid, in which case start over. */ brgez,pn %g4, 1f or %g4, TD_REF, %g4 /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi stxa %g4, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(dmmu_miss_user_set_ref) .macro tl0_dmmu_miss /* * Force kernel store order. */ wrpr %g0, PSTATE_MMU, %pstate /* * Load the virtual page number and context from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Try a fast inline lookup of the primary tsb. */ dmmu_miss_user /* * Not in primary tsb, call c code. Not much else fits inline. */ b,a %xcc, tl0_dmmu_miss_trap nop .align 128 .endm ENTRY(tl0_dmmu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Save the tag access register and call common trap code. */ tl0_split mov %g2, %o3 b %xcc, tl0_trap mov T_DATA_MISS, %o0 END(tl0_dmmu_miss_trap) .macro dmmu_prot_user /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Compute the tte bucket address. */ set (1 << TSB_BUCKET_ADDRESS_BITS) - 1, %g1 and %g1, %g3, %g1 sllx %g1, TSB_BUCKET_SHIFT + TTE_SHIFT, %g1 add %g1, TSB_REG, %g1 /* * Loop over the ttes in this bucket */ /* * Load the tte. Note that this instruction may fault, clobbering * the context of the tag access register (at least), and the contents * of %g3, %g4, %g5, and %g6. Luckily we can recover %g3, and we do * not use %g4 or %g5 until this instruction completes successfully. */ 1: ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Recover the virtual page number, which may have been clobbered. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Check that its valid and writable and that the virtual page * numbers match. */ brgez,pn %g5, 2f andcc %g5, TD_SW, %g0 bz,pn %xcc, 2f cmp %g3, %g4 bne,pn %xcc, 2f nop /* * Set the hardware write bit. */ b,a %xcc, dmmu_prot_set_w nop /* * Check the low bits to see if we've finished the bucket. */ 2: add %g1, 1 << TTE_SHIFT, %g1 andcc %g1, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,a,pt %xcc, 1b nop .endm .macro tl0_dmmu_prot /* * Force kernel store order. */ wrpr %g0, PSTATE_MMU, %pstate /* * Load the virtual page number and context from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Try a fast inline lookup of the tsb. */ dmmu_prot_user /* * Not in tsb. Call c code. */ b,a %xcc, tl0_dmmu_prot_trap nop .align 128 .endm ENTRY(dmmu_prot_set_w) /* * Set the hardware write bit in the tte. */ TTE_SET_W(%g1, %g4, %g5) /* * Delete the old TLB entry and clear the sfsr. */ sllx %g3, TAR_VPN_SHIFT, %g3 stxa %g0, [%g3] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * May have become invalid in which case start over. */ brgez,pn %g4, 1f or %g4, TD_W, %g4 /* * Load the tte data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi stxa %g4, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(dmmu_prot_set_w) ENTRY(tl0_dmmu_prot_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Load the tar, sfar and sfsr. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * Save the mmu registers and call common trap code. */ tl0_split mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 b %xcc, tl0_trap mov T_DATA_PROTECTION, %o0 END(tl0_dmmu_prot_trap) .macro tl0_spill_0_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_spill_1_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_fill_0_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm .macro tl0_fill_1_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm ENTRY(tl0_sftrap) rdpr %tstate, %g1 and %g1, TSTATE_CWP_MASK, %g1 wrpr %g1, 0, %cwp tl0_split b %xcc, tl0_trap mov %g2, %o0 END(tl0_sftrap) .macro tl0_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_syscall tl0_split b %xcc, tl0_syscall mov T_SYSCALL, %o0 .align 32 .endm .macro tl0_soft count .rept \count tl0_gen T_SOFT .endr .endm .macro tl1_split rdpr %wstate, %g1 wrpr %g1, WSTATE_NESTED, %wstate save %sp, -CCFSZ, %sp .endm .macro tl1_setup type tl1_split b %xcc, tl1_trap mov \type | T_KERNEL, %o0 .endm .macro tl1_gen type tl1_setup \type .align 32 .endm .macro tl1_reserved count .rept \count tl1_gen T_RESERVED .endr .endm .macro tl1_insn_excptn wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync b %xcc, tl1_insn_exceptn_trap mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 .align 32 .endm ENTRY(tl1_insn_exceptn_trap) tl1_split mov %g3, %o4 mov %g4, %o5 b %xcc, tl1_trap mov %g2, %o0 END(tl1_insn_exceptn_trap) .macro tl1_data_excptn b,a %xcc, tl1_data_excptn_trap nop .align 32 .endm ENTRY(tl1_data_excptn_trap) wrpr %g0, PSTATE_ALT, %pstate RESUME_SPILLFILL_MMU_CLR_SFSR b %xcc, tl1_sfsr_trap mov T_DATA_EXCEPTION | T_KERNEL, %g2 END(tl1_data_excptn_trap) .macro tl1_align b,a %xcc, tl1_align_trap nop .align 32 .endm ENTRY(tl1_align_trap) wrpr %g0, PSTATE_ALT, %pstate RESUME_SPILLFILL_ALIGN b %xcc, tl1_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 END(tl1_data_excptn_trap) ENTRY(tl1_sfsr_trap) wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split mov %g3, %o4 mov %g4, %o5 b %xcc, tl1_trap mov %g2, %o0 END(tl1_sfsr_trap) .macro tl1_intr level, mask tl1_split set \mask, %o1 b %xcc, tl1_intr mov \level, %o0 .align 32 .endm .macro tl1_intr_level INTR_LEVEL(1) .endm ENTRY(intr_dequeue) save %sp, -CCFSZ, %sp 1: ldx [PCPU(IRHEAD)], %l0 brnz,a,pt %l0, 2f nop ret restore 2: wrpr %g0, PSTATE_NORMAL, %pstate ldx [%l0 + IR_NEXT], %l1 brnz,pt %l1, 3f stx %l1, [PCPU(IRHEAD)] PCPU_ADDR(IRHEAD, %l1) stx %l1, [PCPU(IRTAIL)] 3: ldx [%l0 + IR_FUNC], %o0 ldx [%l0 + IR_ARG], %o1 ldx [%l0 + IR_VEC], %o2 ldx [PCPU(IRFREE)], %l1 stx %l1, [%l0 + IR_NEXT] stx %l0, [PCPU(IRFREE)] wrpr %g0, PSTATE_KERNEL, %pstate call %o0 mov %o1, %o0 ba,a %xcc, 1b nop END(intr_dequeue) /* * Handle a vectored interrupt. * * This is either a data bearing mondo vector interrupt, or a cross trap * request from another cpu. In either case the hardware supplies an * interrupt packet, in the form of 3 data words which are read from internal * registers. A data bearing mondo vector packet consists of an interrupt * number in the first data word, and zero in 2nd and 3rd. We use the * interrupt number to find the function, argument and priority from the * intr_vector table, allocate and fill in an intr_request from the per-cpu * free list, link it onto the per-cpu active list and finally post a softint * at the desired priority. Cross trap requests come in 2 forms, direct * and queued. Direct requests are distinguished by the first data word * being zero. The 2nd data word carries a function to call and the 3rd * an argument to pass. The function is jumped to directly. It executes * in nucleus context on interrupt globals and with all interrupts disabled, * therefore it must be fast, and the things that it can do are limited. * Queued cross trap requests are handled much like mondo vectors, except * that the function, argument and priority are contained in the interrupt * packet itself. They are distinguished by the upper 4 bits of the data * word being non-zero, which specifies the priority of the softint to * deliver. * * Register usage: * %g1 - pointer to intr_request * %g2 - pointer to intr_vector, temp once required data is loaded * %g3 - interrupt number for mondo vectors, unused otherwise * %g4 - function, from the interrupt packet for cross traps, or * loaded from the interrupt registers for mondo vecors * %g5 - argument, as above for %g4 * %g6 - softint priority */ ENTRY(intr_enqueue) /* * Load the interrupt packet from the hardware. */ wr %g0, ASI_SDB_INTR_R, %asi ldxa [%g0 + AA_SDB_INTR_D0] %asi, %g3 ldxa [%g0 + AA_SDB_INTR_D1] %asi, %g4 ldxa [%g0 + AA_SDB_INTR_D2] %asi, %g5 stxa %g0, [%g0] ASI_INTR_RECEIVE membar #Sync #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: data=%#lx %#lx %#lx" , %g1, %g2, %g6, 7, 8, 9) stx %g3, [%g1 + KTR_PARM1] stx %g4, [%g1 + KTR_PARM2] stx %g5, [%g1 + KTR_PARM3] 9: #endif /* * If the first data word is zero this is a direct cross trap request. * The 2nd word points to code to execute and the 3rd is an argument * to pass. Jump to it. */ brnz,a,pt %g3, 1f nop #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: direct ipi func=%#lx arg=%#lx" , %g1, %g2, %g6, 7, 8, 9) stx %g4, [%g1 + KTR_PARM1] stx %g5, [%g1 + KTR_PARM2] 9: #endif jmpl %g4, %g0 nop /* NOTREACHED */ /* * If the high 4 bits of the 1st data word are non-zero, this is a * queued cross trap request to be delivered as a softint. The high * 4 bits of the 1st data word specify a priority, and the 2nd and * 3rd a function and argument. */ 1: srlx %g3, 60, %g6 brnz,a,pn %g6, 2f clr %g3 /* * Find the function, argument and desired priority from the * intr_vector table. */ SET(intr_vectors, %g4, %g2) sllx %g3, IV_SHIFT, %g4 add %g2, %g4, %g2 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: mondo vector func=%#lx arg=%#lx pri=%#lx" , %g4, %g5, %g6, 7, 8, 9) ldx [%g2 + IV_FUNC], %g5 stx %g5, [%g4 + KTR_PARM1] ldx [%g2 + IV_ARG], %g5 stx %g5, [%g4 + KTR_PARM2] ldx [%g2 + IV_PRI], %g5 stx %g5, [%g4 + KTR_PARM3] 9: #endif ldx [%g2 + IV_FUNC], %g4 ldx [%g2 + IV_ARG], %g5 lduw [%g2 + IV_PRI], %g6 ba,a %xcc, 3f nop /* * Get a intr_request from the free list. There should always be one * unless we are getting an interrupt storm from stray interrupts, in * which case the we will deference a NULL pointer and panic. */ 2: #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: queued ipi func=%#lx arg=%#lx pri=%#lx" , %g1, %g2, %g3, 7, 8, 9) stx %g4, [%g1 + KTR_PARM1] stx %g5, [%g1 + KTR_PARM2] stx %g6, [%g1 + KTR_PARM3] 9: clr %g3 #endif 3: ldx [PCPU(IRFREE)], %g1 ldx [%g1 + IR_NEXT], %g2 stx %g2, [PCPU(IRFREE)] /* * Store the vector number, function, argument and priority. */ stw %g3, [%g1 + IR_VEC] stx %g4, [%g1 + IR_FUNC] stx %g5, [%g1 + IR_ARG] stw %g6, [%g1 + IR_PRI] /* * Link it onto the end of the active list. */ stx %g0, [%g1 + IR_NEXT] ldx [PCPU(IRTAIL)], %g4 stx %g1, [%g4] add %g1, IR_NEXT, %g1 stx %g1, [PCPU(IRTAIL)] /* * Trigger a softint at the level indicated by the priority. */ mov 1, %g1 sllx %g1, %g6, %g1 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: softint pil=%#lx pri=%#lx mask=%#lx" , %g2, %g3, %g4, 7, 8, 9) rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM1] stx %g6, [%g2 + KTR_PARM2] stx %g1, [%g2 + KTR_PARM3] 9: #endif wr %g1, 0, %asr20 /* * Done, retry the instruction. */ retry END(intr_enqueue) .macro tl1_immu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g6 /* * Extract the virtual page number from the contents of the tag access * register. */ srlx %g6, TAR_VPN_SHIFT, %g6 /* * Find the index into the kernel tsb. */ set TSB_KERNEL_MASK, %g4 and %g6, %g4, %g3 /* * Compute the tte address. */ ldxa [%g0 + AA_IMMU_TSB] %asi, %g4 sllx %g3, TTE_SHIFT, %g3 add %g3, %g4, %g3 /* * Load the tte. */ ldda [%g3] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Check that its valid and executable and that the virtual page * numbers match. */ brgez,pn %g5, tl1_immu_miss_trap andcc %g5, TD_EXEC, %g0 bz,pn %xcc, tl1_immu_miss_trap cmp %g4, %g6 bne,pn %xcc, tl1_immu_miss_trap EMPTY /* * Set the reference bit if its currently clear. */ andcc %g5, TD_REF, %g0 bnz,a,pt %xcc, 1f nop TTE_SET_REF(%g3, %g5, %g4) /* * May have become invalid, in which case start over. */ brgez,pn %g5, 2f or %g5, TD_REF, %g5 /* * Load the tte data into the TLB and retry the instruction. */ 1: stxa %g5, [%g0] ASI_ITLB_DATA_IN_REG 2: retry .align 128 .endm ENTRY(tl1_immu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 tl1_split mov %g2, %o3 b %xcc, tl1_trap mov T_INSTRUCTION_MISS | T_KERNEL, %o0 END(tl1_immu_miss_trap) .macro tl1_dmmu_miss /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g6 /* * Extract the context from the contents of the tag access register. - * If its non-zero this is a fault on a user address, otherwise get - * the virtual page number. + * If its non-zero this is a fault on a user address. Note that the + * faulting address is passed in %g2. */ sllx %g6, 64 - TAR_VPN_SHIFT, %g5 brnz,a,pn %g5, tl1_dmmu_miss_user mov %g6, %g2 + /* + * Check for the direct mapped physical region. These addresses have + * the high bit set so they are negative. + */ + brlz,pn %g6, tl1_dmmu_miss_direct + EMPTY + /* * Find the index into the kernel tsb. */ - set TSB_KERNEL_MASK, %g4 + set TSB_KERNEL_MASK, %g4 srlx %g6, TAR_VPN_SHIFT, %g6 and %g6, %g4, %g3 /* * Compute the tte address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g4 sllx %g3, TTE_SHIFT, %g3 add %g3, %g4, %g3 /* * Load the tte. */ ldda [%g3] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Check that its valid and that the virtual page numbers match. */ brgez,pn %g5, tl1_dmmu_miss_trap cmp %g4, %g6 bne,pn %xcc, tl1_dmmu_miss_trap EMPTY /* * Set the reference bit if its currently clear. */ andcc %g5, TD_REF, %g0 bnz,a,pt %xcc, 1f nop TTE_SET_REF(%g3, %g5, %g4) /* * May have become invalid, in which case start over. */ brgez,pn %g5, 2f or %g5, TD_REF, %g5 /* * Load the tte data into the TLB and retry the instruction. */ 1: stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG 2: retry .align 128 .endm ENTRY(tl1_dmmu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate KSTACK_CHECK ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 tl1_split mov %g2, %o3 b %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl1_dmmu_miss_trap) +ENTRY(tl1_dmmu_miss_direct) +#if KTR_COMPILE & KTR_TRAP + CATR(KTR_TRAP, "tl1_dmmu_miss_direct: pc=%#lx sp=%#lx tar=%#lx" + , %g1, %g2, %g3, 7, 8, 9) + rdpr %tpc, %g2 + stx %g2, [%g1 + KTR_PARM1] + add %sp, SPOFF, %g2 + stx %g2, [%g1 + KTR_PARM2] + stx %g6, [%g1 + KTR_PARM3] +9: +#endif + + /* + * Check the cache bits in the virtual address to see if this mapping + * is virtually cacheable. We set this up so that the masks fit in + * immediates... Note that the arithmetic shift sign extends, keeping + * all the top bits set. + */ + srax %g6, TLB_DIRECT_SHIFT, %g6 + andcc %g6, TLB_DIRECT_UNCACHEABLE, %g0 + mov TD_CP | TD_CV | TD_W, %g1 + movnz %xcc, TD_CP | TD_W, %g1 + + /* + * Mask off the high bits of the virtual address to get the physical + * address, and or in the tte bits. The high bit is left set in the + * physical address, which corresponds to the tte valid bit, so that + * we don't have to include it in the tte bits. We ignore the cache + * bits, since they get shifted into the soft tte bits anyway. + */ + setx TLB_DIRECT_MASK & ~TD_V, %g3, %g2 + andn %g6, %g2, %g3 + or %g3, %g1, %g3 + + /* + * Load the tte data into the TLB and retry the instruction. + */ + stxa %g3, [%g0] ASI_DTLB_DATA_IN_REG + retry +END(tl1_dmmu_miss_direct) + ENTRY(tl1_dmmu_miss_user) /* * Try a fast inline lookup of the user tsb. */ dmmu_miss_user /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Handle faults during window spill/fill. */ RESUME_SPILLFILL_MMU /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 tl1_split mov %g2, %o3 b %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl1_dmmu_miss_user) .macro tl1_dmmu_prot /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g6 /* * Extract the context from the contents of the tag access register. * If its non-zero this is a fault on a user address, otherwise get * the virtual page number. */ sllx %g6, 64 - TAR_VPN_SHIFT, %g5 brnz,a,pn %g5, tl1_dmmu_prot_user mov %g6, %g2 /* * Find the index into the kernel tsb. */ set TSB_KERNEL_MASK, %g4 srlx %g6, TAR_VPN_SHIFT, %g6 and %g6, %g4, %g5 /* * Compute the tte address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g4 sllx %g5, TTE_SHIFT, %g5 add %g4, %g5, %g3 /* * Load the tte. */ ldda [%g3] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Check that its valid and writeable and that the virtual page * numbers match. */ brgez,pn %g5, tl1_dmmu_prot_trap andcc %g5, TD_SW, %g0 bz,pn %xcc, tl1_dmmu_prot_trap cmp %g4, %g6 bne,pn %xcc, tl1_dmmu_prot_trap EMPTY /* * Delete the old TLB entry and clear the sfsr. */ sllx %g6, TAR_VPN_SHIFT, %g6 or %g6, TLB_DEMAP_NUCLEUS, %g6 stxa %g0, [%g6] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync ba,a %xcc, tl1_dmmu_prot_cont nop .align 128 .endm ENTRY(tl1_dmmu_prot_cont) /* * Set the hardware write bit. */ TTE_SET_W(%g3, %g5, %g6) /* * Load the tte data into the TLB and retry the instruction. */ or %g5, TD_W, %g5 stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG retry END(tl1_dmmu_prot_cont) ENTRY(tl1_dmmu_prot_user) /* * Try a fast inline lookup of the user tsb. */ dmmu_prot_user /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* Handle faults during window spill/fill. */ RESUME_SPILLFILL_MMU_CLR_SFSR b,a %xcc, tl1_dmmu_prot_trap nop END(tl1_dmmu_prot_user) ENTRY(tl1_dmmu_prot_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Load the sfar, sfsr and tar. Clear the sfsr. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 b %xcc, tl1_trap mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl1_dmmu_prot_trap) .macro tl1_spill_0_n SPILL(stx, %sp + SPOFF, 8, EMPTY) saved retry .align 32 RSF_FATAL(T_SPILL) RSF_FATAL(T_SPILL) .endm .macro tl1_spill_2_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_3_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_0_o wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_1_o wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_2_o RSF_SPILL_TOPCB .align 128 .endm .macro tl1_fill_0_n FILL(ldx, %sp + SPOFF, 8, EMPTY) restored retry .align 32 RSF_FATAL(T_FILL) RSF_FATAL(T_FILL) .endm .macro tl1_fill_2_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm .macro tl1_fill_3_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm /* * This is used to spill windows that are still occupied with user * data on kernel entry to the pcb. */ ENTRY(tl1_spill_topcb) wrpr %g0, PSTATE_ALT, %pstate /* Free some globals for our use. */ dec 24, ASP_REG stx %g1, [ASP_REG + 0] stx %g2, [ASP_REG + 8] stx %g3, [ASP_REG + 16] ldx [PCB_REG + PCB_NSAVED], %g1 sllx %g1, PTR_SHIFT, %g2 add %g2, PCB_REG, %g2 stx %sp, [%g2 + PCB_RWSP] sllx %g1, RW_SHIFT, %g2 add %g2, PCB_REG, %g2 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) inc %g1 stx %g1, [PCB_REG + PCB_NSAVED] #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" , %g1, %g2, %g3, 7, 8, 9) rdpr %tpc, %g2 stx %g2, [%g1 + KTR_PARM1] rdpr %tnpc, %g2 stx %g2, [%g1 + KTR_PARM2] stx %sp, [%g1 + KTR_PARM3] ldx [PCB_REG + PCB_NSAVED], %g2 stx %g2, [%g1 + KTR_PARM4] 9: #endif saved ldx [ASP_REG + 16], %g3 ldx [ASP_REG + 8], %g2 ldx [ASP_REG + 0], %g1 inc 24, ASP_REG retry END(tl1_spill_topcb) .macro tl1_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_soft count .rept \count tl1_gen T_SOFT | T_KERNEL .endr .endm .sect .trap .align 0x8000 .globl tl0_base tl0_base: tl0_reserved 8 ! 0x0-0x7 tl0_insn_excptn: tl0_insn_excptn ! 0x8 tl0_reserved 1 ! 0x9 tl0_insn_error: tl0_gen T_INSTRUCTION_ERROR ! 0xa tl0_reserved 5 ! 0xb-0xf tl0_insn_illegal: tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 tl0_priv_opcode: tl0_gen T_PRIVILEGED_OPCODE ! 0x11 tl0_reserved 14 ! 0x12-0x1f tl0_fp_disabled: tl0_gen T_FP_DISABLED ! 0x20 tl0_fp_ieee: tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 tl0_fp_other: tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 tl0_tag_ovflw: tl0_gen T_TAG_OFERFLOW ! 0x23 tl0_clean_window: clean_window ! 0x24 tl0_divide: tl0_gen T_DIVISION_BY_ZERO ! 0x28 tl0_reserved 7 ! 0x29-0x2f tl0_data_excptn: tl0_data_excptn ! 0x30 tl0_reserved 1 ! 0x31 tl0_data_error: tl0_gen T_DATA_ERROR ! 0x32 tl0_reserved 1 ! 0x33 tl0_align: tl0_align ! 0x34 tl0_align_lddf: tl0_gen T_RESERVED ! 0x35 tl0_align_stdf: tl0_gen T_RESERVED ! 0x36 tl0_priv_action: tl0_gen T_PRIVILEGED_ACTION ! 0x37 tl0_reserved 9 ! 0x38-0x40 tl0_intr_level: tl0_intr_level ! 0x41-0x4f tl0_reserved 16 ! 0x50-0x5f tl0_intr_vector: intr_vector ! 0x60 tl0_watch_phys: tl0_gen T_PA_WATCHPOINT ! 0x61 tl0_watch_virt: tl0_gen T_VA_WATCHPOINT ! 0x62 tl0_ecc: tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 tl0_immu_miss: tl0_immu_miss ! 0x64 tl0_dmmu_miss: tl0_dmmu_miss ! 0x68 tl0_dmmu_prot: tl0_dmmu_prot ! 0x6c tl0_reserved 16 ! 0x70-0x7f tl0_spill_0_n: tl0_spill_0_n ! 0x80 tl0_spill_1_n: tl0_spill_1_n ! 0x84 tl0_spill_bad 14 ! 0x88-0xbf tl0_fill_0_n: tl0_fill_0_n ! 0xc0 tl0_fill_1_n: tl0_fill_1_n ! 0xc4 tl0_fill_bad 14 ! 0xc8-0xff tl0_soft: tl0_reserved 1 ! 0x100 tl0_gen T_BREAKPOINT ! 0x101 tl0_gen T_DIVISION_BY_ZERO ! 0x102 tl0_reserved 1 ! 0x103 tl0_gen T_CLEAN_WINDOW ! 0x104 tl0_gen T_RANGE_CHECK ! 0x105 tl0_gen T_FIX_ALIGNMENT ! 0x106 tl0_gen T_INTEGER_OVERFLOW ! 0x107 tl0_reserved 1 ! 0x108 tl0_syscall ! 0x109 tl0_fp_restore ! 0x10a tl0_reserved 5 ! 0x10b-0x10f tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f tl0_reserved 224 ! 0x120-0x1ff tl1_base: tl1_reserved 8 ! 0x200-0x207 tl1_insn_excptn: tl1_insn_excptn ! 0x208 tl1_reserved 1 ! 0x209 tl1_insn_error: tl1_gen T_INSTRUCTION_ERROR ! 0x20a tl1_reserved 5 ! 0x20b-0x20f tl1_insn_illegal: tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 tl1_priv_opcode: tl1_gen T_PRIVILEGED_OPCODE ! 0x211 tl1_reserved 14 ! 0x212-0x21f tl1_fp_disabled: tl1_gen T_FP_DISABLED ! 0x220 tl1_fp_ieee: tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 tl1_fp_other: tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 tl1_tag_ovflw: tl1_gen T_TAG_OFERFLOW ! 0x223 tl1_clean_window: clean_window ! 0x224 tl1_divide: tl1_gen T_DIVISION_BY_ZERO ! 0x228 tl1_reserved 7 ! 0x229-0x22f tl1_data_excptn: tl1_data_excptn ! 0x230 tl1_reserved 1 ! 0x231 tl1_data_error: tl1_gen T_DATA_ERROR ! 0x232 tl1_reserved 1 ! 0x233 tl1_align: tl1_align ! 0x234 tl1_align_lddf: tl1_gen T_RESERVED ! 0x235 tl1_align_stdf: tl1_gen T_RESERVED ! 0x236 tl1_priv_action: tl1_gen T_PRIVILEGED_ACTION ! 0x237 tl1_reserved 9 ! 0x238-0x240 tl1_intr_level: tl1_intr_level ! 0x241-0x24f tl1_reserved 16 ! 0x250-0x25f tl1_intr_vector: intr_vector ! 0x260 tl1_watch_phys: tl1_gen T_PA_WATCHPOINT ! 0x261 tl1_watch_virt: tl1_gen T_VA_WATCHPOINT ! 0x262 tl1_ecc: tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 tl1_immu_miss: tl1_immu_miss ! 0x264 tl1_dmmu_miss: tl1_dmmu_miss ! 0x268 tl1_dmmu_prot: tl1_dmmu_prot ! 0x26c tl1_reserved 16 ! 0x270-0x27f tl1_spill_0_n: tl1_spill_0_n ! 0x280 tl1_spill_bad 1 ! 0x284 tl1_spill_2_n: tl1_spill_2_n ! 0x288 tl1_spill_3_n: tl1_spill_3_n ! 0x29c tl1_spill_bad 4 ! 0x290-0x29f tl1_spill_0_o: tl1_spill_0_o ! 0x2a0 tl1_spill_1_o: tl1_spill_1_o ! 0x2a4 tl1_spill_2_o: tl1_spill_2_o ! 0x2a8 tl1_spill_bad 5 ! 0x2ac-0x2bf tl1_fill_0_n: tl1_fill_0_n ! 0x2c0 tl1_fill_bad 1 ! 0x2c4 tl1_fill_2_n: tl1_fill_2_n ! 0x2d0 tl1_fill_3_n: tl1_fill_3_n ! 0x2d4 tl1_fill_bad 12 ! 0x2d8-0x2ff tl1_reserved 1 ! 0x300 tl1_breakpoint: tl1_gen T_BREAKPOINT ! 0x301 tl1_gen T_RSTRWP_PHYS ! 0x302 tl1_gen T_RSTRWP_VIRT ! 0x303 tl1_reserved 252 ! 0x304-0x3ff /* * User trap entry point. * * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_int sfsr) * * The following setup has been performed: * - the windows have been split and the active user window has been saved * (maybe just to the pcb) * - we are on alternate globals and interrupts are disabled * * We switch to the kernel stack, build a trapframe, switch to normal * globals, enable interrupts and call trap. * * NOTE: We must be very careful setting up the per-cpu pointer. We know that * it has been pre-set in alternate globals, so we read it from there and setup * the normal %g7 *before* enabling interrupts. This avoids any possibility * of cpu migration and using the wrong pcpup. */ ENTRY(tl0_trap) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif and %l5, WSTATE_NORMAL_MASK, %l5 cmp %o0, UT_MAX bge,a,pt %xcc, 2f nop ldx [PCPU(CURTHREAD)], %l6 ldx [%l6 + TD_PROC], %l6 ldx [%l6 + P_MD + MD_UTRAP], %l6 brz,pt %l6, 2f sllx %o0, PTR_SHIFT, %l7 ldx [%l6 + %l7], %l6 brz,pt %l6, 2f andn %l0, TSTATE_CWP_MASK, %l7 ldx [PCB_REG + PCB_NSAVED], %g1 brnz,a,pn %g1, 1f mov T_SPILL, %o0 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: user trap npc=%#lx" , %g1, %g2, %g3, 7, 8, 9) stx %l6, [%g1 + KTR_PARM1] 9: #endif wrpr %l5, %wstate wrpr %l6, %tnpc rdpr %cwp, %l6 wrpr %l6, %l7, %tstate sub %fp, CCFSZ, %sp /* * Need to store %fsr to pass it to the user trap handler. Otherwise, * the ftt field might be zeoed out in a store in another trap or * interrupt. Use the temporary stack for that. */ rd %fprs, %l3 or %l3, FPRS_FEF, %l4 wr %l4, 0, %fprs dec 8, ASP_REG stx %fsr, [ASP_REG] ldx [ASP_REG], %l4 inc 8, ASP_REG wr %l3, 0, %fprs mov %l0, %l5 mov %l1, %l6 mov %l2, %l7 done 1: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: defer user trap npc=%#lx nsaved=%#lx" , %g1, %g2, %g3, 7, 8, 9) stx %l6, [%g1 + KTR_PARM1] ldx [PCB_REG + PCB_NSAVED], %g2 stx %g2, [%g1 + KTR_PARM2] 9: #endif 2: sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] wr %g0, 0, %fprs mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] .Ltl0_trap_reenter: stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] call trap add %sp, CCFSZ + SPOFF, %o0 b,a %xcc, tl0_ret nop END(tl0_trap) /* * void tl0_syscall(u_int type) */ ENTRY(tl0_syscall) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_SYSC CATR(KTR_SYSC, "tl0_syscall: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] wr %g0, 0, %fprs mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] call syscall add %sp, CCFSZ + SPOFF, %o0 b,a %xcc, tl0_ret nop END(tl0_syscall) /* * void tl0_intr(u_int level, u_int mask) */ ENTRY(tl0_intr) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %asr21 and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] wr %g0, 0, %fprs mov %o0, %l3 mov T_INTERRUPT, %o1 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] call critical_enter nop SET(cnt+V_INTR, %l1, %l0) ATOMIC_INC_INT(%l0, %l1, %l2) SET(intr_handlers, %l1, %l0) sllx %l3, IH_SHIFT, %l1 ldx [%l0 + %l1], %l1 KASSERT(%l1, "tl0_intr: ih null") call %l1 add %sp, CCFSZ + SPOFF, %o0 call critical_exit nop b,a %xcc, tl0_ret nop END(tl0_intr) ENTRY(tl0_ret) #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: check ast td=%p (%s) pil=%#lx sflag=%#x" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g2 + TD_PROC], %g2 add %g2, P_COMM, %g3 stx %g3, [%g1 + KTR_PARM2] rdpr %pil, %g3 stx %g3, [%g1 + KTR_PARM3] lduw [%g2 + P_SFLAG], %g3 stx %g3, [%g1 + KTR_PARM4] 9: #endif /* * Check for pending asts atomically with returning. We must raise * the pil before checking, and if no asts are found the pil must * remain raised until the retry is executed, or we risk missing asts * caused by interrupts occuring after the test. If the pil is lowered, * as it is when we call ast, the check must be re-executed. */ 1: wrpr %g0, PIL_TICK, %pil ldx [PCPU(CURTHREAD)], %l0 ldx [%l0 + TD_KSE], %l1 lduw [%l1 + KE_FLAGS], %l2 and %l2, KEF_ASTPENDING | KEF_NEEDRESCHED, %l2 brz,a,pt %l2, 2f nop wrpr %g0, 0, %pil call ast add %sp, CCFSZ + SPOFF, %o0 ba,a %xcc, 1b nop /* * Check for windows that were spilled to the pcb and need to be * copied out. This must be the last thing that is done before the * return to usermode. If there are still user windows in the cpu * and we call a nested function after this, which causes them to be * spilled to the pcb, they will not be copied out and the stack will * be inconsistent. */ 2: ldx [PCB_REG + PCB_NSAVED], %l1 mov T_SPILL, %o0 brnz,a,pn %l1, .Ltl0_trap_reenter wrpr %g0, 0, %pil ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l3 ldub [%sp + SPOFF + CCFSZ + TF_FPRS], %l4 ldub [%sp + SPOFF + CCFSZ + TF_WSTATE], %l5 wrpr %g0, PSTATE_NORMAL, %pstate ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 wrpr %g0, PSTATE_ALT, %pstate wrpr %g0, 0, %pil wrpr %l1, 0, %tpc wrpr %l2, 0, %tnpc wr %l3, 0, %y andn %l0, TSTATE_CWP_MASK, %g1 mov %l4, %g2 srlx %l5, WSTATE_OTHER_SHIFT, %g3 wrpr %g3, WSTATE_TRANSITION, %wstate rdpr %otherwin, %o0 wrpr %o0, 0, %canrestore wrpr %g0, 0, %otherwin wrpr %o0, 0, %cleanwin /* * If this instruction causes a fill trap which fails to fill a window * from the user stack, we will resume at tl0_ret_fill_end and call * back into the kernel. */ restore tl0_ret_fill: rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wr %g2, 0, %fprs wrpr %g3, 0, %wstate #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tnpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry tl0_ret_fill_end: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" , %l0, %l1, %l2, 7, 8, 9) rdpr %pstate, %l1 stx %l1, [%l0 + KTR_PARM1] stx %l5, [%l0 + KTR_PARM2] stx %sp, [%l0 + KTR_PARM3] 9: #endif /* * The fill failed and magic has been performed. Call trap again, * which will copyin the window on the user's behalf. */ wrpr %l5, 0, %wstate wrpr %g0, PSTATE_ALT, %pstate mov PCB_REG, %o0 mov PCPU_REG, %o1 wrpr %g0, PSTATE_NORMAL, %pstate mov %o0, PCB_REG mov %o1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate b %xcc, .Ltl0_trap_reenter mov T_FILL_RET, %o0 END(tl0_ret) /* * Kernel trap entry point * * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_int sfsr) * * This is easy because the stack is already setup and the windows don't need * to be split. We build a trapframe and call trap(), the same as above, but * the outs don't need to be saved. */ ENTRY(tl1_trap) sub %sp, TF_SIZEOF, %sp rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] mov PCB_REG, %l4 mov PCPU_REG, %l5 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] mov %l4, PCB_REG mov %l5, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate call trap add %sp, CCFSZ + SPOFF, %o0 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 ldub [%sp + SPOFF + CCFSZ + TF_PIL], %l3 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 wrpr %l3, 0, %pil wr %l4, 0, %y restore wrpr %g0, 2, %tl rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_trap: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tstate, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry END(tl1_trap) /* * void tl1_intr(u_int level, u_int mask) */ ENTRY(tl1_intr) sub %sp, TF_SIZEOF, %sp rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%p level=%#lx pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %asr21 wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] mov %o0, %l7 mov T_INTERRUPT | T_KERNEL, %o1 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] mov PCB_REG, %l4 mov PCPU_REG, %l5 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] mov %l4, PCB_REG mov %l5, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate call critical_enter nop SET(cnt+V_INTR, %l5, %l4) ATOMIC_INC_INT(%l4, %l5, %l6) SET(intr_handlers, %l5, %l4) sllx %l7, IH_SHIFT, %l5 ldx [%l4 + %l5], %l5 KASSERT(%l5, "tl1_intr: ih null") call %l5 add %sp, CCFSZ + SPOFF, %o0 call critical_exit nop lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 wrpr %l3, 0, %pil wr %l4, 0, %y restore wrpr %g0, 2, %tl rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tstate, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry END(tl1_intr) /* * Freshly forked processes come here when switched to for the first time. * The arguments to fork_exit() have been setup in the locals, we must move * them to the outs. */ ENTRY(fork_trampoline) #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g2 + TD_PROC], %g2 add %g2, P_COMM, %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %cwp, %g2 stx %g2, [%g1 + KTR_PARM3] 9: #endif mov %l0, %o0 mov %l1, %o1 call fork_exit mov %l2, %o2 b,a %xcc, tl0_ret nop END(fork_trampoline) diff --git a/sys/sparc64/sparc64/exception.s b/sys/sparc64/sparc64/exception.s index 1822aa421cc5..14ed0ce8cd16 100644 --- a/sys/sparc64/sparc64/exception.s +++ b/sys/sparc64/sparc64/exception.s @@ -1,2776 +1,2824 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp */ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_ddb.h" #include #include #include #include #include #include #include #include "assym.s" .register %g2,#ignore .register %g3,#ignore .register %g6,#ignore .register %g7,#ignore /* * Atomically set the reference bit in a tte. */ #define TTE_SET_BIT(r1, r2, r3, bit) \ add r1, TTE_DATA, r1 ; \ ldx [r1], r2 ; \ 9: or r2, bit, r3 ; \ casxa [r1] ASI_N, r2, r3 ; \ cmp r2, r3 ; \ bne,pn %xcc, 9b ; \ mov r3, r2 #define TTE_SET_REF(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_REF) #define TTE_SET_W(r1, r2, r3) TTE_SET_BIT(r1, r2, r3, TD_W) /* * Macros for spilling and filling live windows. * * NOTE: These macros use exactly 16 instructions, and it is assumed that the * handler will not use more than 24 instructions total, to leave room for * resume vectors which occupy the last 8 instructions. */ #define SPILL(storer, base, size, asi) \ storer %l0, [base + (0 * size)] asi ; \ storer %l1, [base + (1 * size)] asi ; \ storer %l2, [base + (2 * size)] asi ; \ storer %l3, [base + (3 * size)] asi ; \ storer %l4, [base + (4 * size)] asi ; \ storer %l5, [base + (5 * size)] asi ; \ storer %l6, [base + (6 * size)] asi ; \ storer %l7, [base + (7 * size)] asi ; \ storer %i0, [base + (8 * size)] asi ; \ storer %i1, [base + (9 * size)] asi ; \ storer %i2, [base + (10 * size)] asi ; \ storer %i3, [base + (11 * size)] asi ; \ storer %i4, [base + (12 * size)] asi ; \ storer %i5, [base + (13 * size)] asi ; \ storer %i6, [base + (14 * size)] asi ; \ storer %i7, [base + (15 * size)] asi #define FILL(loader, base, size, asi) \ loader [base + (0 * size)] asi, %l0 ; \ loader [base + (1 * size)] asi, %l1 ; \ loader [base + (2 * size)] asi, %l2 ; \ loader [base + (3 * size)] asi, %l3 ; \ loader [base + (4 * size)] asi, %l4 ; \ loader [base + (5 * size)] asi, %l5 ; \ loader [base + (6 * size)] asi, %l6 ; \ loader [base + (7 * size)] asi, %l7 ; \ loader [base + (8 * size)] asi, %i0 ; \ loader [base + (9 * size)] asi, %i1 ; \ loader [base + (10 * size)] asi, %i2 ; \ loader [base + (11 * size)] asi, %i3 ; \ loader [base + (12 * size)] asi, %i4 ; \ loader [base + (13 * size)] asi, %i5 ; \ loader [base + (14 * size)] asi, %i6 ; \ loader [base + (15 * size)] asi, %i7 #define ERRATUM50(reg) mov reg, reg #define KSTACK_SLOP 1024 /* * Sanity check the kernel stack and bail out if its wrong. * XXX: doesn't handle being on the panic stack. */ #define KSTACK_CHECK \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ add %sp, SPOFF, %g1 ; \ andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ bnz,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [PCPU(CURTHREAD)], %g2 ; \ ldx [%g2 + TD_KSTACK], %g2 ; \ add %g2, KSTACK_SLOP, %g2 ; \ subcc %g1, %g2, %g1 ; \ ble,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ cmp %g1, %g2 ; \ bgt,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG ENTRY(tl1_kstack_fault) rdpr %tl, %g1 1: cmp %g1, 2 be,a 2f nop #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" , %g2, %g3, %g4, 7, 8, 9) rdpr %tl, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tnpc, %g3 stx %g3, [%g2 + KTR_PARM1] 9: #endif sub %g1, 1, %g1 wrpr %g1, 0, %tl ba,a %xcc, 1b nop 2: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" , %g1, %g2, %g3, 7, 8, 9) add %sp, SPOFF, %g2 stx %g2, [%g1 + KTR_PARM1] ldx [PCPU(CURTHREAD)], %g2 ldx [%g2 + TD_KSTACK], %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %canrestore, %g2 stx %g2, [%g1 + KTR_PARM3] rdpr %cansave, %g2 stx %g2, [%g1 + KTR_PARM4] rdpr %otherwin, %g2 stx %g2, [%g1 + KTR_PARM5] rdpr %wstate, %g2 stx %g2, [%g1 + KTR_PARM6] 9: #endif wrpr %g0, 0, %canrestore wrpr %g0, 6, %cansave wrpr %g0, 0, %otherwin wrpr %g0, WSTATE_KERNEL, %wstate sub ASP_REG, SPOFF + CCFSZ, %sp clr %fp b %xcc, tl1_trap mov T_KSTACK_FAULT | T_KERNEL, %o0 END(tl1_kstack_fault) /* * Magic to resume from a spill or fill trap. If we get an alignment or an * mmu fault during a spill or a fill, this macro will detect the fault and * resume at a set instruction offset in the trap handler. * * To check if the previous trap was a spill/fill we convert the trapped pc * to a trap type and verify that it is in the range of spill/fill vectors. * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the * tl bit allows us to detect both ranges with one test. * * This is: * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 * * To calculate the new pc we take advantage of the xor feature of wrpr. * Forcing all the low bits of the trapped pc on we can produce any offset * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. * * 0x7f ^ 0x1f == 0x60 * 0x1f == (0x80 - 0x60) - 1 * * Which are the offset and xor value used to resume from alignment faults. */ /* * Determine if we have trapped inside of a spill/fill vector, and if so resume * at a fixed instruction offset in the trap vector. Must be called on * alternate globals. */ #define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ rdpr %tpc, %g1 ; \ ERRATUM50(%g1) ; \ rdpr %tba, %g2 ; \ sub %g1, %g2, %g2 ; \ srlx %g2, 5, %g2 ; \ andn %g2, 0x200, %g2 ; \ cmp %g2, 0x80 ; \ blu,pt %xcc, 9f ; \ cmp %g2, 0x100 ; \ bgeu,pt %xcc, 9f ; \ or %g1, 0x7f, %g1 ; \ wrpr %g1, xor, %tnpc ; \ stxa_g0_sfsr ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG ; \ done ; \ 9: ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG /* * For certain faults we need to clear the sfsr mmu register before returning. */ #define RSF_CLR_SFSR \ wr %g0, ASI_DMMU, %asi ; \ stxa %g0, [%g0 + AA_DMMU_SFSR] %asi #define RSF_XOR(off) ((0x80 - off) - 1) /* * Instruction offsets in spill and fill trap handlers for handling certain * nested traps, and corresponding xor constants for wrpr. */ #define RSF_OFF_ALIGN 0x60 #define RSF_OFF_MMU 0x70 #define RESUME_SPILLFILL_ALIGN \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) #define RESUME_SPILLFILL_MMU \ RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) #define RESUME_SPILLFILL_MMU_CLR_SFSR \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) /* * Constant to add to %tnpc when taking a fill trap just before returning to * user mode. */ #define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill /* * Retry a spill or fill with a different wstate due to an alignment fault. * We may just be using the wrong stack offset. */ #define RSF_ALIGN_RETRY(ws) \ wrpr %g0, (ws), %wstate ; \ retry ; \ .align 16 /* * Generate a T_SPILL or T_FILL trap if the window operation fails. */ #define RSF_TRAP(type) \ b %xcc, tl0_sftrap ; \ mov type, %g2 ; \ .align 16 /* * Game over if the window operation fails. */ #define RSF_FATAL(type) \ b %xcc, rsf_fatal ; \ mov type, %g2 ; \ .align 16 /* * Magic to resume from a failed fill a few instructions after the corrsponding * restore. This is used on return from the kernel to usermode. */ #define RSF_FILL_MAGIC \ rdpr %tnpc, %g1 ; \ add %g1, RSF_FILL_INC, %g1 ; \ wrpr %g1, 0, %tnpc ; \ done ; \ .align 16 /* * Spill to the pcb if a spill to the user stack in kernel mode fails. */ #define RSF_SPILL_TOPCB \ b,a %xcc, tl1_spill_topcb ; \ nop ; \ .align 16 ENTRY(rsf_fatal) #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" , %g1, %g3, %g4, 7, 8, 9) rdpr %tt, %g3 stx %g3, [%g1 + KTR_PARM1] stx %g2, [%g1 + KTR_PARM2] 9: #endif KSTACK_CHECK sir END(rsf_fatal) .comm intrnames, IV_MAX * 8 .comm eintrnames, 0 .comm intrcnt, IV_MAX * 8 .comm eintrcnt, 0 /* * Trap table and associated macros * * Due to its size a trap table is an inherently hard thing to represent in * code in a clean way. There are approximately 1024 vectors, of 8 or 32 * instructions each, many of which are identical. The way that this is * layed out is the instructions (8 or 32) for the actual trap vector appear * as an AS macro. In general this code branches to tl0_trap or tl1_trap, * but if not supporting code can be placed just after the definition of the * macro. The macros are then instantiated in a different section (.trap), * which is setup to be placed by the linker at the beginning of .text, and the * code around the macros is moved to the end of trap table. In this way the * code that must be sequential in memory can be split up, and located near * its supporting code so that it is easier to follow. */ /* * Clean window traps occur when %cleanwin is zero to ensure that data * is not leaked between address spaces in registers. */ .macro clean_window clr %o0 clr %o1 clr %o2 clr %o3 clr %o4 clr %o5 clr %o6 clr %o7 clr %l0 clr %l1 clr %l2 clr %l3 clr %l4 clr %l5 clr %l6 rdpr %cleanwin, %l7 inc %l7 wrpr %l7, 0, %cleanwin clr %l7 retry .align 128 .endm /* * Stack fixups for entry from user mode. We are still running on the * user stack, and with its live registers, so we must save soon. We * are on alternate globals so we do have some registers. Set the * transitional window state, and do the save. If this traps we * we attempt to spill a window to the user stack. If this fails, * we spill the window to the pcb and continue. Spilling to the pcb * must not fail. * * NOTE: Must be called with alternate globals and clobbers %g1. */ .macro tl0_split rdpr %wstate, %g1 wrpr %g1, WSTATE_TRANSITION, %wstate save .endm .macro tl0_setup type tl0_split b %xcc, tl0_trap mov \type, %o0 .endm /* * Generic trap type. Call trap() with the specified type. */ .macro tl0_gen type tl0_setup \type .align 32 .endm /* * This is used to suck up the massive swaths of reserved trap types. * Generates count "reserved" trap vectors. */ .macro tl0_reserved count .rept \count tl0_gen T_RESERVED .endr .endm .macro tl0_fp_restore wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi ldda [PCB_REG + PCB_FPSTATE + FP_FB0] %asi, %f0 ldda [PCB_REG + PCB_FPSTATE + FP_FB1] %asi, %f16 ldda [PCB_REG + PCB_FPSTATE + FP_FB2] %asi, %f32 ldda [PCB_REG + PCB_FPSTATE + FP_FB3] %asi, %f48 membar #Sync done .align 32 .endm .macro tl0_insn_excptn wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync b %xcc, tl0_sfsr_trap mov T_INSTRUCTION_EXCEPTION, %g2 .align 32 .endm .macro tl0_data_excptn wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync b %xcc, tl0_sfsr_trap mov T_DATA_EXCEPTION, %g2 .align 32 .endm .macro tl0_align wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync b %xcc, tl0_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 .align 32 .endm ENTRY(tl0_sfsr_trap) tl0_split mov %g3, %o4 mov %g4, %o5 b %xcc, tl0_trap mov %g2, %o0 END(tl0_sfsr_trap) .macro tl0_intr level, mask tl0_split set \mask, %o1 b %xcc, tl0_intr mov \level, %o0 .align 32 .endm #define INTR(level, traplvl) \ tl ## traplvl ## _intr level, 1 << level #define TICK(traplvl) \ tl ## traplvl ## _intr PIL_TICK, 1 #define INTR_LEVEL(tl) \ INTR(1, tl) ; \ INTR(2, tl) ; \ INTR(3, tl) ; \ INTR(4, tl) ; \ INTR(5, tl) ; \ INTR(6, tl) ; \ INTR(7, tl) ; \ INTR(8, tl) ; \ INTR(9, tl) ; \ INTR(10, tl) ; \ INTR(11, tl) ; \ INTR(12, tl) ; \ INTR(13, tl) ; \ TICK(tl) ; \ INTR(15, tl) ; .macro tl0_intr_level INTR_LEVEL(0) .endm .macro intr_vector ldxa [%g0] ASI_INTR_RECEIVE, %g1 andcc %g1, IRSR_BUSY, %g0 bnz,a,pt %xcc, intr_enqueue nop sir .align 32 .endm .macro immu_miss_user /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Compute the tte bucket address. */ set (1 << TSB_BUCKET_ADDRESS_BITS) - 1, %g1 and %g1, %g3, %g1 sllx %g1, TSB_BUCKET_SHIFT + TTE_SHIFT, %g1 add %g1, TSB_REG, %g1 /* * Loop over the ttes in this bucket */ /* * Load the tte. Note that this instruction may fault, clobbering * the context of the tag access register (at least), and the contents * of %g3, %g4, %g5, and %g6. Luckily we can recover %g3, and we do * not use %g4 or %g5 until this instruction completes successfully. */ 1: ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Recover the virtual page number, which may have been clobbered. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Check that its valid and executable and that the virtual page * numbers match. */ brgez,pn %g5, 2f andcc %g5, TD_EXEC, %g0 bz,pn %xcc, 2f cmp %g3, %g4 bne,pn %xcc, 2f EMPTY /* * We matched a tte, load the tlb. */ /* * Set the reference bit, if it's currently clear. */ andcc %g5, TD_REF, %g0 bz,a,pn %xcc, tl0_immu_miss_set_ref nop /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_IMMU_TAR] %asi stxa %g5, [%g0] ASI_ITLB_DATA_IN_REG retry /* * Check the low bits to see if we've finished the bucket. */ 2: add %g1, 1 << TTE_SHIFT, %g1 andcc %g1, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,a,pt %xcc, 1b nop .endm .macro tl0_immu_miss /* * Force kernel store order. */ wrpr %g0, PSTATE_MMU, %pstate /* * Load the virtual page number and context from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 immu_miss_user b,a %xcc, tl0_immu_miss_trap nop .align 128 .endm ENTRY(tl0_immu_miss_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g1, %g4, %g5) /* * May have become invalid, in which case start over. */ brgez,pn %g4, 1f or %g4, TD_REF, %g4 /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_IMMU_TAR] %asi stxa %g4, [%g0] ASI_ITLB_DATA_IN_REG 1: retry END(tl0_immu_miss_set_ref) ENTRY(tl0_immu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_IMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Reload the tag access register. */ ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 /* * Save the tag access register, and call common trap code. */ tl0_split mov %g2, %o3 b %xcc, tl0_trap mov T_INSTRUCTION_MISS, %o0 END(tl0_immu_miss_trap) .macro dmmu_miss_user /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Compute the tte bucket address. */ set (1 << TSB_BUCKET_ADDRESS_BITS) - 1, %g1 and %g1, %g3, %g1 sllx %g1, TSB_BUCKET_SHIFT + TTE_SHIFT, %g1 add %g1, TSB_REG, %g1 /* * Loop over the ttes in this bucket */ /* * Load the tte. Note that this instruction may fault, clobbering * the contents of the tag access register (at least), and the contents * of %g3, %g4, %g5, and %g6. Luckily we can recover %g3, and we do * not use %g4 or %g5 until this instruction completes successfully. */ 1: ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Recover the virtual page number, which may have been clobbered. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Check that its valid and that the virtual page numbers match. */ brgez,pn %g5, 2f cmp %g3, %g4 bne,pn %xcc, 2f EMPTY /* * We matched a tte, load the tlb. */ /* * Set the reference bit, if it's currently clear. */ andcc %g5, TD_REF, %g0 bz,a,pn %xcc, dmmu_miss_user_set_ref nop /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG retry /* * Check the low bits to see if we've finished the bucket. */ 2: add %g1, 1 << TTE_SHIFT, %g1 andcc %g1, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,a,pt %xcc, 1b nop .endm ENTRY(dmmu_miss_user_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g1, %g4, %g5) /* * May have become invalid, in which case start over. */ brgez,pn %g4, 1f or %g4, TD_REF, %g4 /* * Load the tte tag and data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi stxa %g4, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(dmmu_miss_user_set_ref) .macro tl0_dmmu_miss /* * Force kernel store order. */ wrpr %g0, PSTATE_MMU, %pstate /* * Load the virtual page number and context from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Try a fast inline lookup of the primary tsb. */ dmmu_miss_user /* * Not in primary tsb, call c code. Not much else fits inline. */ b,a %xcc, tl0_dmmu_miss_trap nop .align 128 .endm ENTRY(tl0_dmmu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Save the tag access register and call common trap code. */ tl0_split mov %g2, %o3 b %xcc, tl0_trap mov T_DATA_MISS, %o0 END(tl0_dmmu_miss_trap) .macro dmmu_prot_user /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Compute the tte bucket address. */ set (1 << TSB_BUCKET_ADDRESS_BITS) - 1, %g1 and %g1, %g3, %g1 sllx %g1, TSB_BUCKET_SHIFT + TTE_SHIFT, %g1 add %g1, TSB_REG, %g1 /* * Loop over the ttes in this bucket */ /* * Load the tte. Note that this instruction may fault, clobbering * the context of the tag access register (at least), and the contents * of %g3, %g4, %g5, and %g6. Luckily we can recover %g3, and we do * not use %g4 or %g5 until this instruction completes successfully. */ 1: ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Recover the virtual page number, which may have been clobbered. */ srlx %g2, TAR_VPN_SHIFT, %g3 /* * Check that its valid and writable and that the virtual page * numbers match. */ brgez,pn %g5, 2f andcc %g5, TD_SW, %g0 bz,pn %xcc, 2f cmp %g3, %g4 bne,pn %xcc, 2f nop /* * Set the hardware write bit. */ b,a %xcc, dmmu_prot_set_w nop /* * Check the low bits to see if we've finished the bucket. */ 2: add %g1, 1 << TTE_SHIFT, %g1 andcc %g1, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,a,pt %xcc, 1b nop .endm .macro tl0_dmmu_prot /* * Force kernel store order. */ wrpr %g0, PSTATE_MMU, %pstate /* * Load the virtual page number and context from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Try a fast inline lookup of the tsb. */ dmmu_prot_user /* * Not in tsb. Call c code. */ b,a %xcc, tl0_dmmu_prot_trap nop .align 128 .endm ENTRY(dmmu_prot_set_w) /* * Set the hardware write bit in the tte. */ TTE_SET_W(%g1, %g4, %g5) /* * Delete the old TLB entry and clear the sfsr. */ sllx %g3, TAR_VPN_SHIFT, %g3 stxa %g0, [%g3] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * May have become invalid in which case start over. */ brgez,pn %g4, 1f or %g4, TD_W, %g4 /* * Load the tte data into the tlb and retry the instruction. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi stxa %g4, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(dmmu_prot_set_w) ENTRY(tl0_dmmu_prot_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Load the tar, sfar and sfsr. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * Save the mmu registers and call common trap code. */ tl0_split mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 b %xcc, tl0_trap mov T_DATA_PROTECTION, %o0 END(tl0_dmmu_prot_trap) .macro tl0_spill_0_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_spill_1_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_fill_0_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm .macro tl0_fill_1_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm ENTRY(tl0_sftrap) rdpr %tstate, %g1 and %g1, TSTATE_CWP_MASK, %g1 wrpr %g1, 0, %cwp tl0_split b %xcc, tl0_trap mov %g2, %o0 END(tl0_sftrap) .macro tl0_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_syscall tl0_split b %xcc, tl0_syscall mov T_SYSCALL, %o0 .align 32 .endm .macro tl0_soft count .rept \count tl0_gen T_SOFT .endr .endm .macro tl1_split rdpr %wstate, %g1 wrpr %g1, WSTATE_NESTED, %wstate save %sp, -CCFSZ, %sp .endm .macro tl1_setup type tl1_split b %xcc, tl1_trap mov \type | T_KERNEL, %o0 .endm .macro tl1_gen type tl1_setup \type .align 32 .endm .macro tl1_reserved count .rept \count tl1_gen T_RESERVED .endr .endm .macro tl1_insn_excptn wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync b %xcc, tl1_insn_exceptn_trap mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 .align 32 .endm ENTRY(tl1_insn_exceptn_trap) tl1_split mov %g3, %o4 mov %g4, %o5 b %xcc, tl1_trap mov %g2, %o0 END(tl1_insn_exceptn_trap) .macro tl1_data_excptn b,a %xcc, tl1_data_excptn_trap nop .align 32 .endm ENTRY(tl1_data_excptn_trap) wrpr %g0, PSTATE_ALT, %pstate RESUME_SPILLFILL_MMU_CLR_SFSR b %xcc, tl1_sfsr_trap mov T_DATA_EXCEPTION | T_KERNEL, %g2 END(tl1_data_excptn_trap) .macro tl1_align b,a %xcc, tl1_align_trap nop .align 32 .endm ENTRY(tl1_align_trap) wrpr %g0, PSTATE_ALT, %pstate RESUME_SPILLFILL_ALIGN b %xcc, tl1_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 END(tl1_data_excptn_trap) ENTRY(tl1_sfsr_trap) wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split mov %g3, %o4 mov %g4, %o5 b %xcc, tl1_trap mov %g2, %o0 END(tl1_sfsr_trap) .macro tl1_intr level, mask tl1_split set \mask, %o1 b %xcc, tl1_intr mov \level, %o0 .align 32 .endm .macro tl1_intr_level INTR_LEVEL(1) .endm ENTRY(intr_dequeue) save %sp, -CCFSZ, %sp 1: ldx [PCPU(IRHEAD)], %l0 brnz,a,pt %l0, 2f nop ret restore 2: wrpr %g0, PSTATE_NORMAL, %pstate ldx [%l0 + IR_NEXT], %l1 brnz,pt %l1, 3f stx %l1, [PCPU(IRHEAD)] PCPU_ADDR(IRHEAD, %l1) stx %l1, [PCPU(IRTAIL)] 3: ldx [%l0 + IR_FUNC], %o0 ldx [%l0 + IR_ARG], %o1 ldx [%l0 + IR_VEC], %o2 ldx [PCPU(IRFREE)], %l1 stx %l1, [%l0 + IR_NEXT] stx %l0, [PCPU(IRFREE)] wrpr %g0, PSTATE_KERNEL, %pstate call %o0 mov %o1, %o0 ba,a %xcc, 1b nop END(intr_dequeue) /* * Handle a vectored interrupt. * * This is either a data bearing mondo vector interrupt, or a cross trap * request from another cpu. In either case the hardware supplies an * interrupt packet, in the form of 3 data words which are read from internal * registers. A data bearing mondo vector packet consists of an interrupt * number in the first data word, and zero in 2nd and 3rd. We use the * interrupt number to find the function, argument and priority from the * intr_vector table, allocate and fill in an intr_request from the per-cpu * free list, link it onto the per-cpu active list and finally post a softint * at the desired priority. Cross trap requests come in 2 forms, direct * and queued. Direct requests are distinguished by the first data word * being zero. The 2nd data word carries a function to call and the 3rd * an argument to pass. The function is jumped to directly. It executes * in nucleus context on interrupt globals and with all interrupts disabled, * therefore it must be fast, and the things that it can do are limited. * Queued cross trap requests are handled much like mondo vectors, except * that the function, argument and priority are contained in the interrupt * packet itself. They are distinguished by the upper 4 bits of the data * word being non-zero, which specifies the priority of the softint to * deliver. * * Register usage: * %g1 - pointer to intr_request * %g2 - pointer to intr_vector, temp once required data is loaded * %g3 - interrupt number for mondo vectors, unused otherwise * %g4 - function, from the interrupt packet for cross traps, or * loaded from the interrupt registers for mondo vecors * %g5 - argument, as above for %g4 * %g6 - softint priority */ ENTRY(intr_enqueue) /* * Load the interrupt packet from the hardware. */ wr %g0, ASI_SDB_INTR_R, %asi ldxa [%g0 + AA_SDB_INTR_D0] %asi, %g3 ldxa [%g0 + AA_SDB_INTR_D1] %asi, %g4 ldxa [%g0 + AA_SDB_INTR_D2] %asi, %g5 stxa %g0, [%g0] ASI_INTR_RECEIVE membar #Sync #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: data=%#lx %#lx %#lx" , %g1, %g2, %g6, 7, 8, 9) stx %g3, [%g1 + KTR_PARM1] stx %g4, [%g1 + KTR_PARM2] stx %g5, [%g1 + KTR_PARM3] 9: #endif /* * If the first data word is zero this is a direct cross trap request. * The 2nd word points to code to execute and the 3rd is an argument * to pass. Jump to it. */ brnz,a,pt %g3, 1f nop #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: direct ipi func=%#lx arg=%#lx" , %g1, %g2, %g6, 7, 8, 9) stx %g4, [%g1 + KTR_PARM1] stx %g5, [%g1 + KTR_PARM2] 9: #endif jmpl %g4, %g0 nop /* NOTREACHED */ /* * If the high 4 bits of the 1st data word are non-zero, this is a * queued cross trap request to be delivered as a softint. The high * 4 bits of the 1st data word specify a priority, and the 2nd and * 3rd a function and argument. */ 1: srlx %g3, 60, %g6 brnz,a,pn %g6, 2f clr %g3 /* * Find the function, argument and desired priority from the * intr_vector table. */ SET(intr_vectors, %g4, %g2) sllx %g3, IV_SHIFT, %g4 add %g2, %g4, %g2 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: mondo vector func=%#lx arg=%#lx pri=%#lx" , %g4, %g5, %g6, 7, 8, 9) ldx [%g2 + IV_FUNC], %g5 stx %g5, [%g4 + KTR_PARM1] ldx [%g2 + IV_ARG], %g5 stx %g5, [%g4 + KTR_PARM2] ldx [%g2 + IV_PRI], %g5 stx %g5, [%g4 + KTR_PARM3] 9: #endif ldx [%g2 + IV_FUNC], %g4 ldx [%g2 + IV_ARG], %g5 lduw [%g2 + IV_PRI], %g6 ba,a %xcc, 3f nop /* * Get a intr_request from the free list. There should always be one * unless we are getting an interrupt storm from stray interrupts, in * which case the we will deference a NULL pointer and panic. */ 2: #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: queued ipi func=%#lx arg=%#lx pri=%#lx" , %g1, %g2, %g3, 7, 8, 9) stx %g4, [%g1 + KTR_PARM1] stx %g5, [%g1 + KTR_PARM2] stx %g6, [%g1 + KTR_PARM3] 9: clr %g3 #endif 3: ldx [PCPU(IRFREE)], %g1 ldx [%g1 + IR_NEXT], %g2 stx %g2, [PCPU(IRFREE)] /* * Store the vector number, function, argument and priority. */ stw %g3, [%g1 + IR_VEC] stx %g4, [%g1 + IR_FUNC] stx %g5, [%g1 + IR_ARG] stw %g6, [%g1 + IR_PRI] /* * Link it onto the end of the active list. */ stx %g0, [%g1 + IR_NEXT] ldx [PCPU(IRTAIL)], %g4 stx %g1, [%g4] add %g1, IR_NEXT, %g1 stx %g1, [PCPU(IRTAIL)] /* * Trigger a softint at the level indicated by the priority. */ mov 1, %g1 sllx %g1, %g6, %g1 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "intr_enqueue: softint pil=%#lx pri=%#lx mask=%#lx" , %g2, %g3, %g4, 7, 8, 9) rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM1] stx %g6, [%g2 + KTR_PARM2] stx %g1, [%g2 + KTR_PARM3] 9: #endif wr %g1, 0, %asr20 /* * Done, retry the instruction. */ retry END(intr_enqueue) .macro tl1_immu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g6 /* * Extract the virtual page number from the contents of the tag access * register. */ srlx %g6, TAR_VPN_SHIFT, %g6 /* * Find the index into the kernel tsb. */ set TSB_KERNEL_MASK, %g4 and %g6, %g4, %g3 /* * Compute the tte address. */ ldxa [%g0 + AA_IMMU_TSB] %asi, %g4 sllx %g3, TTE_SHIFT, %g3 add %g3, %g4, %g3 /* * Load the tte. */ ldda [%g3] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Check that its valid and executable and that the virtual page * numbers match. */ brgez,pn %g5, tl1_immu_miss_trap andcc %g5, TD_EXEC, %g0 bz,pn %xcc, tl1_immu_miss_trap cmp %g4, %g6 bne,pn %xcc, tl1_immu_miss_trap EMPTY /* * Set the reference bit if its currently clear. */ andcc %g5, TD_REF, %g0 bnz,a,pt %xcc, 1f nop TTE_SET_REF(%g3, %g5, %g4) /* * May have become invalid, in which case start over. */ brgez,pn %g5, 2f or %g5, TD_REF, %g5 /* * Load the tte data into the TLB and retry the instruction. */ 1: stxa %g5, [%g0] ASI_ITLB_DATA_IN_REG 2: retry .align 128 .endm ENTRY(tl1_immu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 tl1_split mov %g2, %o3 b %xcc, tl1_trap mov T_INSTRUCTION_MISS | T_KERNEL, %o0 END(tl1_immu_miss_trap) .macro tl1_dmmu_miss /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g6 /* * Extract the context from the contents of the tag access register. - * If its non-zero this is a fault on a user address, otherwise get - * the virtual page number. + * If its non-zero this is a fault on a user address. Note that the + * faulting address is passed in %g2. */ sllx %g6, 64 - TAR_VPN_SHIFT, %g5 brnz,a,pn %g5, tl1_dmmu_miss_user mov %g6, %g2 + /* + * Check for the direct mapped physical region. These addresses have + * the high bit set so they are negative. + */ + brlz,pn %g6, tl1_dmmu_miss_direct + EMPTY + /* * Find the index into the kernel tsb. */ - set TSB_KERNEL_MASK, %g4 + set TSB_KERNEL_MASK, %g4 srlx %g6, TAR_VPN_SHIFT, %g6 and %g6, %g4, %g3 /* * Compute the tte address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g4 sllx %g3, TTE_SHIFT, %g3 add %g3, %g4, %g3 /* * Load the tte. */ ldda [%g3] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Check that its valid and that the virtual page numbers match. */ brgez,pn %g5, tl1_dmmu_miss_trap cmp %g4, %g6 bne,pn %xcc, tl1_dmmu_miss_trap EMPTY /* * Set the reference bit if its currently clear. */ andcc %g5, TD_REF, %g0 bnz,a,pt %xcc, 1f nop TTE_SET_REF(%g3, %g5, %g4) /* * May have become invalid, in which case start over. */ brgez,pn %g5, 2f or %g5, TD_REF, %g5 /* * Load the tte data into the TLB and retry the instruction. */ 1: stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG 2: retry .align 128 .endm ENTRY(tl1_dmmu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate KSTACK_CHECK ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 tl1_split mov %g2, %o3 b %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl1_dmmu_miss_trap) +ENTRY(tl1_dmmu_miss_direct) +#if KTR_COMPILE & KTR_TRAP + CATR(KTR_TRAP, "tl1_dmmu_miss_direct: pc=%#lx sp=%#lx tar=%#lx" + , %g1, %g2, %g3, 7, 8, 9) + rdpr %tpc, %g2 + stx %g2, [%g1 + KTR_PARM1] + add %sp, SPOFF, %g2 + stx %g2, [%g1 + KTR_PARM2] + stx %g6, [%g1 + KTR_PARM3] +9: +#endif + + /* + * Check the cache bits in the virtual address to see if this mapping + * is virtually cacheable. We set this up so that the masks fit in + * immediates... Note that the arithmetic shift sign extends, keeping + * all the top bits set. + */ + srax %g6, TLB_DIRECT_SHIFT, %g6 + andcc %g6, TLB_DIRECT_UNCACHEABLE, %g0 + mov TD_CP | TD_CV | TD_W, %g1 + movnz %xcc, TD_CP | TD_W, %g1 + + /* + * Mask off the high bits of the virtual address to get the physical + * address, and or in the tte bits. The high bit is left set in the + * physical address, which corresponds to the tte valid bit, so that + * we don't have to include it in the tte bits. We ignore the cache + * bits, since they get shifted into the soft tte bits anyway. + */ + setx TLB_DIRECT_MASK & ~TD_V, %g3, %g2 + andn %g6, %g2, %g3 + or %g3, %g1, %g3 + + /* + * Load the tte data into the TLB and retry the instruction. + */ + stxa %g3, [%g0] ASI_DTLB_DATA_IN_REG + retry +END(tl1_dmmu_miss_direct) + ENTRY(tl1_dmmu_miss_user) /* * Try a fast inline lookup of the user tsb. */ dmmu_miss_user /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Handle faults during window spill/fill. */ RESUME_SPILLFILL_MMU /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 tl1_split mov %g2, %o3 b %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl1_dmmu_miss_user) .macro tl1_dmmu_prot /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g6 /* * Extract the context from the contents of the tag access register. * If its non-zero this is a fault on a user address, otherwise get * the virtual page number. */ sllx %g6, 64 - TAR_VPN_SHIFT, %g5 brnz,a,pn %g5, tl1_dmmu_prot_user mov %g6, %g2 /* * Find the index into the kernel tsb. */ set TSB_KERNEL_MASK, %g4 srlx %g6, TAR_VPN_SHIFT, %g6 and %g6, %g4, %g5 /* * Compute the tte address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g4 sllx %g5, TTE_SHIFT, %g5 add %g4, %g5, %g3 /* * Load the tte. */ ldda [%g3] ASI_NUCLEUS_QUAD_LDD, %g4 /*, %g5 */ /* * Check that its valid and writeable and that the virtual page * numbers match. */ brgez,pn %g5, tl1_dmmu_prot_trap andcc %g5, TD_SW, %g0 bz,pn %xcc, tl1_dmmu_prot_trap cmp %g4, %g6 bne,pn %xcc, tl1_dmmu_prot_trap EMPTY /* * Delete the old TLB entry and clear the sfsr. */ sllx %g6, TAR_VPN_SHIFT, %g6 or %g6, TLB_DEMAP_NUCLEUS, %g6 stxa %g0, [%g6] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync ba,a %xcc, tl1_dmmu_prot_cont nop .align 128 .endm ENTRY(tl1_dmmu_prot_cont) /* * Set the hardware write bit. */ TTE_SET_W(%g3, %g5, %g6) /* * Load the tte data into the TLB and retry the instruction. */ or %g5, TD_W, %g5 stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG retry END(tl1_dmmu_prot_cont) ENTRY(tl1_dmmu_prot_user) /* * Try a fast inline lookup of the user tsb. */ dmmu_prot_user /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g2, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* Handle faults during window spill/fill. */ RESUME_SPILLFILL_MMU_CLR_SFSR b,a %xcc, tl1_dmmu_prot_trap nop END(tl1_dmmu_prot_user) ENTRY(tl1_dmmu_prot_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Load the sfar, sfsr and tar. Clear the sfsr. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 b %xcc, tl1_trap mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl1_dmmu_prot_trap) .macro tl1_spill_0_n SPILL(stx, %sp + SPOFF, 8, EMPTY) saved retry .align 32 RSF_FATAL(T_SPILL) RSF_FATAL(T_SPILL) .endm .macro tl1_spill_2_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_3_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_0_o wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_1_o wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_2_o RSF_SPILL_TOPCB .align 128 .endm .macro tl1_fill_0_n FILL(ldx, %sp + SPOFF, 8, EMPTY) restored retry .align 32 RSF_FATAL(T_FILL) RSF_FATAL(T_FILL) .endm .macro tl1_fill_2_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm .macro tl1_fill_3_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm /* * This is used to spill windows that are still occupied with user * data on kernel entry to the pcb. */ ENTRY(tl1_spill_topcb) wrpr %g0, PSTATE_ALT, %pstate /* Free some globals for our use. */ dec 24, ASP_REG stx %g1, [ASP_REG + 0] stx %g2, [ASP_REG + 8] stx %g3, [ASP_REG + 16] ldx [PCB_REG + PCB_NSAVED], %g1 sllx %g1, PTR_SHIFT, %g2 add %g2, PCB_REG, %g2 stx %sp, [%g2 + PCB_RWSP] sllx %g1, RW_SHIFT, %g2 add %g2, PCB_REG, %g2 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) inc %g1 stx %g1, [PCB_REG + PCB_NSAVED] #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" , %g1, %g2, %g3, 7, 8, 9) rdpr %tpc, %g2 stx %g2, [%g1 + KTR_PARM1] rdpr %tnpc, %g2 stx %g2, [%g1 + KTR_PARM2] stx %sp, [%g1 + KTR_PARM3] ldx [PCB_REG + PCB_NSAVED], %g2 stx %g2, [%g1 + KTR_PARM4] 9: #endif saved ldx [ASP_REG + 16], %g3 ldx [ASP_REG + 8], %g2 ldx [ASP_REG + 0], %g1 inc 24, ASP_REG retry END(tl1_spill_topcb) .macro tl1_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_soft count .rept \count tl1_gen T_SOFT | T_KERNEL .endr .endm .sect .trap .align 0x8000 .globl tl0_base tl0_base: tl0_reserved 8 ! 0x0-0x7 tl0_insn_excptn: tl0_insn_excptn ! 0x8 tl0_reserved 1 ! 0x9 tl0_insn_error: tl0_gen T_INSTRUCTION_ERROR ! 0xa tl0_reserved 5 ! 0xb-0xf tl0_insn_illegal: tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 tl0_priv_opcode: tl0_gen T_PRIVILEGED_OPCODE ! 0x11 tl0_reserved 14 ! 0x12-0x1f tl0_fp_disabled: tl0_gen T_FP_DISABLED ! 0x20 tl0_fp_ieee: tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 tl0_fp_other: tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 tl0_tag_ovflw: tl0_gen T_TAG_OFERFLOW ! 0x23 tl0_clean_window: clean_window ! 0x24 tl0_divide: tl0_gen T_DIVISION_BY_ZERO ! 0x28 tl0_reserved 7 ! 0x29-0x2f tl0_data_excptn: tl0_data_excptn ! 0x30 tl0_reserved 1 ! 0x31 tl0_data_error: tl0_gen T_DATA_ERROR ! 0x32 tl0_reserved 1 ! 0x33 tl0_align: tl0_align ! 0x34 tl0_align_lddf: tl0_gen T_RESERVED ! 0x35 tl0_align_stdf: tl0_gen T_RESERVED ! 0x36 tl0_priv_action: tl0_gen T_PRIVILEGED_ACTION ! 0x37 tl0_reserved 9 ! 0x38-0x40 tl0_intr_level: tl0_intr_level ! 0x41-0x4f tl0_reserved 16 ! 0x50-0x5f tl0_intr_vector: intr_vector ! 0x60 tl0_watch_phys: tl0_gen T_PA_WATCHPOINT ! 0x61 tl0_watch_virt: tl0_gen T_VA_WATCHPOINT ! 0x62 tl0_ecc: tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 tl0_immu_miss: tl0_immu_miss ! 0x64 tl0_dmmu_miss: tl0_dmmu_miss ! 0x68 tl0_dmmu_prot: tl0_dmmu_prot ! 0x6c tl0_reserved 16 ! 0x70-0x7f tl0_spill_0_n: tl0_spill_0_n ! 0x80 tl0_spill_1_n: tl0_spill_1_n ! 0x84 tl0_spill_bad 14 ! 0x88-0xbf tl0_fill_0_n: tl0_fill_0_n ! 0xc0 tl0_fill_1_n: tl0_fill_1_n ! 0xc4 tl0_fill_bad 14 ! 0xc8-0xff tl0_soft: tl0_reserved 1 ! 0x100 tl0_gen T_BREAKPOINT ! 0x101 tl0_gen T_DIVISION_BY_ZERO ! 0x102 tl0_reserved 1 ! 0x103 tl0_gen T_CLEAN_WINDOW ! 0x104 tl0_gen T_RANGE_CHECK ! 0x105 tl0_gen T_FIX_ALIGNMENT ! 0x106 tl0_gen T_INTEGER_OVERFLOW ! 0x107 tl0_reserved 1 ! 0x108 tl0_syscall ! 0x109 tl0_fp_restore ! 0x10a tl0_reserved 5 ! 0x10b-0x10f tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f tl0_reserved 224 ! 0x120-0x1ff tl1_base: tl1_reserved 8 ! 0x200-0x207 tl1_insn_excptn: tl1_insn_excptn ! 0x208 tl1_reserved 1 ! 0x209 tl1_insn_error: tl1_gen T_INSTRUCTION_ERROR ! 0x20a tl1_reserved 5 ! 0x20b-0x20f tl1_insn_illegal: tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 tl1_priv_opcode: tl1_gen T_PRIVILEGED_OPCODE ! 0x211 tl1_reserved 14 ! 0x212-0x21f tl1_fp_disabled: tl1_gen T_FP_DISABLED ! 0x220 tl1_fp_ieee: tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 tl1_fp_other: tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 tl1_tag_ovflw: tl1_gen T_TAG_OFERFLOW ! 0x223 tl1_clean_window: clean_window ! 0x224 tl1_divide: tl1_gen T_DIVISION_BY_ZERO ! 0x228 tl1_reserved 7 ! 0x229-0x22f tl1_data_excptn: tl1_data_excptn ! 0x230 tl1_reserved 1 ! 0x231 tl1_data_error: tl1_gen T_DATA_ERROR ! 0x232 tl1_reserved 1 ! 0x233 tl1_align: tl1_align ! 0x234 tl1_align_lddf: tl1_gen T_RESERVED ! 0x235 tl1_align_stdf: tl1_gen T_RESERVED ! 0x236 tl1_priv_action: tl1_gen T_PRIVILEGED_ACTION ! 0x237 tl1_reserved 9 ! 0x238-0x240 tl1_intr_level: tl1_intr_level ! 0x241-0x24f tl1_reserved 16 ! 0x250-0x25f tl1_intr_vector: intr_vector ! 0x260 tl1_watch_phys: tl1_gen T_PA_WATCHPOINT ! 0x261 tl1_watch_virt: tl1_gen T_VA_WATCHPOINT ! 0x262 tl1_ecc: tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 tl1_immu_miss: tl1_immu_miss ! 0x264 tl1_dmmu_miss: tl1_dmmu_miss ! 0x268 tl1_dmmu_prot: tl1_dmmu_prot ! 0x26c tl1_reserved 16 ! 0x270-0x27f tl1_spill_0_n: tl1_spill_0_n ! 0x280 tl1_spill_bad 1 ! 0x284 tl1_spill_2_n: tl1_spill_2_n ! 0x288 tl1_spill_3_n: tl1_spill_3_n ! 0x29c tl1_spill_bad 4 ! 0x290-0x29f tl1_spill_0_o: tl1_spill_0_o ! 0x2a0 tl1_spill_1_o: tl1_spill_1_o ! 0x2a4 tl1_spill_2_o: tl1_spill_2_o ! 0x2a8 tl1_spill_bad 5 ! 0x2ac-0x2bf tl1_fill_0_n: tl1_fill_0_n ! 0x2c0 tl1_fill_bad 1 ! 0x2c4 tl1_fill_2_n: tl1_fill_2_n ! 0x2d0 tl1_fill_3_n: tl1_fill_3_n ! 0x2d4 tl1_fill_bad 12 ! 0x2d8-0x2ff tl1_reserved 1 ! 0x300 tl1_breakpoint: tl1_gen T_BREAKPOINT ! 0x301 tl1_gen T_RSTRWP_PHYS ! 0x302 tl1_gen T_RSTRWP_VIRT ! 0x303 tl1_reserved 252 ! 0x304-0x3ff /* * User trap entry point. * * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_int sfsr) * * The following setup has been performed: * - the windows have been split and the active user window has been saved * (maybe just to the pcb) * - we are on alternate globals and interrupts are disabled * * We switch to the kernel stack, build a trapframe, switch to normal * globals, enable interrupts and call trap. * * NOTE: We must be very careful setting up the per-cpu pointer. We know that * it has been pre-set in alternate globals, so we read it from there and setup * the normal %g7 *before* enabling interrupts. This avoids any possibility * of cpu migration and using the wrong pcpup. */ ENTRY(tl0_trap) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif and %l5, WSTATE_NORMAL_MASK, %l5 cmp %o0, UT_MAX bge,a,pt %xcc, 2f nop ldx [PCPU(CURTHREAD)], %l6 ldx [%l6 + TD_PROC], %l6 ldx [%l6 + P_MD + MD_UTRAP], %l6 brz,pt %l6, 2f sllx %o0, PTR_SHIFT, %l7 ldx [%l6 + %l7], %l6 brz,pt %l6, 2f andn %l0, TSTATE_CWP_MASK, %l7 ldx [PCB_REG + PCB_NSAVED], %g1 brnz,a,pn %g1, 1f mov T_SPILL, %o0 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: user trap npc=%#lx" , %g1, %g2, %g3, 7, 8, 9) stx %l6, [%g1 + KTR_PARM1] 9: #endif wrpr %l5, %wstate wrpr %l6, %tnpc rdpr %cwp, %l6 wrpr %l6, %l7, %tstate sub %fp, CCFSZ, %sp /* * Need to store %fsr to pass it to the user trap handler. Otherwise, * the ftt field might be zeoed out in a store in another trap or * interrupt. Use the temporary stack for that. */ rd %fprs, %l3 or %l3, FPRS_FEF, %l4 wr %l4, 0, %fprs dec 8, ASP_REG stx %fsr, [ASP_REG] ldx [ASP_REG], %l4 inc 8, ASP_REG wr %l3, 0, %fprs mov %l0, %l5 mov %l1, %l6 mov %l2, %l7 done 1: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: defer user trap npc=%#lx nsaved=%#lx" , %g1, %g2, %g3, 7, 8, 9) stx %l6, [%g1 + KTR_PARM1] ldx [PCB_REG + PCB_NSAVED], %g2 stx %g2, [%g1 + KTR_PARM2] 9: #endif 2: sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] wr %g0, 0, %fprs mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] .Ltl0_trap_reenter: stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] call trap add %sp, CCFSZ + SPOFF, %o0 b,a %xcc, tl0_ret nop END(tl0_trap) /* * void tl0_syscall(u_int type) */ ENTRY(tl0_syscall) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_SYSC CATR(KTR_SYSC, "tl0_syscall: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] wr %g0, 0, %fprs mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] call syscall add %sp, CCFSZ + SPOFF, %o0 b,a %xcc, tl0_ret nop END(tl0_syscall) /* * void tl0_intr(u_int level, u_int mask) */ ENTRY(tl0_intr) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %asr21 and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stw %l3, [%sp + SPOFF + CCFSZ + TF_Y] stb %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stb %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] wr %g0, 0, %fprs mov %o0, %l3 mov T_INTERRUPT, %o1 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] call critical_enter nop SET(cnt+V_INTR, %l1, %l0) ATOMIC_INC_INT(%l0, %l1, %l2) SET(intr_handlers, %l1, %l0) sllx %l3, IH_SHIFT, %l1 ldx [%l0 + %l1], %l1 KASSERT(%l1, "tl0_intr: ih null") call %l1 add %sp, CCFSZ + SPOFF, %o0 call critical_exit nop b,a %xcc, tl0_ret nop END(tl0_intr) ENTRY(tl0_ret) #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: check ast td=%p (%s) pil=%#lx sflag=%#x" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g2 + TD_PROC], %g2 add %g2, P_COMM, %g3 stx %g3, [%g1 + KTR_PARM2] rdpr %pil, %g3 stx %g3, [%g1 + KTR_PARM3] lduw [%g2 + P_SFLAG], %g3 stx %g3, [%g1 + KTR_PARM4] 9: #endif /* * Check for pending asts atomically with returning. We must raise * the pil before checking, and if no asts are found the pil must * remain raised until the retry is executed, or we risk missing asts * caused by interrupts occuring after the test. If the pil is lowered, * as it is when we call ast, the check must be re-executed. */ 1: wrpr %g0, PIL_TICK, %pil ldx [PCPU(CURTHREAD)], %l0 ldx [%l0 + TD_KSE], %l1 lduw [%l1 + KE_FLAGS], %l2 and %l2, KEF_ASTPENDING | KEF_NEEDRESCHED, %l2 brz,a,pt %l2, 2f nop wrpr %g0, 0, %pil call ast add %sp, CCFSZ + SPOFF, %o0 ba,a %xcc, 1b nop /* * Check for windows that were spilled to the pcb and need to be * copied out. This must be the last thing that is done before the * return to usermode. If there are still user windows in the cpu * and we call a nested function after this, which causes them to be * spilled to the pcb, they will not be copied out and the stack will * be inconsistent. */ 2: ldx [PCB_REG + PCB_NSAVED], %l1 mov T_SPILL, %o0 brnz,a,pn %l1, .Ltl0_trap_reenter wrpr %g0, 0, %pil ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l3 ldub [%sp + SPOFF + CCFSZ + TF_FPRS], %l4 ldub [%sp + SPOFF + CCFSZ + TF_WSTATE], %l5 wrpr %g0, PSTATE_NORMAL, %pstate ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 wrpr %g0, PSTATE_ALT, %pstate wrpr %g0, 0, %pil wrpr %l1, 0, %tpc wrpr %l2, 0, %tnpc wr %l3, 0, %y andn %l0, TSTATE_CWP_MASK, %g1 mov %l4, %g2 srlx %l5, WSTATE_OTHER_SHIFT, %g3 wrpr %g3, WSTATE_TRANSITION, %wstate rdpr %otherwin, %o0 wrpr %o0, 0, %canrestore wrpr %g0, 0, %otherwin wrpr %o0, 0, %cleanwin /* * If this instruction causes a fill trap which fails to fill a window * from the user stack, we will resume at tl0_ret_fill_end and call * back into the kernel. */ restore tl0_ret_fill: rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wr %g2, 0, %fprs wrpr %g3, 0, %wstate #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tnpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry tl0_ret_fill_end: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" , %l0, %l1, %l2, 7, 8, 9) rdpr %pstate, %l1 stx %l1, [%l0 + KTR_PARM1] stx %l5, [%l0 + KTR_PARM2] stx %sp, [%l0 + KTR_PARM3] 9: #endif /* * The fill failed and magic has been performed. Call trap again, * which will copyin the window on the user's behalf. */ wrpr %l5, 0, %wstate wrpr %g0, PSTATE_ALT, %pstate mov PCB_REG, %o0 mov PCPU_REG, %o1 wrpr %g0, PSTATE_NORMAL, %pstate mov %o0, PCB_REG mov %o1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate b %xcc, .Ltl0_trap_reenter mov T_FILL_RET, %o0 END(tl0_ret) /* * Kernel trap entry point * * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_int sfsr) * * This is easy because the stack is already setup and the windows don't need * to be split. We build a trapframe and call trap(), the same as above, but * the outs don't need to be saved. */ ENTRY(tl1_trap) sub %sp, TF_SIZEOF, %sp rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] stw %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stw %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] mov PCB_REG, %l4 mov PCPU_REG, %l5 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] mov %l4, PCB_REG mov %l5, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate call trap add %sp, CCFSZ + SPOFF, %o0 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 ldub [%sp + SPOFF + CCFSZ + TF_PIL], %l3 lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 wrpr %l3, 0, %pil wr %l4, 0, %y restore wrpr %g0, 2, %tl rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_trap: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tstate, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry END(tl1_trap) /* * void tl1_intr(u_int level, u_int mask) */ ENTRY(tl1_intr) sub %sp, TF_SIZEOF, %sp rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%p level=%#lx pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %asr21 wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stb %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stw %l4, [%sp + SPOFF + CCFSZ + TF_Y] mov %o0, %l7 mov T_INTERRUPT | T_KERNEL, %o1 stw %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stw %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] mov PCB_REG, %l4 mov PCPU_REG, %l5 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] mov %l4, PCB_REG mov %l5, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate call critical_enter nop SET(cnt+V_INTR, %l5, %l4) ATOMIC_INC_INT(%l4, %l5, %l6) SET(intr_handlers, %l5, %l4) sllx %l7, IH_SHIFT, %l5 ldx [%l4 + %l5], %l5 KASSERT(%l5, "tl1_intr: ih null") call %l5 add %sp, CCFSZ + SPOFF, %o0 call critical_exit nop lduw [%sp + SPOFF + CCFSZ + TF_Y], %l4 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 wrpr %l3, 0, %pil wr %l4, 0, %y restore wrpr %g0, 2, %tl rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tstate, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry END(tl1_intr) /* * Freshly forked processes come here when switched to for the first time. * The arguments to fork_exit() have been setup in the locals, we must move * them to the outs. */ ENTRY(fork_trampoline) #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g2 + TD_PROC], %g2 add %g2, P_COMM, %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %cwp, %g2 stx %g2, [%g1 + KTR_PARM3] 9: #endif mov %l0, %o0 mov %l1, %o1 call fork_exit mov %l2, %o2 b,a %xcc, tl0_ret nop END(fork_trampoline) diff --git a/sys/sparc64/sparc64/genassym.c b/sys/sparc64/sparc64/genassym.c index 5a5665683d11..b578c65806c4 100644 --- a/sys/sparc64/sparc64/genassym.c +++ b/sys/sparc64/sparc64/genassym.c @@ -1,294 +1,307 @@ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)genassym.c 5.11 (Berkeley) 5/10/91 * $FreeBSD$ */ #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include ASSYM(KERNBASE, KERNBASE); ASSYM(EFAULT, EFAULT); ASSYM(ENAMETOOLONG, ENAMETOOLONG); ASSYM(KSTACK_PAGES, KSTACK_PAGES); ASSYM(KSTACK_GUARD_PAGES, KSTACK_GUARD_PAGES); ASSYM(PCPU_PAGES, PCPU_PAGES); ASSYM(UAREA_PAGES, UAREA_PAGES); ASSYM(PAGE_SIZE, PAGE_SIZE); ASSYM(PIL_TICK, PIL_TICK); ASSYM(FPRS_DL, FPRS_DL); ASSYM(FPRS_DU, FPRS_DU); ASSYM(FPRS_FEF, FPRS_FEF); ASSYM(LSU_VW, LSU_VW); ASSYM(TAR_VPN_SHIFT, TAR_VPN_SHIFT); +ASSYM(TLB_DAR_SLOT_SHIFT, TLB_DAR_SLOT_SHIFT); ASSYM(TLB_DEMAP_NUCLEUS, TLB_DEMAP_NUCLEUS); ASSYM(TLB_DEMAP_PRIMARY, TLB_DEMAP_PRIMARY); ASSYM(TLB_DEMAP_CONTEXT, TLB_DEMAP_CONTEXT); ASSYM(TLB_DEMAP_PAGE, TLB_DEMAP_PAGE); +ASSYM(TLB_DIRECT_MASK, TLB_DIRECT_MASK); +ASSYM(TLB_DIRECT_UNCACHEABLE, TLB_DIRECT_UNCACHEABLE); +ASSYM(TLB_DIRECT_SHIFT, TLB_DIRECT_SHIFT); ASSYM(TSB_BUCKET_ADDRESS_BITS, TSB_BUCKET_ADDRESS_BITS); ASSYM(TSB_BUCKET_SHIFT, TSB_BUCKET_SHIFT); ASSYM(TSB_KERNEL_MASK, TSB_KERNEL_MASK); ASSYM(INT_SHIFT, INT_SHIFT); ASSYM(PTR_SHIFT, PTR_SHIFT); ASSYM(PAGE_SHIFT, PAGE_SHIFT); ASSYM(PAGE_MASK, PAGE_MASK); ASSYM(CPU_CLKSYNC, CPU_CLKSYNC); ASSYM(CPU_INIT, CPU_INIT); ASSYM(CSA_MID, offsetof(struct cpu_start_args, csa_mid)); ASSYM(CSA_PCPU, offsetof(struct cpu_start_args, csa_pcpu)); ASSYM(CSA_STATE, offsetof(struct cpu_start_args, csa_state)); ASSYM(CSA_TICK, offsetof(struct cpu_start_args, csa_tick)); ASSYM(CSA_VER, offsetof(struct cpu_start_args, csa_ver)); ASSYM(CSA_TTES, offsetof(struct cpu_start_args, csa_ttes)); ASSYM(DC_TAG_SHIFT, DC_TAG_SHIFT); ASSYM(DC_TAG_MASK, DC_TAG_MASK); ASSYM(DC_VALID_SHIFT, DC_VALID_SHIFT); ASSYM(DC_VALID_MASK, DC_VALID_MASK); ASSYM(IC_TAG_SHIFT, IC_TAG_SHIFT); ASSYM(IC_TAG_MASK, IC_TAG_MASK); ASSYM(IC_VALID_SHIFT, IC_VALID_SHIFT); ASSYM(IC_VALID_MASK, IC_VALID_MASK); ASSYM(DC_SIZE, offsetof(struct cacheinfo, dc_size)); ASSYM(DC_LINESIZE, offsetof(struct cacheinfo, dc_linesize)); ASSYM(IC_SIZE, offsetof(struct cacheinfo, ic_size)); ASSYM(IC_LINESIZE, offsetof(struct cacheinfo, ic_linesize)); ASSYM(ICA_PA, offsetof(struct ipi_cache_args, ica_pa)); +ASSYM(GMON_PROF_OFF, GMON_PROF_OFF); +ASSYM(GMON_PROF_HIRES, GMON_PROF_HIRES); +ASSYM(GM_STATE, offsetof(struct gmonparam, state)); + ASSYM(KTR_PROC, KTR_PROC); ASSYM(KTR_TRAP, KTR_TRAP); ASSYM(KTR_SMP, KTR_SMP); ASSYM(KTR_SYSC, KTR_SYSC); ASSYM(KTR_INTR, KTR_INTR); ASSYM(KTR_CT1, KTR_CT1); ASSYM(KTR_CT2, KTR_CT2); ASSYM(KTR_CT3, KTR_CT3); ASSYM(KTR_CT4, KTR_CT4); ASSYM(KTR_CT5, KTR_CT5); ASSYM(KTR_CT6, KTR_CT6); ASSYM(KTR_CT7, KTR_CT7); ASSYM(KTR_CT8, KTR_CT8); ASSYM(KTR_SIZEOF, sizeof(struct ktr_entry)); ASSYM(KTR_LINE, offsetof(struct ktr_entry, ktr_line)); ASSYM(KTR_FILE, offsetof(struct ktr_entry, ktr_file)); ASSYM(KTR_DESC, offsetof(struct ktr_entry, ktr_desc)); ASSYM(KTR_CPU, offsetof(struct ktr_entry, ktr_cpu)); ASSYM(KTR_TIMESTAMP, offsetof(struct ktr_entry, ktr_timestamp)); ASSYM(KTR_PARM1, offsetof(struct ktr_entry, ktr_parms[0])); ASSYM(KTR_PARM2, offsetof(struct ktr_entry, ktr_parms[1])); ASSYM(KTR_PARM3, offsetof(struct ktr_entry, ktr_parms[2])); ASSYM(KTR_PARM4, offsetof(struct ktr_entry, ktr_parms[3])); ASSYM(KTR_PARM5, offsetof(struct ktr_entry, ktr_parms[4])); ASSYM(KTR_PARM6, offsetof(struct ktr_entry, ktr_parms[5])); ASSYM(TTE_VPN, offsetof(struct tte, tte_vpn)); ASSYM(TTE_DATA, offsetof(struct tte, tte_data)); ASSYM(TTE_SHIFT, TTE_SHIFT); ASSYM(TD_EXEC, TD_EXEC); ASSYM(TD_REF, TD_REF); ASSYM(TD_SW, TD_SW); +ASSYM(TD_V, TD_V); +ASSYM(TD_8K, TD_8K); +ASSYM(TD_CP, TD_CP); +ASSYM(TD_CV, TD_CV); ASSYM(TD_L, TD_L); ASSYM(TD_W, TD_W); ASSYM(V_INTR, offsetof(struct vmmeter, v_intr)); ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(PC_CPUMASK, offsetof(struct pcpu, pc_cpumask)); ASSYM(PC_IRHEAD, offsetof(struct pcpu, pc_irhead)); ASSYM(PC_IRTAIL, offsetof(struct pcpu, pc_irtail)); ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree)); ASSYM(PC_MID, offsetof(struct pcpu, pc_mid)); ASSYM(PC_TLB_CTX, offsetof(struct pcpu, pc_tlb_ctx)); ASSYM(PC_TLB_CTX_MAX, offsetof(struct pcpu, pc_tlb_ctx_max)); ASSYM(PC_TLB_CTX_MIN, offsetof(struct pcpu, pc_tlb_ctx_min)); ASSYM(PC_VMSPACE, offsetof(struct pcpu, pc_vmspace)); ASSYM(PC_SIZEOF, sizeof(struct pcpu)); ASSYM(IH_SHIFT, IH_SHIFT); ASSYM(IRSR_BUSY, IRSR_BUSY); ASSYM(IR_NEXT, offsetof(struct intr_request, ir_next)); ASSYM(IR_FUNC, offsetof(struct intr_request, ir_func)); ASSYM(IR_ARG, offsetof(struct intr_request, ir_arg)); ASSYM(IR_PRI, offsetof(struct intr_request, ir_pri)); ASSYM(IR_VEC, offsetof(struct intr_request, ir_vec)); ASSYM(ITA_MASK, offsetof(struct ipi_tlb_args, ita_mask)); ASSYM(ITA_PMAP, offsetof(struct ipi_tlb_args, ita_pmap)); ASSYM(ITA_START, offsetof(struct ipi_tlb_args, ita_start)); ASSYM(ITA_END, offsetof(struct ipi_tlb_args, ita_end)); ASSYM(ITA_VA, offsetof(struct ipi_tlb_args, ita_va)); ASSYM(IV_SHIFT, IV_SHIFT); ASSYM(IV_FUNC, offsetof(struct intr_vector, iv_func)); ASSYM(IV_ARG, offsetof(struct intr_vector, iv_arg)); ASSYM(IV_PRI, offsetof(struct intr_vector, iv_pri)); ASSYM(IV_MAX, IV_MAX); ASSYM(KEF_ASTPENDING, KEF_ASTPENDING); ASSYM(KEF_NEEDRESCHED, KEF_NEEDRESCHED); ASSYM(MD_UTRAP, offsetof(struct mdproc, md_utrap)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(P_COMM, offsetof(struct proc, p_comm)); ASSYM(P_MD, offsetof(struct proc, p_md)); ASSYM(P_PID, offsetof(struct proc, p_pid)); ASSYM(P_SFLAG, offsetof(struct proc, p_sflag)); ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); ASSYM(RW_SHIFT, RW_SHIFT); ASSYM(KE_FLAGS, offsetof(struct kse, ke_flags)); ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); ASSYM(TD_KSE, offsetof(struct thread, td_kse)); ASSYM(TD_KSTACK, offsetof(struct thread, td_kstack)); ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); ASSYM(TD_PROC, offsetof(struct thread, td_proc)); ASSYM(PCB_SIZEOF, sizeof(struct pcb)); ASSYM(PCB_FPSTATE, offsetof(struct pcb, pcb_fpstate)); ASSYM(PCB_FP, offsetof(struct pcb, pcb_fp)); ASSYM(PCB_PC, offsetof(struct pcb, pcb_pc)); ASSYM(PCB_Y, offsetof(struct pcb, pcb_pc)); ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); ASSYM(PCB_NSAVED, offsetof(struct pcb, pcb_nsaved)); ASSYM(PCB_RWSP, offsetof(struct pcb, pcb_rwsp)); ASSYM(PCB_RW, offsetof(struct pcb, pcb_rw)); ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); ASSYM(PM_CONTEXT, offsetof(struct pmap, pm_context)); ASSYM(PM_TSB, offsetof(struct pmap, pm_tsb)); ASSYM(FP_FB0, offsetof(struct fpstate, fp_fb[0])); ASSYM(FP_FB1, offsetof(struct fpstate, fp_fb[1])); ASSYM(FP_FB2, offsetof(struct fpstate, fp_fb[2])); ASSYM(FP_FB3, offsetof(struct fpstate, fp_fb[3])); ASSYM(CCFSZ, sizeof(struct frame)); ASSYM(SPOFF, SPOFF); ASSYM(SF_UC, offsetof(struct sigframe, sf_uc)); ASSYM(_JB_FP, offsetof(struct _jmp_buf, _jb[_JB_FP])); ASSYM(_JB_PC, offsetof(struct _jmp_buf, _jb[_JB_PC])); ASSYM(_JB_SP, offsetof(struct _jmp_buf, _jb[_JB_SP])); ASSYM(_JB_SIGFLAG, offsetof(struct _jmp_buf, _jb[_JB_SIGFLAG])); ASSYM(_JB_SIGMASK, offsetof(struct _jmp_buf, _jb[_JB_SIGMASK])); ASSYM(TF_G0, offsetof(struct trapframe, tf_global[0])); ASSYM(TF_G1, offsetof(struct trapframe, tf_global[1])); ASSYM(TF_G2, offsetof(struct trapframe, tf_global[2])); ASSYM(TF_G3, offsetof(struct trapframe, tf_global[3])); ASSYM(TF_G4, offsetof(struct trapframe, tf_global[4])); ASSYM(TF_G5, offsetof(struct trapframe, tf_global[5])); ASSYM(TF_G6, offsetof(struct trapframe, tf_global[6])); ASSYM(TF_G7, offsetof(struct trapframe, tf_global[7])); ASSYM(TF_O0, offsetof(struct trapframe, tf_out[0])); ASSYM(TF_O1, offsetof(struct trapframe, tf_out[1])); ASSYM(TF_O2, offsetof(struct trapframe, tf_out[2])); ASSYM(TF_O3, offsetof(struct trapframe, tf_out[3])); ASSYM(TF_O4, offsetof(struct trapframe, tf_out[4])); ASSYM(TF_O5, offsetof(struct trapframe, tf_out[5])); ASSYM(TF_O6, offsetof(struct trapframe, tf_out[6])); ASSYM(TF_O7, offsetof(struct trapframe, tf_out[7])); ASSYM(TF_FSR, offsetof(struct trapframe, tf_fsr)); ASSYM(TF_SFAR, offsetof(struct trapframe, tf_sfar)); ASSYM(TF_LEVEL, offsetof(struct trapframe, tf_level)); ASSYM(TF_TAR, offsetof(struct trapframe, tf_tar)); ASSYM(TF_TNPC, offsetof(struct trapframe, tf_tnpc)); ASSYM(TF_TPC, offsetof(struct trapframe, tf_tpc)); ASSYM(TF_TSTATE, offsetof(struct trapframe, tf_tstate)); ASSYM(TF_SFSR, offsetof(struct trapframe, tf_sfsr)); ASSYM(TF_TYPE, offsetof(struct trapframe, tf_type)); ASSYM(TF_Y, offsetof(struct trapframe, tf_y)); ASSYM(TF_FPRS, offsetof(struct trapframe, tf_fprs)); ASSYM(TF_PIL, offsetof(struct trapframe, tf_pil)); ASSYM(TF_WSTATE, offsetof(struct trapframe, tf_wstate)); ASSYM(TF_SIZEOF, sizeof(struct trapframe)); ASSYM(UT_MAX, UT_MAX); diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 8a8db4f4522a..3067d12a6042 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -1,1701 +1,1725 @@ /* * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 * $FreeBSD$ */ /* * Manages physical address maps. * * In addition to hardware address maps, this module is called upon to * provide software-use-only maps which may or may not be stored in the * same form as hardware maps. These pseudo-maps are used to store * intermediate results from copy operations to and from address spaces. * * Since the information managed by this module is also stored by the * logical address mapping module, this module may throw away valid virtual * to physical mappings at almost any time. However, invalidations of * mappings must be done as requested. * * In order to cope with hardware architectures which make virtual to * physical map invalidates expensive, this module may delay invalidate * reduced protection operations until such time as they are actually * necessary. This module is given full information as to which processors * are currently using which maps, and to when physical maps must be made * correct. */ #include "opt_msgbuf.h" #include "opt_pmap.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define PMAP_DEBUG #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif struct mem_region { vm_offset_t mr_start; vm_offset_t mr_size; }; struct ofw_map { vm_offset_t om_start; vm_offset_t om_size; u_long om_tte; }; /* * Virtual and physical address of message buffer. */ struct msgbuf *msgbufp; vm_offset_t msgbuf_phys; /* * Physical addresses of first and last available physical page. */ vm_offset_t avail_start; vm_offset_t avail_end; int pmap_pagedaemon_waken; /* * Map of physical memory reagions. */ vm_offset_t phys_avail[128]; static struct mem_region mra[128]; static struct ofw_map translations[128]; static int translations_size; /* * First and last available kernel virtual addresses. */ vm_offset_t virtual_avail; vm_offset_t virtual_end; vm_offset_t kernel_vm_end; /* * Kernel pmap. */ struct pmap kernel_pmap_store; static boolean_t pmap_initialized = FALSE; /* * Allocate physical memory for use in pmap_bootstrap. */ static vm_offset_t pmap_bootstrap_alloc(vm_size_t size); +static vm_offset_t pmap_map_direct(vm_page_t m); + /* * If user pmap is processed with pmap_remove and with pmap_remove and the * resident count drops to 0, there are no more pages to remove, so we * need not continue. */ #define PMAP_REMOVE_DONE(pm) \ ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) /* * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() * and pmap_protect() instead of trying each virtual address. */ #define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) #ifdef PMAP_STATS static long pmap_enter_nupdate; static long pmap_enter_nreplace; static long pmap_enter_nnew; static long pmap_ncache_enter; static long pmap_ncache_enter_nc; static long pmap_niflush; SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, "Statistics"); SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nupdate, CTLFLAG_RD, &pmap_enter_nupdate, 0, "Number of pmap_enter() updates"); SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nreplace, CTLFLAG_RD, &pmap_enter_nreplace, 0, "Number of pmap_enter() replacements"); SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nnew, CTLFLAG_RD, &pmap_enter_nnew, 0, "Number of pmap_enter() additions"); SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_ncache_enter, CTLFLAG_RD, &pmap_ncache_enter, 0, "Number of pmap_cache_enter() calls"); SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_ncache_enter_nc, CTLFLAG_RD, &pmap_ncache_enter_nc, 0, "Number of pmap_cache_enter() nc"); SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_niflush, CTLFLAG_RD, &pmap_niflush, 0, "Number of pmap I$ flushes"); #define PMAP_STATS_INC(var) atomic_add_long(&var, 1) #else #define PMAP_STATS_INC(var) #endif /* * Quick sort callout for comparing memory regions. */ static int mr_cmp(const void *a, const void *b); static int om_cmp(const void *a, const void *b); static int mr_cmp(const void *a, const void *b) { const struct mem_region *mra; const struct mem_region *mrb; mra = a; mrb = b; if (mra->mr_start < mrb->mr_start) return (-1); else if (mra->mr_start > mrb->mr_start) return (1); else return (0); } static int om_cmp(const void *a, const void *b) { const struct ofw_map *oma; const struct ofw_map *omb; oma = a; omb = b; if (oma->om_start < omb->om_start) return (-1); else if (oma->om_start > omb->om_start) return (1); else return (0); } /* * Bootstrap the system enough to run with virtual memory. */ void pmap_bootstrap(vm_offset_t ekva) { struct pmap *pm; struct tte *tp; vm_offset_t off; vm_offset_t pa; vm_offset_t va; vm_size_t physsz; ihandle_t pmem; ihandle_t vmem; int sz; int i; int j; /* * Set the start and end of kva. The kernel is loaded at the first * available 4 meg super page, so round up to the end of the page. */ virtual_avail = roundup2(ekva, PAGE_SIZE_4M); virtual_end = VM_MAX_KERNEL_ADDRESS; /* * Find out what physical memory is available from the prom and * initialize the phys_avail array. This must be done before * pmap_bootstrap_alloc is called. */ if ((pmem = OF_finddevice("/memory")) == -1) panic("pmap_bootstrap: finddevice /memory"); if ((sz = OF_getproplen(pmem, "available")) == -1) panic("pmap_bootstrap: getproplen /memory/available"); if (sizeof(phys_avail) < sz) panic("pmap_bootstrap: phys_avail too small"); if (sizeof(mra) < sz) panic("pmap_bootstrap: mra too small"); bzero(mra, sz); if (OF_getprop(pmem, "available", mra, sz) == -1) panic("pmap_bootstrap: getprop /memory/available"); sz /= sizeof(*mra); CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); qsort(mra, sz, sizeof (*mra), mr_cmp); physsz = 0; for (i = 0, j = 0; i < sz; i++, j += 2) { CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, mra[i].mr_size); phys_avail[j] = mra[i].mr_start; phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; physsz += mra[i].mr_size; } physmem = btoc(physsz); /* * Allocate the kernel tsb and lock it in the tlb. */ pa = pmap_bootstrap_alloc(KVA_PAGES * PAGE_SIZE_4M); if (pa & PAGE_MASK_4M) panic("pmap_bootstrap: tsb unaligned\n"); tsb_kernel_phys = pa; tsb_kernel = (struct tte *)virtual_avail; virtual_avail += KVA_PAGES * PAGE_SIZE_4M; pmap_map_tsb(); bzero(tsb_kernel, KVA_PAGES * PAGE_SIZE_4M); /* * Enter fake 8k pages for the 4MB kernel pages, so that * pmap_kextract() will work for them. */ for (i = 0; i < kernel_tlb_slots; i++) { pa = kernel_tlbs[i].te_pa; va = kernel_tlbs[i].te_va; for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { tp = tsb_kvtotte(va + off); tp->tte_vpn = TV_VPN(va + off); tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; } } /* * Allocate a kernel stack with guard page for thread0 and map it into * the kernel tsb. */ pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); kstack0_phys = pa; kstack0 = virtual_avail + (KSTACK_GUARD_PAGES * PAGE_SIZE); virtual_avail += (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE; for (i = 0; i < KSTACK_PAGES; i++) { pa = kstack0_phys + i * PAGE_SIZE; va = kstack0 + i * PAGE_SIZE; tp = tsb_kvtotte(va); tp->tte_vpn = TV_VPN(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; } /* * Allocate the message buffer. */ msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE); /* * Add the prom mappings to the kernel tsb. */ if ((vmem = OF_finddevice("/virtual-memory")) == -1) panic("pmap_bootstrap: finddevice /virtual-memory"); if ((sz = OF_getproplen(vmem, "translations")) == -1) panic("pmap_bootstrap: getproplen translations"); if (sizeof(translations) < sz) panic("pmap_bootstrap: translations too small"); bzero(translations, sz); if (OF_getprop(vmem, "translations", translations, sz) == -1) panic("pmap_bootstrap: getprop /virtual-memory/translations"); sz /= sizeof(*translations); translations_size = sz; CTR0(KTR_PMAP, "pmap_bootstrap: translations"); qsort(translations, sz, sizeof (*translations), om_cmp); for (i = 0; i < sz; i++) { CTR3(KTR_PMAP, "translation: start=%#lx size=%#lx tte=%#lx", translations[i].om_start, translations[i].om_size, translations[i].om_tte); if (translations[i].om_start < VM_MIN_PROM_ADDRESS || translations[i].om_start > VM_MAX_PROM_ADDRESS) continue; for (off = 0; off < translations[i].om_size; off += PAGE_SIZE) { va = translations[i].om_start + off; tp = tsb_kvtotte(va); tp->tte_vpn = TV_VPN(va); tp->tte_data = translations[i].om_tte + off; } } /* * Calculate the first and last available physical addresses. */ avail_start = phys_avail[0]; for (i = 0; phys_avail[i + 2] != 0; i += 2) ; avail_end = phys_avail[i + 1]; Maxmem = sparc64_btop(avail_end); /* * Allocate virtual address space for the message buffer. */ msgbufp = (struct msgbuf *)virtual_avail; virtual_avail += round_page(MSGBUF_SIZE); /* * Initialize the kernel pmap (which is statically allocated). */ pm = kernel_pmap; for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = TLB_CTX_KERNEL; pm->pm_active = ~0; /* XXX flush all non-locked tlb entries */ } void pmap_map_tsb(void) { vm_offset_t va; vm_offset_t pa; u_long data; u_long s; int i; s = intr_disable(); /* * Map the 4mb tsb pages. */ for (i = 0; i < KVA_PAGES; i++) { va = (vm_offset_t)tsb_kernel + i * PAGE_SIZE_4M; pa = tsb_kernel_phys + i * PAGE_SIZE_4M; /* XXX - cheetah */ data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | TLB_TAR_CTX(TLB_CTX_KERNEL)); stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); } /* * Load the tsb registers. */ stxa(AA_DMMU_TSB, ASI_DMMU, (vm_offset_t)tsb_kernel); stxa(AA_IMMU_TSB, ASI_IMMU, (vm_offset_t)tsb_kernel); membar(Sync); flush(tsb_kernel); /* * Set the secondary context to be the kernel context (needed for * fp block operations in the kernel and the cache code). */ stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); membar(Sync); intr_restore(s); } /* * Allocate a physical page of memory directly from the phys_avail map. * Can only be called from pmap_bootstrap before avail start and end are * calculated. */ static vm_offset_t pmap_bootstrap_alloc(vm_size_t size) { vm_offset_t pa; int i; size = round_page(size); for (i = 0; phys_avail[i + 1] != 0; i += 2) { if (phys_avail[i + 1] - phys_avail[i] < size) continue; pa = phys_avail[i]; phys_avail[i] += size; return (pa); } panic("pmap_bootstrap_alloc"); } void pmap_context_rollover(void) { u_long data; u_long tag; int i; mtx_assert(&sched_lock, MA_OWNED); CTR0(KTR_PMAP, "pmap_context_rollover"); for (i = 0; i < tlb_slot_count; i++) { /* XXX - cheetah */ data = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG); tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); if ((data & TD_V) != 0 && (data & TD_L) == 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) stxa_sync(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG, 0); data = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG); tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); if ((data & TD_V) != 0 && (data & TD_L) == 0 && TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) stxa_sync(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, 0); } PCPU_SET(tlb_ctx, PCPU_GET(tlb_ctx_min)); } static __inline u_int pmap_context_alloc(void) { u_int context; mtx_assert(&sched_lock, MA_OWNED); context = PCPU_GET(tlb_ctx); if (context + 1 == PCPU_GET(tlb_ctx_max)) pmap_context_rollover(); else PCPU_SET(tlb_ctx, context + 1); return (context); } /* * Initialize the pmap module. */ void pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) { vm_offset_t addr; vm_size_t size; int result; int i; for (i = 0; i < vm_page_array_size; i++) { vm_page_t m; m = &vm_page_array[i]; STAILQ_INIT(&m->md.tte_list); m->md.flags = 0; + m->md.color = 0; } for (i = 0; i < translations_size; i++) { addr = translations[i].om_start; size = translations[i].om_size; if (addr < 0xf0000000) /* XXX */ continue; result = vm_map_find(kernel_map, NULL, 0, &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0); if (result != KERN_SUCCESS || addr != translations[i].om_start) panic("pmap_init: vm_map_find"); } pmap_initialized = TRUE; } /* * Initialize the address space (zone) for the pv_entries. Set a * high water mark so that the system can recover from excessive * numbers of pv entries. */ void pmap_init2(void) { } /* * Extract the physical page address associated with the given * map/virtual_address pair. */ vm_offset_t pmap_extract(pmap_t pm, vm_offset_t va) { struct tte *tp; if (pm == kernel_pmap) return (pmap_kextract(va)); tp = tsb_tte_lookup(pm, va); if (tp == NULL) return (0); else return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); } /* * Extract the physical page address associated with the given kernel virtual * address. */ vm_offset_t pmap_kextract(vm_offset_t va) { struct tte *tp; tp = tsb_kvtotte(va); if ((tp->tte_data & TD_V) == 0) return (0); return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); } int pmap_cache_enter(vm_page_t m, vm_offset_t va) { struct tte *tp; + int color; int c, i; CTR2(KTR_PMAP, "pmap_cache_enter: m=%p va=%#lx", m, va); PMAP_STATS_INC(pmap_ncache_enter); - for (i = 0, c = 0; i < DCACHE_COLORS; i++) { - if (i != DCACHE_COLOR(va) && m->md.colors[i] != 0) + color = DCACHE_COLOR(va); + m->md.colors[color]++; + if (m->md.color == color) { + CTR0(KTR_PMAP, "pmap_cache_enter: cacheable"); + return (1); + } + for (c = 0, i = 0; i < DCACHE_COLORS; i++) { + if (m->md.colors[i] != 0) c++; } - m->md.colors[DCACHE_COLOR(va)]++; - if (c == 0) { + if (c == 1) { + m->md.color = color; + dcache_page_inval(VM_PAGE_TO_PHYS(m)); CTR0(KTR_PMAP, "pmap_cache_enter: cacheable"); return (1); } PMAP_STATS_INC(pmap_ncache_enter_nc); - if ((m->md.flags & PG_UNCACHEABLE) != 0) { + if (m->md.color == -1) { CTR0(KTR_PMAP, "pmap_cache_enter: already uncacheable"); return (0); } CTR0(KTR_PMAP, "pmap_cache_enter: marking uncacheable"); STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { tp->tte_data &= ~TD_CV; tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } dcache_page_inval(VM_PAGE_TO_PHYS(m)); - m->md.flags |= PG_UNCACHEABLE; + m->md.color = -1; return (0); } void pmap_cache_remove(vm_page_t m, vm_offset_t va) { struct tte *tp; + int color; int c, i; CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, m->md.colors[DCACHE_COLOR(va)]); KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, ("pmap_cache_remove: no mappings %d <= 0", m->md.colors[DCACHE_COLOR(va)])); - m->md.colors[DCACHE_COLOR(va)]--; - for (i = 0, c = 0; i < DCACHE_COLORS; i++) { + color = DCACHE_COLOR(va); + m->md.colors[color]--; + if (m->md.color != -1 || m->md.colors[color] != 0) + return; + for (c = 0, i = 0; i < DCACHE_COLORS; i++) { if (m->md.colors[i] != 0) c++; } - if (c > 1 || (m->md.flags & PG_UNCACHEABLE) == 0) + if (c > 1) return; STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { tp->tte_data |= TD_CV; tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } - m->md.flags &= ~PG_UNCACHEABLE; + m->md.color = color; } /* * Map a wired page into kernel virtual address space. */ void pmap_kenter(vm_offset_t va, vm_offset_t pa) { vm_offset_t ova; struct tte *tp; vm_page_t om; vm_page_t m; u_long data; tp = tsb_kvtotte(va); m = PHYS_TO_VM_PAGE(pa); CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", va, pa, tp, tp->tte_data); if ((tp->tte_data & TD_V) != 0) { om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); ova = TTE_GET_VA(tp); STAILQ_REMOVE(&om->md.tte_list, tp, tte, tte_link); pmap_cache_remove(om, ova); if (va != ova) tlb_page_demap(kernel_pmap, ova); } data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_P | TD_W; if (pmap_cache_enter(m, va) != 0) data |= TD_CV; tp->tte_vpn = TV_VPN(va); tp->tte_data = data; STAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); tp->tte_pmap = kernel_pmap; } /* * Map a wired page into kernel virtual address space. This additionally * takes a flag argument wich is or'ed to the TTE data. This is used by * bus_space_map(). * NOTE: if the mapping is non-cacheable, it's the caller's responsibility * to flush entries that might still be in the cache, if applicable. */ void pmap_kenter_flags(vm_offset_t va, vm_offset_t pa, u_long flags) { struct tte *tp; tp = tsb_kvtotte(va); CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", va, pa, tp, tp->tte_data); tp->tte_vpn = TV_VPN(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; } /* * Make a temporary mapping for a physical address. This is only intended * to be used for panic dumps. */ void * pmap_kenter_temporary(vm_offset_t pa, int i) { TODO; } /* * Remove a wired page from kernel virtual address space. */ void pmap_kremove(vm_offset_t va) { struct tte *tp; vm_page_t m; tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); pmap_cache_remove(m, va); TTE_ZERO(tp); } /* * Inverse of pmap_kenter_flags, used by bus_space_unmap(). */ void pmap_kremove_flags(vm_offset_t va) { struct tte *tp; tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); TTE_ZERO(tp); } /* * Map a range of physical addresses into kernel virtual address space. * * The value passed in *virt is a suggested virtual address for the mapping. * Architectures which can support a direct-mapped physical to virtual region * can return the appropriate address within that region, leaving '*virt' * unchanged. We cannot and therefore do not; *virt is updated with the * first usable address after the mapped region. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_offset_t pa_start, vm_offset_t pa_end, int prot) { struct tte *tp; vm_offset_t sva; vm_offset_t va; vm_offset_t pa; pa = pa_start; sva = *virt; va = sva; for (; pa < pa_end; pa += PAGE_SIZE, va += PAGE_SIZE) { tp = tsb_kvtotte(va); tp->tte_vpn = TV_VPN(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; } tlb_range_demap(kernel_pmap, sva, sva + (pa_end - pa_start) - 1); *virt = va; return (sva); } +static vm_offset_t +pmap_map_direct(vm_page_t m) +{ + vm_offset_t pa; + vm_offset_t va; + + pa = VM_PAGE_TO_PHYS(m); + if (m->md.color == -1) + va = TLB_DIRECT_MASK | pa | TLB_DIRECT_UNCACHEABLE; + else + va = TLB_DIRECT_MASK | pa | + (m->md.color << TLB_DIRECT_COLOR_SHIFT); + return (va << TLB_DIRECT_SHIFT); +} + /* * Map a list of wired pages into kernel virtual address space. This is * intended for temporary mappings which do not need page modification or * references recorded. Existing mappings in the region are overwritten. */ void pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) { vm_offset_t va; int i; va = sva; for (i = 0; i < count; i++, va += PAGE_SIZE) pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); } /* * As above, but take an additional flags argument and call * pmap_kenter_flags(). */ void pmap_qenter_flags(vm_offset_t sva, vm_page_t *m, int count, u_long fl) { vm_offset_t va; int i; va = sva; for (i = 0; i < count; i++, va += PAGE_SIZE) pmap_kenter_flags(va, VM_PAGE_TO_PHYS(m[i]), fl); tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); } /* * Remove page mappings from kernel virtual address space. Intended for * temporary mappings entered by pmap_qenter. */ void pmap_qremove(vm_offset_t sva, int count) { vm_offset_t va; int i; va = sva; for (i = 0; i < count; i++, va += PAGE_SIZE) pmap_kremove(va); tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); } /* * Create the kernel stack and pcb for a new thread. * This routine directly affects the fork perf for a process and * create performance for a thread. */ void pmap_new_thread(struct thread *td) { vm_page_t ma[KSTACK_PAGES]; vm_object_t ksobj; vm_offset_t ks; vm_page_t m; u_int i; /* * Allocate object for the kstack, */ ksobj = vm_object_allocate(OBJT_DEFAULT, KSTACK_PAGES); td->td_kstack_obj = ksobj; /* * Get a kernel virtual address for the kstack for this thread. */ ks = kmem_alloc_nofault(kernel_map, (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE); if (ks == 0) panic("pmap_new_thread: kstack allocation failed"); if (KSTACK_GUARD_PAGES != 0) { tlb_page_demap(kernel_pmap, ks); ks += KSTACK_GUARD_PAGES * PAGE_SIZE; } td->td_kstack = ks; for (i = 0; i < KSTACK_PAGES; i++) { /* * Get a kernel stack page. */ m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); ma[i] = m; /* * Wire the page. */ m->wire_count++; cnt.v_wire_count++; vm_page_wakeup(m); vm_page_flag_clear(m, PG_ZERO); vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); m->valid = VM_PAGE_BITS_ALL; } /* * Enter the page into the kernel address space. */ pmap_qenter(ks, ma, KSTACK_PAGES); } /* * Dispose the kernel stack for a thread that has exited. * This routine directly impacts the exit perf of a process and thread. */ void pmap_dispose_thread(struct thread *td) { vm_object_t ksobj; vm_offset_t ks; vm_page_t m; int i; ksobj = td->td_kstack_obj; ks = td->td_kstack; for (i = 0; i < KSTACK_PAGES; i++) { m = vm_page_lookup(ksobj, i); if (m == NULL) panic("pmap_dispose_thread: kstack already missing?"); vm_page_lock_queues(); vm_page_busy(m); vm_page_unwire(m, 0); vm_page_free(m); vm_page_unlock_queues(); } pmap_qremove(ks, KSTACK_PAGES); kmem_free(kernel_map, ks - (KSTACK_GUARD_PAGES * PAGE_SIZE), (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE); vm_object_deallocate(ksobj); } /* * Allow the kernel stack for a thread to be prejudicially paged out. */ void pmap_swapout_thread(struct thread *td) { vm_object_t ksobj; vm_offset_t ks; vm_page_t m; int i; ksobj = td->td_kstack_obj; ks = (vm_offset_t)td->td_kstack; for (i = 0; i < KSTACK_PAGES; i++) { m = vm_page_lookup(ksobj, i); if (m == NULL) panic("pmap_swapout_thread: kstack already missing?"); vm_page_lock_queues(); vm_page_dirty(m); vm_page_unwire(m, 0); vm_page_unlock_queues(); } pmap_qremove(ks, KSTACK_PAGES); } /* * Bring the kernel stack for a specified thread back in. */ void pmap_swapin_thread(struct thread *td) { vm_page_t ma[KSTACK_PAGES]; vm_object_t ksobj; vm_offset_t ks; vm_page_t m; int rv; int i; ksobj = td->td_kstack_obj; ks = td->td_kstack; for (i = 0; i < KSTACK_PAGES; i++) { m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); if (m->valid != VM_PAGE_BITS_ALL) { rv = vm_pager_get_pages(ksobj, &m, 1, 0); if (rv != VM_PAGER_OK) panic("pmap_swapin_thread: cannot get kstack"); m = vm_page_lookup(ksobj, i); m->valid = VM_PAGE_BITS_ALL; } ma[i] = m; vm_page_lock_queues(); vm_page_wire(m); vm_page_wakeup(m); vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); vm_page_unlock_queues(); } pmap_qenter(ks, ma, KSTACK_PAGES); } /* * Initialize the pmap associated with process 0. */ void pmap_pinit0(pmap_t pm) { int i; for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = 0; pm->pm_active = 0; pm->pm_tsb = NULL; pm->pm_tsb_obj = NULL; bzero(&pm->pm_stats, sizeof(pm->pm_stats)); } /* * Initialize a preallocated and zeroed pmap structure, uch as one in a * vmspace structure. */ void pmap_pinit(pmap_t pm) { vm_page_t ma[TSB_PAGES]; vm_page_t m; int i; /* * Allocate kva space for the tsb. */ if (pm->pm_tsb == NULL) { pm->pm_tsb = (struct tte *)kmem_alloc_pageable(kernel_map, TSB_BSIZE); } /* * Allocate an object for it. */ if (pm->pm_tsb_obj == NULL) pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); for (i = 0; i < TSB_PAGES; i++) { m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_RETRY | VM_ALLOC_ZERO); if ((m->flags & PG_ZERO) == 0) pmap_zero_page(m); m->wire_count++; cnt.v_wire_count++; vm_page_flag_clear(m, PG_MAPPED | PG_BUSY); m->valid = VM_PAGE_BITS_ALL; ma[i] = m; } pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = -1; pm->pm_active = 0; bzero(&pm->pm_stats, sizeof(pm->pm_stats)); } void pmap_pinit2(pmap_t pmap) { /* XXX: Remove this stub when no longer called */ } /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pm) { vm_object_t obj; vm_page_t m; CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb); obj = pm->pm_tsb_obj; KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); KASSERT(pmap_resident_count(pm) == 0, ("pmap_release: resident pages %ld != 0", pmap_resident_count(pm))); while (!TAILQ_EMPTY(&obj->memq)) { m = TAILQ_FIRST(&obj->memq); if (vm_page_sleep_busy(m, FALSE, "pmaprl")) continue; vm_page_busy(m); KASSERT(m->hold_count == 0, ("pmap_release: freeing held tsb page")); m->wire_count--; cnt.v_wire_count--; vm_page_free_zero(m); } pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); } /* * Grow the number of kernel page table entries. Unneeded. */ void pmap_growkernel(vm_offset_t addr) { } /* * This routine is very drastic, but can save the system * in a pinch. */ void pmap_collect(void) { } int pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, vm_offset_t va) { vm_page_t m; m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); if ((tp->tte_data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((tp->tte_data & TD_PV) != 0) { if ((tp->tte_data & TD_W) != 0 && pmap_track_modified(pm, va)) vm_page_dirty(m); if ((tp->tte_data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if (STAILQ_EMPTY(&m->md.tte_list)) vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE); pm->pm_stats.resident_count--; } pmap_cache_remove(m, va); TTE_ZERO(tp); if (PMAP_REMOVE_DONE(pm)) return (0); return (1); } /* * Remove the given range of addresses from the specified map. */ void pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) { struct tte *tp; vm_offset_t va; CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", pm->pm_context[PCPU_GET(cpuid)], start, end); if (PMAP_REMOVE_DONE(pm)) return; if (end - start > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, start, end, pmap_remove_tte); tlb_context_demap(pm); } else { for (va = start; va < end; va += PAGE_SIZE) { if ((tp = tsb_tte_lookup(pm, va)) != NULL) { if (!pmap_remove_tte(pm, NULL, tp, va)) break; } } tlb_range_demap(pm, start, end - 1); } } void pmap_remove_all(vm_page_t m) { struct pmap *pm; struct tte *tpn; struct tte *tp; vm_offset_t va; KASSERT((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0, ("pv_remove_all: illegal for unmanaged page %#lx", VM_PAGE_TO_PHYS(m))); for (tp = STAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { tpn = STAILQ_NEXT(tp, tte_link); if ((tp->tte_data & TD_PV) == 0) continue; pm = TTE_GET_PMAP(tp); va = TTE_GET_VA(tp); if ((tp->tte_data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((tp->tte_data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if ((tp->tte_data & TD_W) != 0 && pmap_track_modified(pm, va)) vm_page_dirty(m); tp->tte_data &= ~TD_V; tlb_page_demap(pm, va); STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); pm->pm_stats.resident_count--; pmap_cache_remove(m, va); TTE_ZERO(tp); } vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE); } int pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, vm_offset_t va) { vm_page_t m; if ((tp->tte_data & TD_PV) != 0) { m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); if ((tp->tte_data & TD_REF) != 0) { vm_page_flag_set(m, PG_REFERENCED); tp->tte_data &= ~TD_REF; } if ((tp->tte_data & TD_W) != 0 && pmap_track_modified(pm, va)) { vm_page_dirty(m); } } tp->tte_data &= ~(TD_W | TD_SW); return (0); } /* * Set the physical protection on the specified range of this map as requested. */ void pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va; struct tte *tp; CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot); if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pm, sva, eva); return; } if (prot & VM_PROT_WRITE) return; if (eva - sva > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); tlb_context_demap(pm); } else { for (va = sva; va < eva; va += PAGE_SIZE) { if ((tp = tsb_tte_lookup(pm, va)) != NULL) pmap_protect_tte(pm, NULL, tp, va); } tlb_range_demap(pm, sva, eva - 1); } } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. */ void pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { struct tte *tp; vm_offset_t pa; u_long data; pa = VM_PAGE_TO_PHYS(m); CTR6(KTR_PMAP, "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); /* * If there is an existing mapping, and the physical address has not * changed, must be protection or wiring change. */ if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { CTR0(KTR_PMAP, "pmap_enter: update"); PMAP_STATS_INC(pmap_enter_nupdate); /* * Wiring change, just update stats. */ if (wired) { if ((tp->tte_data & TD_WIRED) == 0) { tp->tte_data |= TD_WIRED; pm->pm_stats.wired_count++; } } else { if ((tp->tte_data & TD_WIRED) != 0) { tp->tte_data &= ~TD_WIRED; pm->pm_stats.wired_count--; } } /* * Save the old bits and clear the ones we're interested in. */ data = tp->tte_data; tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); /* * If we're turning off write permissions, sense modify status. */ if ((prot & VM_PROT_WRITE) != 0) { tp->tte_data |= TD_SW; if (wired) { tp->tte_data |= TD_W; } } else if ((data & TD_W) != 0 && pmap_track_modified(pm, va)) { vm_page_dirty(m); } /* * If we're turning on execute permissions, flush the icache. */ if ((prot & VM_PROT_EXECUTE) != 0) { if ((data & TD_EXEC) == 0) { PMAP_STATS_INC(pmap_niflush); icache_page_inval(pa); } tp->tte_data |= TD_EXEC; } /* * Delete the old mapping. */ tlb_page_demap(pm, TTE_GET_VA(tp)); } else { /* * If there is an existing mapping, but its for a different * phsyical address, delete the old mapping. */ if (tp != NULL) { CTR0(KTR_PMAP, "pmap_enter: replace"); PMAP_STATS_INC(pmap_enter_nreplace); pmap_remove_tte(pm, NULL, tp, va); tlb_page_demap(pm, va); } else { CTR0(KTR_PMAP, "pmap_enter: new"); PMAP_STATS_INC(pmap_enter_nnew); } /* * Now set up the data and install the new mapping. */ data = TD_V | TD_8K | TD_PA(pa) | TD_CP; if (pm == kernel_pmap) data |= TD_P; if (prot & VM_PROT_WRITE) data |= TD_SW; if (prot & VM_PROT_EXECUTE) { data |= TD_EXEC; PMAP_STATS_INC(pmap_niflush); icache_page_inval(pa); } /* * If its wired update stats. We also don't need reference or * modify tracking for wired mappings, so set the bits now. */ if (wired) { pm->pm_stats.wired_count++; data |= TD_REF | TD_WIRED; if ((prot & VM_PROT_WRITE) != 0) data |= TD_W; } tsb_tte_enter(pm, m, va, data); } } void pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size, int limit) { /* XXX */ } void pmap_prefault(pmap_t pm, vm_offset_t va, vm_map_entry_t entry) { /* XXX */ } /* * Change the wiring attribute for a map/virtual-address pair. * The mapping must already exist in the pmap. */ void pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) { struct tte *tp; if ((tp = tsb_tte_lookup(pm, va)) != NULL) { if (wired) { if ((tp->tte_data & TD_WIRED) == 0) pm->pm_stats.wired_count++; tp->tte_data |= TD_WIRED; } else { if ((tp->tte_data & TD_WIRED) != 0) pm->pm_stats.wired_count--; tp->tte_data &= ~TD_WIRED; } } } static int pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) { vm_page_t m; u_long data; if (tsb_tte_lookup(dst_pmap, va) == NULL) { data = tp->tte_data & ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); tsb_tte_enter(dst_pmap, m, va, data); } return (1); } void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { struct tte *tp; vm_offset_t va; if (dst_addr != src_addr) return; if (len > PMAP_TSB_THRESH) { tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, pmap_copy_tte); tlb_context_demap(dst_pmap); } else { for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) pmap_copy_tte(src_pmap, dst_pmap, tp, va); } tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); } } /* * Zero a page of physical memory by temporarily mapping it into the tlb. */ void pmap_zero_page(vm_page_t m) { - vm_offset_t pa; + vm_offset_t va; - pa = VM_PAGE_TO_PHYS(m); - CTR1(KTR_PMAP, "pmap_zero_page: pa=%#lx", pa); - dcache_page_inval(pa); - aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); + va = pmap_map_direct(m); + CTR2(KTR_PMAP, "pmap_zero_page: pa=%#lx va=%#lx", + VM_PAGE_TO_PHYS(m), va); + bzero((void *)va, PAGE_SIZE); } void pmap_zero_page_area(vm_page_t m, int off, int size) { - vm_offset_t pa; + vm_offset_t va; - pa = VM_PAGE_TO_PHYS(m); - CTR3(KTR_PMAP, "pmap_zero_page_area: pa=%#lx off=%#x size=%#x", - pa, off, size); KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); - dcache_page_inval(pa); - aszero(ASI_PHYS_USE_EC, pa + off, size); + va = pmap_map_direct(m); + CTR4(KTR_PMAP, "pmap_zero_page_area: pa=%#lx va=%#lx off=%#x size=%#x", + VM_PAGE_TO_PHYS(m), va, off, size); + bzero((void *)(va + off), size); } void pmap_zero_page_idle(vm_page_t m) { - vm_offset_t pa = VM_PAGE_TO_PHYS(m); + vm_offset_t va; - CTR1(KTR_PMAP, "pmap_zero_page_idle: pa=%#lx", pa); -#ifdef SMP - mtx_lock(&Giant); -#endif - dcache_inval_phys(pa, pa + PAGE_SIZE - 1); -#ifdef SMP - mtx_unlock(&Giant); -#endif - aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); + va = pmap_map_direct(m); + CTR2(KTR_PMAP, "pmap_zero_page_idle: pa=%#lx va=%#lx", + VM_PAGE_TO_PHYS(m), va); + bzero((void *)va, PAGE_SIZE); } /* * Copy a page of physical memory by temporarily mapping it into the tlb. */ void pmap_copy_page(vm_page_t msrc, vm_page_t mdst) { vm_offset_t dst; vm_offset_t src; - dst = VM_PAGE_TO_PHYS(mdst); - src = VM_PAGE_TO_PHYS(msrc); - CTR2(KTR_PMAP, "pmap_copy_page: src=%#lx dst=%#lx", src, dst); - dcache_page_inval(dst); - ascopy(ASI_PHYS_USE_EC, src, dst, PAGE_SIZE); + src = pmap_map_direct(msrc); + dst = pmap_map_direct(mdst); + CTR4(KTR_PMAP, "pmap_zero_page: src=%#lx va=%#lx dst=%#lx va=%#lx", + VM_PAGE_TO_PHYS(msrc), src, VM_PAGE_TO_PHYS(mdst), dst); + bcopy((void *)src, (void *)dst, PAGE_SIZE); } /* * Make the specified page pageable (or not). Unneeded. */ void pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, boolean_t pageable) { } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pm, vm_page_t m) { struct tte *tp; int loops; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); loops = 0; STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if (TTE_GET_PMAP(tp) == pm) return (TRUE); if (++loops >= 16) break; } return (FALSE); } /* * Remove all pages from specified address space, this aids process exit * speeds. This is much faster than pmap_remove n the case of running down * an entire address space. Only works for the current pmap. */ void pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva) { } /* * Lower the permission for all mappings to a given page. */ void pmap_page_protect(vm_page_t m, vm_prot_t prot) { if ((prot & VM_PROT_WRITE) == 0) { if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) pmap_clear_write(m); else pmap_remove_all(m); } } vm_offset_t pmap_phys_address(int ppn) { return (sparc64_ptob(ppn)); } /* * pmap_ts_referenced: * * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct tte *tpf; struct tte *tpn; struct tte *tp; int count; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (0); count = 0; if ((tp = STAILQ_FIRST(&m->md.tte_list)) != NULL) { tpf = tp; do { tpn = STAILQ_NEXT(tp, tte_link); STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); STAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); if ((tp->tte_data & TD_PV) == 0 || !pmap_track_modified(TTE_GET_PMAP(tp), TTE_GET_VA(tp))) continue; if ((tp->tte_data & TD_REF) != 0) { tp->tte_data &= ~TD_REF; if (++count > 4) break; } } while ((tp = tpn) != NULL && tp != tpf); } return (count); } boolean_t pmap_is_modified(vm_page_t m) { struct tte *tp; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return FALSE; STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0 || !pmap_track_modified(TTE_GET_PMAP(tp), TTE_GET_VA(tp))) continue; if ((tp->tte_data & TD_W) != 0) return (TRUE); } return (FALSE); } void pmap_clear_modify(vm_page_t m) { struct tte *tp; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return; STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if ((tp->tte_data & TD_W) != 0) { tp->tte_data &= ~TD_W; tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } } void pmap_clear_reference(vm_page_t m) { struct tte *tp; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return; STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if ((tp->tte_data & TD_REF) != 0) { tp->tte_data &= ~TD_REF; tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } } void pmap_clear_write(vm_page_t m) { struct tte *tp; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return; STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if ((tp->tte_data & (TD_SW | TD_W)) != 0) { if ((tp->tte_data & TD_W) != 0 && pmap_track_modified(TTE_GET_PMAP(tp), TTE_GET_VA(tp))) vm_page_dirty(m); tp->tte_data &= ~(TD_SW | TD_W); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } } int pmap_mincore(pmap_t pm, vm_offset_t addr) { TODO; return (0); } /* * Activate a user pmap. The pmap must be activated before its address space * can be accessed in any way. */ void pmap_activate(struct thread *td) { struct vmspace *vm; vm_offset_t tsb; u_long context; pmap_t pm; /* * Load all the data we need up front to encourage the compiler to * not issue any loads while we have interrupts disable below. */ vm = td->td_proc->p_vmspace; pm = &vm->vm_pmap; tsb = (vm_offset_t)pm->pm_tsb; KASSERT(pm->pm_active == 0, ("pmap_activate: pmap already active?")); KASSERT(pm->pm_context[PCPU_GET(cpuid)] != 0, ("pmap_activate: activating nucleus context?")); mtx_lock_spin(&sched_lock); wrpr(pstate, 0, PSTATE_MMU); mov(tsb, TSB_REG); wrpr(pstate, 0, PSTATE_KERNEL); context = pmap_context_alloc(); pm->pm_context[PCPU_GET(cpuid)] = context; pm->pm_active |= PCPU_GET(cpumask); PCPU_SET(vmspace, vm); stxa(AA_DMMU_PCXR, ASI_DMMU, context); membar(Sync); mtx_unlock_spin(&sched_lock); } vm_offset_t pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) { return (va); }