Index: head/sys/powerpc/booke/trap_subr.S =================================================================== --- head/sys/powerpc/booke/trap_subr.S (revision 287014) +++ head/sys/powerpc/booke/trap_subr.S (revision 287015) @@ -1,920 +1,924 @@ /*- * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski * Copyright (C) 2006 Semihalf, Marian Balakowicz * Copyright (C) 2006 Juniper Networks, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ */ /* * NOTICE: This is not a standalone file. to use it, #include it in * your port's locore.S, like so: * * #include */ /* * SPRG usage notes * * SPRG0 - pcpu pointer * SPRG1 - all interrupts except TLB miss, critical, machine check * SPRG2 - critical * SPRG3 - machine check * SPRG4-6 - scratch * */ /* Get the per-CPU data structure */ #define GET_CPUINFO(r) mfsprg0 r #define RES_GRANULE 32 #define RES_LOCK 0 /* offset to the 'lock' word */ #define RES_RECURSE 4 /* offset to the 'recurse' word */ /* * Standard interrupt prolog * * sprg_sp - SPRG{1-3} reg used to temporarily store the SP * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave}) * isrr0-1 - save restore registers with CPU state at interrupt time (may be * SRR0-1, CSRR0-1, MCSRR0-1 * * 1. saves in the given savearea: * - R30-31 * - DEAR, ESR * - xSRR0-1 * * 2. saves CR -> R30 * * 3. switches to kstack if needed * * 4. notes: * - R31 can be used as scratch register until a new frame is layed on * the stack with FRAME_SETUP * * - potential TLB miss: NO. Saveareas are always acessible via TLB1 * permanent entries, and within this prolog we do not dereference any * locations potentially not in the TLB */ #define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ mtspr sprg_sp, %r1; /* Save SP */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ stw %r30, (savearea+CPUSAVE_R30)(%r1); \ stw %r31, (savearea+CPUSAVE_R31)(%r1); \ mfdear %r30; \ mfesr %r31; \ stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ mfspr %r30, isrr0; \ mfspr %r31, isrr1; /* MSR at interrupt time */ \ stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \ stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \ isync; \ mfspr %r1, sprg_sp; /* Restore SP */ \ mfcr %r30; /* Save CR */ \ /* switch to per-thread kstack if intr taken in user mode */ \ mtcr %r31; /* MSR at interrupt time */ \ bf 17, 1f; \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1: #define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1) \ mtspr sprg_sp, %r1; /* Save SP */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ stw %r30, (savearea+CPUSAVE_R30)(%r1); \ stw %r31, (savearea+CPUSAVE_R31)(%r1); \ mfdear %r30; \ mfesr %r31; \ stw %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); \ stw %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); \ mfspr %r30, isrr0; \ mfspr %r31, isrr1; /* MSR at interrupt time */ \ stw %r30, (savearea+CPUSAVE_SRR0)(%r1); \ stw %r31, (savearea+CPUSAVE_SRR1)(%r1); \ mfspr %r30, SPR_SRR0; \ mfspr %r31, SPR_SRR1; /* MSR at interrupt time */ \ stw %r30, (savearea+CPUSAVE_SRR0+8)(%r1); \ stw %r31, (savearea+CPUSAVE_SRR1+8)(%r1); \ isync; \ mfspr %r1, sprg_sp; /* Restore SP */ \ mfcr %r30; /* Save CR */ \ /* switch to per-thread kstack if intr taken in user mode */ \ mtcr %r31; /* MSR at interrupt time */ \ bf 17, 1f; \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ lwz %r1, PC_CURPCB(%r1); /* Per-thread kernel stack */ \ 1: /* * FRAME_SETUP assumes: * SPRG{1-3} SP at the time interrupt occured * savearea r30-r31, DEAR, ESR, xSRR0-1 * r30 CR * r31 scratch * r1 kernel stack * * sprg_sp - SPRG reg containing SP at the time interrupt occured * savearea - temp save * exc - exception number (EXC_xxx) * * 1. sets a new frame * 2. saves in the frame: * - R0, R1 (SP at the time of interrupt), R2, LR, CR * - R3-31 (R30-31 first restored from savearea) * - XER, CTR, DEAR, ESR (from savearea), xSRR0-1 * * Notes: * - potential TLB miss: YES, since we make dereferences to kstack, which * can happen not covered (we can have up to two DTLB misses if fortunate * enough i.e. when kstack crosses page boundary and both pages are * untranslated) */ #define FRAME_SETUP(sprg_sp, savearea, exc) \ mfspr %r31, sprg_sp; /* get saved SP */ \ /* establish a new stack frame and put everything on it */ \ stwu %r31, -FRAMELEN(%r1); \ stw %r0, FRAME_0+8(%r1); /* save r0 in the trapframe */ \ stw %r31, FRAME_1+8(%r1); /* save SP " " */ \ stw %r2, FRAME_2+8(%r1); /* save r2 " " */ \ mflr %r31; \ stw %r31, FRAME_LR+8(%r1); /* save LR " " */ \ stw %r30, FRAME_CR+8(%r1); /* save CR " " */ \ GET_CPUINFO(%r2); \ lwz %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \ lwz %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \ /* save R3-31 */ \ stmw %r3, FRAME_3+8(%r1) ; \ /* save DEAR, ESR */ \ lwz %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2); \ lwz %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2); \ stw %r28, FRAME_BOOKE_DEAR+8(%r1); \ stw %r29, FRAME_BOOKE_ESR+8(%r1); \ /* save XER, CTR, exc number */ \ mfxer %r3; \ mfctr %r4; \ stw %r3, FRAME_XER+8(%r1); \ stw %r4, FRAME_CTR+8(%r1); \ li %r5, exc; \ stw %r5, FRAME_EXC+8(%r1); \ /* save DBCR0 */ \ mfspr %r3, SPR_DBCR0; \ stw %r3, FRAME_BOOKE_DBCR0+8(%r1); \ /* save xSSR0-1 */ \ lwz %r30, (savearea+CPUSAVE_SRR0)(%r2); \ lwz %r31, (savearea+CPUSAVE_SRR1)(%r2); \ stw %r30, FRAME_SRR0+8(%r1); \ stw %r31, FRAME_SRR1+8(%r1); \ lwz %r2,PC_CURTHREAD(%r2) /* set curthread pointer */ /* * * isrr0-1 - save restore registers to restore CPU state to (may be * SRR0-1, CSRR0-1, MCSRR0-1 * * Notes: * - potential TLB miss: YES. The deref'd kstack may be not covered */ #define FRAME_LEAVE(isrr0, isrr1) \ /* restore CTR, XER, LR, CR */ \ lwz %r4, FRAME_CTR+8(%r1); \ lwz %r5, FRAME_XER+8(%r1); \ lwz %r6, FRAME_LR+8(%r1); \ lwz %r7, FRAME_CR+8(%r1); \ mtctr %r4; \ mtxer %r5; \ mtlr %r6; \ mtcr %r7; \ /* restore DBCR0 */ \ lwz %r4, FRAME_BOOKE_DBCR0+8(%r1); \ mtspr SPR_DBCR0, %r4; \ /* restore xSRR0-1 */ \ lwz %r30, FRAME_SRR0+8(%r1); \ lwz %r31, FRAME_SRR1+8(%r1); \ mtspr isrr0, %r30; \ mtspr isrr1, %r31; \ /* restore R2-31, SP */ \ lmw %r2, FRAME_2+8(%r1) ; \ lwz %r0, FRAME_0+8(%r1); \ lwz %r1, FRAME_1+8(%r1); \ isync /* * TLB miss prolog * * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area * * Notes: * - potential TLB miss: NO. It is crucial that we do not generate a TLB * miss within the TLB prolog itself! * - TLBSAVE is always translated */ #define TLB_PROLOG \ mtsprg4 %r1; /* Save SP */ \ mtsprg5 %r28; \ mtsprg6 %r29; \ /* calculate TLB nesting level and TLBSAVE instance address */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \ rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \ addi %r28, %r28, 1; \ stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \ addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ add %r1, %r1, %r29; /* current TLBSAVE ptr */ \ \ /* save R20-31 */ \ mfsprg5 %r28; \ mfsprg6 %r29; \ stmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \ /* save LR, CR */ \ mflr %r30; \ mfcr %r31; \ stw %r30, (TLBSAVE_BOOKE_LR)(%r1); \ stw %r31, (TLBSAVE_BOOKE_CR)(%r1); \ /* save SRR0-1 */ \ mfsrr0 %r30; /* execution addr at interrupt time */ \ mfsrr1 %r31; /* MSR at interrupt time*/ \ stw %r30, (TLBSAVE_BOOKE_SRR0)(%r1); /* save SRR0 */ \ stw %r31, (TLBSAVE_BOOKE_SRR1)(%r1); /* save SRR1 */ \ isync; \ mfsprg4 %r1 /* * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area * * same notes as for the TLB_PROLOG */ #define TLB_RESTORE \ mtsprg4 %r1; /* Save SP */ \ GET_CPUINFO(%r1); /* Per-cpu structure */ \ /* calculate TLB nesting level and TLBSAVE instance addr */ \ lwz %r28, PC_BOOKE_TLB_LEVEL(%r1); \ subi %r28, %r28, 1; \ stw %r28, PC_BOOKE_TLB_LEVEL(%r1); \ rlwinm %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */ \ addi %r29, %r29, PC_BOOKE_TLBSAVE@l; \ add %r1, %r1, %r29; \ \ /* restore LR, CR */ \ lwz %r30, (TLBSAVE_BOOKE_LR)(%r1); \ lwz %r31, (TLBSAVE_BOOKE_CR)(%r1); \ mtlr %r30; \ mtcr %r31; \ /* restore SRR0-1 */ \ lwz %r30, (TLBSAVE_BOOKE_SRR0)(%r1); \ lwz %r31, (TLBSAVE_BOOKE_SRR1)(%r1); \ mtsrr0 %r30; \ mtsrr1 %r31; \ /* restore R20-31 */ \ lmw %r20, (TLBSAVE_BOOKE_R20)(%r1); \ mfsprg4 %r1 #ifdef SMP #define TLB_LOCK \ GET_CPUINFO(%r20); \ lwz %r21, PC_CURTHREAD(%r20); \ lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \ \ 1: lwarx %r23, 0, %r22; \ cmpwi %r23, TLB_UNLOCKED; \ beq 2f; \ \ /* check if this is recursion */ \ cmplw cr0, %r21, %r23; \ bne- 1b; \ \ 2: /* try to acquire lock */ \ stwcx. %r21, 0, %r22; \ bne- 1b; \ \ /* got it, update recursion counter */ \ lwz %r21, RES_RECURSE(%r22); \ addi %r21, %r21, 1; \ stw %r21, RES_RECURSE(%r22); \ isync; \ msync #define TLB_UNLOCK \ GET_CPUINFO(%r20); \ lwz %r21, PC_CURTHREAD(%r20); \ lwz %r22, PC_BOOKE_TLB_LOCK(%r20); \ \ /* update recursion counter */ \ lwz %r23, RES_RECURSE(%r22); \ subi %r23, %r23, 1; \ stw %r23, RES_RECURSE(%r22); \ \ cmpwi %r23, 0; \ bne 1f; \ isync; \ msync; \ \ /* release the lock */ \ li %r23, TLB_UNLOCKED; \ stw %r23, 0(%r22); \ 1: isync; \ msync #else #define TLB_LOCK #define TLB_UNLOCK #endif /* SMP */ #define INTERRUPT(label) \ .globl label; \ .align 5; \ CNAME(label): /* * Interrupt handling routines in BookE can be flexibly placed and do not have * to live in pre-defined vectors location. Note they need to be TLB-mapped at * all times in order to be able to handle exceptions. We thus arrange for * them to be part of kernel text which is always TLB-accessible. * * The interrupt handling routines have to be 16 bytes aligned: we align them * to 32 bytes (cache line length) which supposedly performs better. * */ .text .globl CNAME(interrupt_vector_base) .align 5 interrupt_vector_base: /***************************************************************************** * Critical input interrupt ****************************************************************************/ INTERRUPT(int_critical_input) STANDARD_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) rfci /***************************************************************************** * Machine check interrupt ****************************************************************************/ INTERRUPT(int_machine_check) STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1) FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1) rfmci /***************************************************************************** * Data storage interrupt ****************************************************************************/ INTERRUPT(int_data_storage) STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI) b trap_common /***************************************************************************** * Instruction storage interrupt ****************************************************************************/ INTERRUPT(int_instr_storage) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI) b trap_common /***************************************************************************** * External input interrupt ****************************************************************************/ INTERRUPT(int_external_input) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) b clear_we INTERRUPT(int_alignment) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI) b trap_common INTERRUPT(int_program) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM) b trap_common /***************************************************************************** * System call ****************************************************************************/ INTERRUPT(int_syscall) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC) b trap_common /***************************************************************************** * Decrementer interrupt ****************************************************************************/ INTERRUPT(int_decrementer) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) b clear_we /***************************************************************************** * Fixed interval timer ****************************************************************************/ INTERRUPT(int_fixed_interval_timer) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT) b trap_common /***************************************************************************** * Watchdog interrupt ****************************************************************************/ INTERRUPT(int_watchdog) STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG) b trap_common #ifdef HWPMC_HOOKS /***************************************************************************** * PMC Interrupt ****************************************************************************/ INTERRUPT(int_performance_counter) STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1) FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF) addi %r3, %r1, 8 bl CNAME(powerpc_interrupt) b trapexit #endif /***************************************************************************** * Data TLB miss interrupt * * There can be nested TLB misses - while handling a TLB miss we reference * data structures that may be not covered by translations. We support up to * TLB_NESTED_MAX-1 nested misses. * * Registers use: * r31 - dear * r30 - unused * r29 - saved mas0 * r28 - saved mas1 * r27 - saved mas2 * r26 - pmap address * r25 - pte address * * r20:r23 - scratch registers ****************************************************************************/ INTERRUPT(int_data_tlb_error) TLB_PROLOG TLB_LOCK mfdear %r31 /* * Save MAS0-MAS2 registers. There might be another tlb miss during * pte lookup overwriting current contents (which was hw filled). */ mfspr %r29, SPR_MAS0 mfspr %r28, SPR_MAS1 mfspr %r27, SPR_MAS2 /* Check faulting address. */ lis %r21, VM_MAXUSER_ADDRESS@h ori %r21, %r21, VM_MAXUSER_ADDRESS@l cmplw cr0, %r31, %r21 blt search_user_pmap /* If it's kernel address, allow only supervisor mode misses. */ mfsrr1 %r21 mtcr %r21 bt 17, search_failed /* check MSR[PR] */ search_kernel_pmap: /* Load r26 with kernel_pmap address */ bl 1f .long kernel_pmap_store-. 1: mflr %r21 lwz %r26, 0(%r21) add %r26, %r21, %r26 /* kernel_pmap_store in r26 */ /* Force kernel tid, set TID to 0 in MAS1. */ li %r21, 0 rlwimi %r28, %r21, 0, 8, 15 /* clear TID bits */ tlb_miss_handle: /* This may result in nested tlb miss. */ bl pte_lookup /* returns PTE address in R25 */ cmpwi %r25, 0 /* pte found? */ beq search_failed /* Finish up, write TLB entry. */ bl tlb_fill_entry tlb_miss_return: TLB_UNLOCK TLB_RESTORE rfi search_user_pmap: /* Load r26 with current user space process pmap */ GET_CPUINFO(%r26) lwz %r26, PC_CURPMAP(%r26) b tlb_miss_handle search_failed: /* * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with * the faulting virtual address anyway, but put a fake RPN and no * access rights. This should cause a following {D,I}SI exception. */ lis %r23, 0xffff0000@h /* revoke all permissions */ /* Load MAS registers. */ mtspr SPR_MAS0, %r29 isync mtspr SPR_MAS1, %r28 isync mtspr SPR_MAS2, %r27 isync mtspr SPR_MAS3, %r23 isync tlbwe msync isync b tlb_miss_return /***************************************************************************** * * Return pte address that corresponds to given pmap/va. If there is no valid * entry return 0. * * input: r26 - pmap * input: r31 - dear * output: r25 - pte address * * scratch regs used: r21 * ****************************************************************************/ pte_lookup: cmpwi %r26, 0 beq 1f /* fail quickly if pmap is invalid */ srwi %r21, %r31, PDIR_SHIFT /* pdir offset */ slwi %r21, %r21, PDIR_ENTRY_SHIFT /* multiply by pdir entry size */ addi %r25, %r26, PM_PDIR /* pmap pm_dir[] address */ add %r25, %r25, %r21 /* offset within pm_pdir[] table */ /* * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx] * This load may cause a Data TLB miss for non-kernel pmap! */ lwz %r25, 0(%r25) cmpwi %r25, 0 beq 2f lis %r21, PTBL_MASK@h ori %r21, %r21, PTBL_MASK@l and %r21, %r21, %r31 /* ptbl offset, multiply by ptbl entry size */ srwi %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT) add %r25, %r25, %r21 /* address of pte entry */ /* * Get pte->flags * This load may cause a Data TLB miss for non-kernel pmap! */ lwz %r21, PTE_FLAGS(%r25) andis. %r21, %r21, PTE_VALID@h bne 2f 1: li %r25, 0 2: blr /***************************************************************************** * * Load MAS1-MAS3 registers with data, write TLB entry * * input: * r29 - mas0 * r28 - mas1 * r27 - mas2 * r25 - pte * * output: none * * scratch regs: r21-r23 * ****************************************************************************/ tlb_fill_entry: /* * Update PTE flags: we have to do it atomically, as pmap_protect() * running on other CPUs could attempt to update the flags at the same * time. */ li %r23, PTE_FLAGS 1: lwarx %r21, %r23, %r25 /* get pte->flags */ oris %r21, %r21, PTE_REFERENCED@h /* set referenced bit */ andi. %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */ beq 2f oris %r21, %r21, PTE_MODIFIED@h /* set modified bit */ 2: stwcx. %r21, %r23, %r25 /* write it back */ bne- 1b /* Update MAS2. */ rlwimi %r27, %r21, 0, 27, 30 /* insert WIMG bits from pte */ /* Setup MAS3 value in r23. */ lwz %r23, PTE_RPN(%r25) /* get pte->rpn */ + rlwinm %r22, %r23, 12, 0, 20 /* extract MAS3 portion of RPN */ - rlwimi %r23, %r21, 24, 26, 31 /* insert protection bits from pte */ + rlwimi %r22, %r21, 24, 26, 31 /* insert protection bits from pte */ + rlwinm %r23, %r23, 12, 28, 31 /* MAS7 portion of RPN */ /* Load MAS registers. */ mtspr SPR_MAS0, %r29 isync mtspr SPR_MAS1, %r28 isync mtspr SPR_MAS2, %r27 isync - mtspr SPR_MAS3, %r23 + mtspr SPR_MAS3, %r22 + isync + mtspr SPR_MAS7, %r23 isync tlbwe isync msync blr /***************************************************************************** * Instruction TLB miss interrupt * * Same notes as for the Data TLB miss ****************************************************************************/ INTERRUPT(int_inst_tlb_error) TLB_PROLOG TLB_LOCK mfsrr0 %r31 /* faulting address */ /* * Save MAS0-MAS2 registers. There might be another tlb miss during pte * lookup overwriting current contents (which was hw filled). */ mfspr %r29, SPR_MAS0 mfspr %r28, SPR_MAS1 mfspr %r27, SPR_MAS2 mfsrr1 %r21 mtcr %r21 /* check MSR[PR] */ bt 17, search_user_pmap b search_kernel_pmap .globl interrupt_vector_top interrupt_vector_top: /***************************************************************************** * Debug interrupt ****************************************************************************/ INTERRUPT(int_debug) STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1) FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG) GET_CPUINFO(%r3) lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3) bl 0f .long interrupt_vector_base-. .long interrupt_vector_top-. 0: mflr %r5 lwz %r4,0(%r5) /* interrupt_vector_base in r4 */ add %r4,%r4,%r5 cmplw cr0, %r3, %r4 blt 1f lwz %r4,4(%r5) /* interrupt_vector_top in r4 */ add %r4,%r4,%r5 addi %r4,%r4,4 cmplw cr0, %r3, %r4 bge 1f /* Disable single-stepping for the interrupt handlers. */ lwz %r3, FRAME_SRR1+8(%r1); rlwinm %r3, %r3, 0, 23, 21 stw %r3, FRAME_SRR1+8(%r1); /* Restore srr0 and srr1 as they could have been clobbered. */ GET_CPUINFO(%r4) lwz %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r4); mtspr SPR_SRR0, %r3 lwz %r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r4); mtspr SPR_SRR1, %r4 b 9f 1: addi %r3, %r1, 8 bl CNAME(trap) /* * Handle ASTs, needed for proper support of single-stepping. * We actually need to return to the process with an rfi. */ b trapexit 9: FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1) rfci /***************************************************************************** * Common trap code ****************************************************************************/ trap_common: /* Call C trap dispatcher */ addi %r3, %r1, 8 bl CNAME(trap) .globl CNAME(trapexit) /* exported for db_backtrace use */ CNAME(trapexit): /* disable interrupts */ wrteei 0 /* Test AST pending - makes sense for user process only */ lwz %r5, FRAME_SRR1+8(%r1) mtcr %r5 bf 17, 1f GET_CPUINFO(%r3) lwz %r4, PC_CURTHREAD(%r3) lwz %r4, TD_FLAGS(%r4) lis %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h ori %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l and. %r4, %r4, %r5 beq 1f /* re-enable interrupts before calling ast() */ wrteei 1 addi %r3, %r1, 8 bl CNAME(ast) .globl CNAME(asttrapexit) /* db_backtrace code sentinel #2 */ CNAME(asttrapexit): b trapexit /* test ast ret value ? */ 1: FRAME_LEAVE(SPR_SRR0, SPR_SRR1) rfi #if defined(KDB) /* * Deliberate entry to dbtrap */ .globl CNAME(breakpoint) CNAME(breakpoint): mtsprg1 %r1 mfmsr %r3 mtsrr1 %r3 andi. %r3, %r3, ~(PSL_EE | PSL_ME)@l mtmsr %r3 /* disable interrupts */ isync GET_CPUINFO(%r3) stw %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3) stw %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3) mflr %r31 mtsrr0 %r31 mfdear %r30 mfesr %r31 stw %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3) stw %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3) mfsrr0 %r30 mfsrr1 %r31 stw %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3) stw %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3) isync mfcr %r30 /* * Now the kdb trap catching code. */ dbtrap: FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG) /* Call C trap code: */ addi %r3, %r1, 8 bl CNAME(db_trap_glue) or. %r3, %r3, %r3 bne dbleave /* This wasn't for KDB, so switch to real trap: */ b trap_common dbleave: FRAME_LEAVE(SPR_SRR0, SPR_SRR1) rfi #endif /* KDB */ clear_we: lwz %r3, (FRAME_SRR1+8)(%r1) rlwinm %r3, %r3, 0, 14, 12 stw %r3, (FRAME_SRR1+8)(%r1) b trapexit #ifdef SMP ENTRY(tlb_lock) GET_CPUINFO(%r5) lwz %r5, PC_CURTHREAD(%r5) 1: lwarx %r4, 0, %r3 cmpwi %r4, TLB_UNLOCKED bne 1b stwcx. %r5, 0, %r3 bne- 1b isync msync blr ENTRY(tlb_unlock) isync msync li %r4, TLB_UNLOCKED stw %r4, 0(%r3) isync msync blr /* * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes); * only a single word from this granule will actually be used as a spin lock * for mutual exclusion between TLB miss handler and pmap layer that * manipulates page table contents. */ .data .align 5 GLOBAL(tlb0_miss_locks) .space RES_GRANULE * MAXCPU #endif Index: head/sys/powerpc/include/_types.h =================================================================== --- head/sys/powerpc/include/_types.h (revision 287014) +++ head/sys/powerpc/include/_types.h (revision 287015) @@ -1,161 +1,165 @@ /*- * Copyright (c) 2002 Mike Barcroft * Copyright (c) 1990, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * From: @(#)ansi.h 8.2 (Berkeley) 1/4/94 * From: @(#)types.h 8.3 (Berkeley) 1/5/94 * $FreeBSD$ */ #ifndef _MACHINE__TYPES_H_ #define _MACHINE__TYPES_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif /* * Basic types upon which most other types are built. */ typedef signed char __int8_t; typedef unsigned char __uint8_t; typedef short __int16_t; typedef unsigned short __uint16_t; typedef int __int32_t; typedef unsigned int __uint32_t; #ifdef __LP64__ typedef long __int64_t; typedef unsigned long __uint64_t; #else #ifndef lint __extension__ #endif /* LONGLONG */ typedef long long __int64_t; #ifndef lint __extension__ #endif /* LONGLONG */ typedef unsigned long long __uint64_t; #endif /* * Standard type definitions. */ typedef __uint32_t __clock_t; /* clock()... */ typedef double __double_t; typedef float __float_t; #ifdef __LP64__ typedef __int64_t __critical_t; typedef __int64_t __intfptr_t; typedef __int64_t __intptr_t; #else typedef __int32_t __critical_t; typedef __int32_t __intfptr_t; typedef __int32_t __intptr_t; #endif typedef __int64_t __intmax_t; typedef __int32_t __int_fast8_t; typedef __int32_t __int_fast16_t; typedef __int32_t __int_fast32_t; typedef __int64_t __int_fast64_t; typedef __int8_t __int_least8_t; typedef __int16_t __int_least16_t; typedef __int32_t __int_least32_t; typedef __int64_t __int_least64_t; #ifdef __LP64__ typedef __int64_t __ptrdiff_t; /* ptr1 - ptr2 */ typedef __int64_t __register_t; typedef __int64_t __segsz_t; /* segment size (in pages) */ typedef __uint64_t __size_t; /* sizeof() */ typedef __int64_t __ssize_t; /* byte count or error */ typedef __int64_t __time_t; /* time()... */ typedef __uint64_t __uintfptr_t; typedef __uint64_t __uintptr_t; #else typedef __int32_t __ptrdiff_t; /* ptr1 - ptr2 */ typedef __int32_t __register_t; typedef __int32_t __segsz_t; /* segment size (in pages) */ typedef __uint32_t __size_t; /* sizeof() */ typedef __int32_t __ssize_t; /* byte count or error */ typedef __int32_t __time_t; /* time()... */ typedef __uint32_t __uintfptr_t; typedef __uint32_t __uintptr_t; #endif typedef __uint64_t __uintmax_t; typedef __uint32_t __uint_fast8_t; typedef __uint32_t __uint_fast16_t; typedef __uint32_t __uint_fast32_t; typedef __uint64_t __uint_fast64_t; typedef __uint8_t __uint_least8_t; typedef __uint16_t __uint_least16_t; typedef __uint32_t __uint_least32_t; typedef __uint64_t __uint_least64_t; #ifdef __LP64__ typedef __uint64_t __u_register_t; typedef __uint64_t __vm_offset_t; typedef __uint64_t __vm_paddr_t; typedef __uint64_t __vm_size_t; #else typedef __uint32_t __u_register_t; typedef __uint32_t __vm_offset_t; +#ifdef BOOKE +typedef __uint64_t __vm_paddr_t; +#else typedef __uint32_t __vm_paddr_t; +#endif typedef __uint32_t __vm_size_t; #endif typedef __int64_t __vm_ooffset_t; typedef __uint64_t __vm_pindex_t; typedef int ___wchar_t; #define __WCHAR_MIN __INT_MIN /* min value for a wchar_t */ #define __WCHAR_MAX __INT_MAX /* max value for a wchar_t */ /* * Unusual type definitions. */ #if defined(__GNUCLIKE_BUILTIN_VARARGS) typedef __builtin_va_list __va_list; /* internally known to gcc */ #else typedef struct { char __gpr; char __fpr; char __pad[2]; char *__stack; char *__base; } __va_list; #endif /* post GCC 2.95 */ #if defined(__GNUC_VA_LIST_COMPATIBILITY) && !defined(__GNUC_VA_LIST) \ && !defined(__NO_GNUC_VA_LIST) #define __GNUC_VA_LIST typedef __va_list __gnuc_va_list; /* compatibility w/GNU headers*/ #endif #endif /* !_MACHINE__TYPES_H_ */ Index: head/sys/powerpc/include/pte.h =================================================================== --- head/sys/powerpc/include/pte.h (revision 287014) +++ head/sys/powerpc/include/pte.h (revision 287015) @@ -1,277 +1,282 @@ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $NetBSD: pte.h,v 1.2 1998/08/31 14:43:40 tsubai Exp $ * $FreeBSD$ */ #ifndef _MACHINE_PTE_H_ #define _MACHINE_PTE_H_ #if defined(AIM) /* * Page Table Entries */ #ifndef LOCORE /* 32-bit PTE */ struct pte { u_int32_t pte_hi; u_int32_t pte_lo; }; struct pteg { struct pte pt[8]; }; /* 64-bit (long) PTE */ struct lpte { u_int64_t pte_hi; u_int64_t pte_lo; }; struct lpteg { struct lpte pt[8]; }; #endif /* LOCORE */ /* 32-bit PTE definitions */ /* High word: */ #define PTE_VALID 0x80000000 #define PTE_VSID_SHFT 7 #define PTE_HID 0x00000040 #define PTE_API 0x0000003f /* Low word: */ #define PTE_RPGN 0xfffff000 #define PTE_REF 0x00000100 #define PTE_CHG 0x00000080 #define PTE_WIMG 0x00000078 #define PTE_W 0x00000040 #define PTE_I 0x00000020 #define PTE_M 0x00000010 #define PTE_G 0x00000008 #define PTE_PP 0x00000003 #define PTE_SO 0x00000000 /* Super. Only (U: XX, S: RW) */ #define PTE_SW 0x00000001 /* Super. Write-Only (U: RO, S: RW) */ #define PTE_BW 0x00000002 /* Supervisor (U: RW, S: RW) */ #define PTE_BR 0x00000003 /* Both Read Only (U: RO, S: RO) */ #define PTE_RW PTE_BW #define PTE_RO PTE_BR #define PTE_EXEC 0x00000200 /* pseudo bit in attrs; page is exec */ /* 64-bit PTE definitions */ /* High quadword: */ #define LPTE_VSID_SHIFT 12 #define LPTE_AVPN_MASK 0xFFFFFFFFFFFFFF80ULL #define LPTE_API 0x0000000000000F80ULL #define LPTE_SWBITS 0x0000000000000078ULL #define LPTE_WIRED 0x0000000000000010ULL #define LPTE_LOCKED 0x0000000000000008ULL #define LPTE_BIG 0x0000000000000004ULL /* 4kb/16Mb page */ #define LPTE_HID 0x0000000000000002ULL #define LPTE_VALID 0x0000000000000001ULL /* Low quadword: */ #define EXTEND_PTE(x) UINT64_C(x) /* make constants 64-bit */ #define LPTE_RPGN 0xfffffffffffff000ULL #define LPTE_REF EXTEND_PTE( PTE_REF ) #define LPTE_CHG EXTEND_PTE( PTE_CHG ) #define LPTE_WIMG EXTEND_PTE( PTE_WIMG ) #define LPTE_W EXTEND_PTE( PTE_W ) #define LPTE_I EXTEND_PTE( PTE_I ) #define LPTE_M EXTEND_PTE( PTE_M ) #define LPTE_G EXTEND_PTE( PTE_G ) #define LPTE_NOEXEC 0x0000000000000004ULL #define LPTE_PP EXTEND_PTE( PTE_PP ) #define LPTE_SO EXTEND_PTE( PTE_SO ) /* Super. Only */ #define LPTE_SW EXTEND_PTE( PTE_SW ) /* Super. Write-Only */ #define LPTE_BW EXTEND_PTE( PTE_BW ) /* Supervisor */ #define LPTE_BR EXTEND_PTE( PTE_BR ) /* Both Read Only */ #define LPTE_RW LPTE_BW #define LPTE_RO LPTE_BR #ifndef LOCORE typedef struct pte pte_t; typedef struct lpte lpte_t; #endif /* LOCORE */ /* * Extract bits from address */ #define ADDR_SR_SHFT 28 #define ADDR_PIDX 0x0ffff000UL #define ADDR_PIDX_SHFT 12 #define ADDR_API_SHFT 22 #define ADDR_API_SHFT64 16 #define ADDR_POFF 0x00000fffUL /* * Bits in DSISR: */ #define DSISR_DIRECT 0x80000000 #define DSISR_NOTFOUND 0x40000000 #define DSISR_PROTECT 0x08000000 #define DSISR_INVRX 0x04000000 #define DSISR_STORE 0x02000000 #define DSISR_DABR 0x00400000 #define DSISR_SEGMENT 0x00200000 #define DSISR_EAR 0x00100000 /* * Bits in SRR1 on ISI: */ #define ISSRR1_NOTFOUND 0x40000000 #define ISSRR1_DIRECT 0x10000000 #define ISSRR1_PROTECT 0x08000000 #define ISSRR1_SEGMENT 0x00200000 #else /* BOOKE */ #include /* * 1st level - page table directory (pdir) * * pdir consists of 1024 entries, each being a pointer to * second level entity, i.e. the actual page table (ptbl). */ #define PDIR_SHIFT 22 #define PDIR_SIZE (1 << PDIR_SHIFT) /* va range mapped by pdir */ #define PDIR_MASK (~(PDIR_SIZE - 1)) #define PDIR_NENTRIES 1024 /* number of page tables in pdir */ /* Returns pdir entry number for given va */ #define PDIR_IDX(va) ((va) >> PDIR_SHIFT) #define PDIR_ENTRY_SHIFT 2 /* entry size is 2^2 = 4 bytes */ /* * 2nd level - page table (ptbl) * * Page table covers 1024 page table entries. Page * table entry (pte) is 32 bit wide and defines mapping * for a single page. */ #define PTBL_SHIFT PAGE_SHIFT #define PTBL_SIZE PAGE_SIZE /* va range mapped by ptbl entry */ #define PTBL_MASK ((PDIR_SIZE - 1) & ~((1 << PAGE_SHIFT) - 1)) #define PTBL_NENTRIES 1024 /* number of pages mapped by ptbl */ /* Returns ptbl entry number for given va */ #define PTBL_IDX(va) (((va) & PTBL_MASK) >> PTBL_SHIFT) /* Size of ptbl in pages, 1024 entries, each sizeof(struct pte_entry). */ #define PTBL_PAGES 2 #define PTBL_ENTRY_SHIFT 3 /* entry size is 2^3 = 8 bytes */ /* * Flags for pte_remove() routine. */ #define PTBL_HOLD 0x00000001 /* do not unhold ptbl pages */ #define PTBL_UNHOLD 0x00000002 /* unhold and attempt to free ptbl pages */ #define PTBL_HOLD_FLAG(pmap) (((pmap) == kernel_pmap) ? PTBL_HOLD : PTBL_UNHOLD) /* * Page Table Entry definitions and macros. + * + * RPN need only be 32-bit because Book-E has 36-bit addresses, and the smallest + * page size is 4k (12-bit mask), so RPN can really fit into 24 bits. */ #ifndef LOCORE struct pte { - vm_paddr_t rpn; + vm_offset_t rpn; uint32_t flags; }; typedef struct pte pte_t; #endif /* RPN mask, TLB0 4K pages */ #define PTE_PA_MASK PAGE_MASK #if defined(BOOKE_E500) /* PTE bits assigned to MAS2, MAS3 flags */ #define PTE_W MAS2_W #define PTE_I MAS2_I #define PTE_M MAS2_M #define PTE_G MAS2_G #define PTE_MAS2_MASK (MAS2_G | MAS2_M | MAS2_I | MAS2_W) #define PTE_MAS3_SHIFT 8 #define PTE_UX (MAS3_UX << PTE_MAS3_SHIFT) #define PTE_SX (MAS3_SX << PTE_MAS3_SHIFT) #define PTE_UW (MAS3_UW << PTE_MAS3_SHIFT) #define PTE_SW (MAS3_SW << PTE_MAS3_SHIFT) #define PTE_UR (MAS3_UR << PTE_MAS3_SHIFT) #define PTE_SR (MAS3_SR << PTE_MAS3_SHIFT) #define PTE_MAS3_MASK ((MAS3_UX | MAS3_SX | MAS3_UW \ | MAS3_SW | MAS3_UR | MAS3_SR) << PTE_MAS3_SHIFT) #elif defined(BOOKE_PPC4XX) #define PTE_WL1 TLB_WL1 #define PTE_IL2I TLB_IL2I #define PTE_IL2D TLB_IL2D #define PTE_W TLB_W #define PTE_I TLB_I #define PTE_M TLB_M #define PTE_G TLB_G #define PTE_UX TLB_UX #define PTE_SX TLB_SX #define PTE_UW TLB_UW #define PTE_SW TLB_SW #define PTE_UR TLB_UR #define PTE_SR TLB_SR #endif /* Other PTE flags */ #define PTE_VALID 0x80000000 /* Valid */ #define PTE_MODIFIED 0x40000000 /* Modified */ #define PTE_WIRED 0x20000000 /* Wired */ #define PTE_MANAGED 0x10000000 /* Managed */ #define PTE_REFERENCED 0x04000000 /* Referenced */ /* Macro argument must of pte_t type. */ -#define PTE_PA(pte) ((pte)->rpn & ~PTE_PA_MASK) +#define PTE_PA_SHIFT 12 +#define PTE_RPN_FROM_PA(pa) ((pa) >> PTE_PA_SHIFT) +#define PTE_PA(pte) ((vm_paddr_t)((pte)->rpn) << PTE_PA_SHIFT) #define PTE_ISVALID(pte) ((pte)->flags & PTE_VALID) #define PTE_ISWIRED(pte) ((pte)->flags & PTE_WIRED) #define PTE_ISMANAGED(pte) ((pte)->flags & PTE_MANAGED) #define PTE_ISMODIFIED(pte) ((pte)->flags & PTE_MODIFIED) #define PTE_ISREFERENCED(pte) ((pte)->flags & PTE_REFERENCED) #endif /* BOOKE */ #endif /* _MACHINE_PTE_H_ */ Index: head/sys/powerpc/include/tlb.h =================================================================== --- head/sys/powerpc/include/tlb.h (revision 287014) +++ head/sys/powerpc/include/tlb.h (revision 287015) @@ -1,217 +1,220 @@ /*- * Copyright (C) 2006-2012 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_TLB_H_ #define _MACHINE_TLB_H_ #if defined(BOOKE_E500) /* PowerPC E500 MAS registers */ #define MAS0_TLBSEL(x) ((x << 28) & 0x10000000) #define MAS0_ESEL(x) ((x << 16) & 0x000F0000) #define MAS0_TLBSEL1 0x10000000 #define MAS0_TLBSEL0 0x00000000 #define MAS0_ESEL_TLB1MASK 0x000F0000 #define MAS0_ESEL_TLB0MASK 0x00030000 #define MAS0_ESEL_SHIFT 16 #define MAS0_NV_MASK 0x00000003 #define MAS0_NV_SHIFT 0 #define MAS1_VALID 0x80000000 #define MAS1_IPROT 0x40000000 #define MAS1_TID_MASK 0x00FF0000 #define MAS1_TID_SHIFT 16 #define MAS1_TS_MASK 0x00001000 #define MAS1_TS_SHIFT 12 #define MAS1_TSIZE_MASK 0x00000F00 #define MAS1_TSIZE_SHIFT 8 #define TLB_SIZE_4K 1 #define TLB_SIZE_16K 2 #define TLB_SIZE_64K 3 #define TLB_SIZE_256K 4 #define TLB_SIZE_1M 5 #define TLB_SIZE_4M 6 #define TLB_SIZE_16M 7 #define TLB_SIZE_64M 8 #define TLB_SIZE_256M 9 #define TLB_SIZE_1G 10 #define TLB_SIZE_4G 11 #define MAS2_EPN_MASK 0xFFFFF000 #define MAS2_EPN_SHIFT 12 #define MAS2_X0 0x00000040 #define MAS2_X1 0x00000020 #define MAS2_W 0x00000010 #define MAS2_I 0x00000008 #define MAS2_M 0x00000004 #define MAS2_G 0x00000002 #define MAS2_E 0x00000001 #define MAS3_RPN 0xFFFFF000 #define MAS3_RPN_SHIFT 12 #define MAS3_U0 0x00000200 #define MAS3_U1 0x00000100 #define MAS3_U2 0x00000080 #define MAS3_U3 0x00000040 #define MAS3_UX 0x00000020 #define MAS3_SX 0x00000010 #define MAS3_UW 0x00000008 #define MAS3_SW 0x00000004 #define MAS3_UR 0x00000002 #define MAS3_SR 0x00000001 #define MAS4_TLBSELD1 0x10000000 #define MAS4_TLBSELD0 0x00000000 #define MAS4_TIDSELD_MASK 0x00030000 #define MAS4_TIDSELD_SHIFT 16 #define MAS4_TSIZED_MASK 0x00000F00 #define MAS4_TSIZED_SHIFT 8 #define MAS4_X0D 0x00000040 #define MAS4_X1D 0x00000020 #define MAS4_WD 0x00000010 #define MAS4_ID 0x00000008 #define MAS4_MD 0x00000004 #define MAS4_GD 0x00000002 #define MAS4_ED 0x00000001 #define MAS6_SPID0_MASK 0x00FF0000 #define MAS6_SPID0_SHIFT 16 #define MAS6_SAS 0x00000001 +#define MAS7_RPN 0x0000000F + #define MAS1_GETTID(mas1) (((mas1) & MAS1_TID_MASK) >> MAS1_TID_SHIFT) #define MAS2_TLB0_ENTRY_IDX_MASK 0x0007f000 #define MAS2_TLB0_ENTRY_IDX_SHIFT 12 /* * Maximum number of TLB1 entries used for a permanent mapping of kernel * region (kernel image plus statically allocated data). */ #define KERNEL_REGION_MAX_TLB_ENTRIES 4 #define _TLB_ENTRY_IO (MAS2_I | MAS2_G) #ifdef SMP #define _TLB_ENTRY_MEM (MAS2_M) #else #define _TLB_ENTRY_MEM (0) #endif #if !defined(LOCORE) typedef struct tlb_entry { vm_paddr_t phys; vm_offset_t virt; vm_size_t size; uint32_t mas1; uint32_t mas2; uint32_t mas3; + uint32_t mas7; } tlb_entry_t; void tlb0_print_tlbentries(void); void tlb1_inval_entry(unsigned int); void tlb1_init(void); void tlb1_print_entries(void); void tlb1_print_tlbentries(void); #endif /* !LOCORE */ #elif defined(BOOKE_PPC4XX) /* TLB Words */ #define TLB_PAGEID 0 #define TLB_XLAT 1 #define TLB_ATTRIB 2 /* Page identification fields */ #define TLB_EPN_MASK (0xFFFFFC00 >> 0) #define TLB_VALID (0x80000000 >> 22) #define TLB_TS (0x80000000 >> 23) #define TLB_SIZE_1K (0x00000000 >> 24) #define TLB_SIZE_MASK (0xF0000000 >> 24) /* Translation fields */ #define TLB_RPN_MASK (0xFFFFFC00 >> 0) #define TLB_ERPN_MASK (0xF0000000 >> 28) /* Storage attribute and access control fields */ #define TLB_WL1 (0x80000000 >> 11) #define TLB_IL1I (0x80000000 >> 12) #define TLB_IL1D (0x80000000 >> 13) #define TLB_IL2I (0x80000000 >> 14) #define TLB_IL2D (0x80000000 >> 15) #define TLB_U0 (0x80000000 >> 16) #define TLB_U1 (0x80000000 >> 17) #define TLB_U2 (0x80000000 >> 18) #define TLB_U3 (0x80000000 >> 19) #define TLB_W (0x80000000 >> 20) #define TLB_I (0x80000000 >> 21) #define TLB_M (0x80000000 >> 22) #define TLB_G (0x80000000 >> 23) #define TLB_E (0x80000000 >> 24) #define TLB_UX (0x80000000 >> 26) #define TLB_UW (0x80000000 >> 27) #define TLB_UR (0x80000000 >> 28) #define TLB_SX (0x80000000 >> 29) #define TLB_SW (0x80000000 >> 30) #define TLB_SR (0x80000000 >> 31) #define TLB_SIZE 64 #define TLB_SIZE_4K (0x10000000 >> 24) #define TLB_SIZE_16K (0x20000000 >> 24) #define TLB_SIZE_64K (0x30000000 >> 24) #define TLB_SIZE_256K (0x40000000 >> 24) #define TLB_SIZE_1M (0x50000000 >> 24) #define TLB_SIZE_16M (0x70000000 >> 24) #define TLB_SIZE_256M (0x90000000 >> 24) #define TLB_SIZE_1G (0xA0000000 >> 24) #endif /* BOOKE_E500 */ #define TID_KERNEL 0 /* TLB TID to use for kernel (shared) translations */ #define TID_KRESERVED 1 /* Number of TIDs reserved for kernel */ #define TID_URESERVED 0 /* Number of TIDs reserved for user */ #define TID_MIN (TID_KRESERVED + TID_URESERVED) #define TID_MAX 255 #define TID_NONE -1 #define TLB_UNLOCKED 0 #if !defined(LOCORE) typedef int tlbtid_t; struct pmap; void tlb_lock(uint32_t *); void tlb_unlock(uint32_t *); #endif /* !LOCORE */ #endif /* _MACHINE_TLB_H_ */ Index: head/sys/powerpc/powerpc/uma_machdep.c =================================================================== --- head/sys/powerpc/powerpc/uma_machdep.c (revision 287014) +++ head/sys/powerpc/powerpc/uma_machdep.c (revision 287015) @@ -1,98 +1,105 @@ /*- * Copyright (c) 2003 The FreeBSD Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int hw_uma_mdpages; SYSCTL_INT(_hw, OID_AUTO, uma_mdpages, CTLFLAG_RD, &hw_uma_mdpages, 0, "UMA MD pages in use"); void * uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait) { void *va; + vm_paddr_t pa; vm_page_t m; int pflags; *flags = UMA_SLAB_PRIV; pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED; for (;;) { m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ); if (m == NULL) { if (wait & M_NOWAIT) return (NULL); VM_WAIT; } else break; } - va = (void *) VM_PAGE_TO_PHYS(m); + pa = VM_PAGE_TO_PHYS(m); + + /* On book-e sizeof(void *) < sizeof(vm_paddr_t) */ + if ((vm_offset_t)pa != pa) + return (NULL); + + va = (void *)(vm_offset_t)pa; if (!hw_direct_map) pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m)); if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) bzero(va, PAGE_SIZE); atomic_add_int(&hw_uma_mdpages, 1); return (va); } void uma_small_free(void *mem, vm_size_t size, u_int8_t flags) { vm_page_t m; if (!hw_direct_map) pmap_remove(kernel_pmap,(vm_offset_t)mem, (vm_offset_t)mem + PAGE_SIZE); m = PHYS_TO_VM_PAGE((vm_offset_t)mem); m->wire_count--; vm_page_free(m); atomic_subtract_int(&vm_cnt.v_wire_count, 1); atomic_subtract_int(&hw_uma_mdpages, 1); }