Index: head/sys/amd64/acpica/acpi_wakecode.S =================================================================== --- head/sys/amd64/acpica/acpi_wakecode.S (revision 331253) +++ head/sys/amd64/acpica/acpi_wakecode.S (revision 331254) @@ -1,285 +1,285 @@ /*- * Copyright (c) 2001 Takanori Watanabe * Copyright (c) 2001 Mitsuru IWASAKI * Copyright (c) 2003 Peter Wemm * Copyright (c) 2008-2012 Jung-uk Kim * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include -#include "assym.s" +#include "assym.inc" /* * Resume entry point for real mode. * * If XFirmwareWakingVector is zero and FirmwareWakingVector is non-zero * in FACS, the BIOS enters here in real mode after POST with CS set to * (FirmwareWakingVector >> 4) and IP set to (FirmwareWakingVector & 0xf). * Depending on the previous sleep state, we may need to initialize more * of the system (i.e., S3 suspend-to-RAM vs. S4 suspend-to-disk). * * Note: If XFirmwareWakingVector is non-zero, it should disable address * translation/paging and interrupts, load all segment registers with * a flat 4 GB address space, and set EFLAGS.IF to zero. Currently * this mode is not supported by this code. */ .data /* So we can modify it */ ALIGN_TEXT .code16 wakeup_start: /* * Set up segment registers for real mode, a small stack for * any calls we make, and clear any flags. */ cli /* make sure no interrupts */ mov %cs, %ax /* copy %cs to %ds. Remember these */ mov %ax, %ds /* are offsets rather than selectors */ mov %ax, %ss movw $PAGE_SIZE, %sp xorw %ax, %ax pushw %ax popfw /* To debug resume hangs, beep the speaker if the user requested. */ testb $~0, resume_beep - wakeup_start jz 1f movb $0, resume_beep - wakeup_start /* Set PIC timer2 to beep. */ movb $(TIMER_SEL2 | TIMER_SQWAVE | TIMER_16BIT), %al outb %al, $TIMER_MODE /* Turn on speaker. */ inb $IO_PPI, %al orb $PIT_SPKR, %al outb %al, $IO_PPI /* Set frequency. */ movw $0x4c0, %ax outb %al, $TIMER_CNTR2 shrw $8, %ax outb %al, $TIMER_CNTR2 1: /* Re-initialize video BIOS if the reset_video tunable is set. */ testb $~0, reset_video - wakeup_start jz 1f movb $0, reset_video - wakeup_start lcall $0xc000, $3 /* When we reach here, int 0x10 should be ready. Hide cursor. */ movb $0x01, %ah movb $0x20, %ch int $0x10 /* Re-start in case the previous BIOS call clobbers them. */ jmp wakeup_start 1: /* * Find relocation base and patch the gdt descript and ljmp targets */ xorl %ebx, %ebx mov %cs, %bx sall $4, %ebx /* %ebx is now our relocation base */ /* * Load the descriptor table pointer. We'll need it when running * in 16-bit protected mode. */ lgdtl bootgdtdesc - wakeup_start /* Enable protected mode */ movl $CR0_PE, %eax mov %eax, %cr0 /* * Now execute a far jump to turn on protected mode. This * causes the segment registers to turn into selectors and causes * %cs to be loaded from the gdt. * * The following instruction is: * ljmpl $bootcode32 - bootgdt, $wakeup_32 - wakeup_start * but gas cannot assemble that. And besides, we patch the targets * in early startup and its a little clearer what we are patching. */ wakeup_sw32: .byte 0x66 /* size override to 32 bits */ .byte 0xea /* opcode for far jump */ .long wakeup_32 - wakeup_start /* offset in segment */ .word bootcode32 - bootgdt /* index in gdt for 32 bit code */ /* * At this point, we are running in 32 bit legacy protected mode. */ ALIGN_TEXT .code32 wakeup_32: mov $bootdata32 - bootgdt, %eax mov %ax, %ds /* Turn on the PAE bit for when paging is enabled */ mov %cr4, %eax orl $CR4_PAE, %eax mov %eax, %cr4 /* * Enable EFER.LME so that we get long mode when all the prereqs are * in place. In this case, it turns on when CR0_PG is finally enabled. * Also it picks up a few other EFER bits that we'll use need we're * here, like SYSCALL and NX enable. */ movl $MSR_EFER, %ecx movl wakeup_efer - wakeup_start(%ebx), %eax movl wakeup_efer + 4 - wakeup_start(%ebx), %edx wrmsr /* * Point to the embedded page tables for startup. Note that this * only gets accessed after we're actually in 64 bit mode, however * we can only set the bottom 32 bits of %cr3 in this state. This * means we are required to use a temporary page table that is below * the 4GB limit. %ebx is still our relocation base. We could just * subtract 3 * PAGE_SIZE, but that would be too easy. */ leal wakeup_pagetables - wakeup_start(%ebx), %eax movl (%eax), %eax mov %eax, %cr3 /* * Finally, switch to long bit mode by enabling paging. We have * to be very careful here because all the segmentation disappears * out from underneath us. The spec says we can depend on the * subsequent pipelined branch to execute, but *only if* everything * is still identity mapped. If any mappings change, the pipeline * will flush. */ mov %cr0, %eax orl $CR0_PG, %eax mov %eax, %cr0 /* * At this point paging is enabled, and we are in "compatibility" mode. * We do another far jump to reload %cs with the 64 bit selector. * %cr3 points to a 4-level page table page. * We cannot yet jump all the way to the kernel because we can only * specify a 32 bit linear address. So, yet another trampoline. * * The following instruction is: * ljmp $bootcode64 - bootgdt, $wakeup_64 - wakeup_start * but gas cannot assemble that. And besides, we patch the targets * in early startup and its a little clearer what we are patching. */ wakeup_sw64: .byte 0xea /* opcode for far jump */ .long wakeup_64 - wakeup_start /* offset in segment */ .word bootcode64 - bootgdt /* index in gdt for 64 bit code */ /* * Yeehar! We're running in 64-bit mode! We can mostly ignore our * segment registers, and get on with it. * Note that we are running at the correct virtual address, but with * a 1:1 1GB mirrored mapping over entire address space. We had better * switch to a real %cr3 promptly so that we can get to the direct map * space. Remember that jmp is relative and that we've been relocated, * so use an indirect jump. */ ALIGN_TEXT .code64 wakeup_64: mov $bootdata64 - bootgdt, %eax mov %ax, %ds /* Restore arguments. */ movq wakeup_pcb - wakeup_start(%rbx), %rdi movq wakeup_ret - wakeup_start(%rbx), %rax /* Restore GDT. */ lgdt wakeup_gdt - wakeup_start(%rbx) /* Jump to return address. */ jmp *%rax .data resume_beep: .byte 0 reset_video: .byte 0 ALIGN_DATA bootgdt: .long 0x00000000 .long 0x00000000 .long 0x00000000 .long 0x00000000 .long 0x00000000 .long 0x00000000 .long 0x00000000 .long 0x00000000 bootcode64: .long 0x0000ffff .long 0x00af9b00 bootdata64: .long 0x0000ffff .long 0x00af9300 bootcode32: .long 0x0000ffff .long 0x00cf9b00 bootdata32: .long 0x0000ffff .long 0x00cf9300 bootgdtend: wakeup_pagetables: .long 0 bootgdtdesc: .word bootgdtend - bootgdt /* Length */ .long bootgdt - wakeup_start /* Offset plus %ds << 4 */ ALIGN_DATA wakeup_pcb: .quad 0 wakeup_ret: .quad 0 wakeup_efer: .quad 0 wakeup_gdt: .word 0 .quad 0 dummy: Index: head/sys/amd64/amd64/apic_vector.S =================================================================== --- head/sys/amd64/amd64/apic_vector.S (revision 331253) +++ head/sys/amd64/amd64/apic_vector.S (revision 331254) @@ -1,307 +1,307 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * Copyright (c) 2014-2018 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by * Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD$ */ /* * Interrupt entry points for external interrupts triggered by I/O APICs * as well as IPI handlers. */ #include "opt_smp.h" -#include "assym.s" +#include "assym.inc" #include #include #include #ifdef SMP #define LK lock ; #else #define LK #endif .text SUPERALIGN_TEXT /* End Of Interrupt to APIC */ as_lapic_eoi: cmpl $0,x2apic_mode jne 1f movq lapic_map,%rax movl $0,LA_EOI(%rax) ret 1: movl $MSR_APIC_EOI,%ecx xorl %eax,%eax xorl %edx,%edx wrmsr ret /* * I/O Interrupt Entry Point. Rather than having one entry point for * each interrupt source, we use one entry point for each 32-bit word * in the ISR. The handler determines the highest bit set in the ISR, * translates that into a vector, and passes the vector to the * lapic_handle_intr() function. */ .macro ISR_VEC index, vec_name INTR_HANDLER \vec_name FAKE_MCOUNT(TF_RIP(%rsp)) cmpl $0,x2apic_mode je 1f movl $(MSR_APIC_ISR0 + \index),%ecx rdmsr jmp 2f 1: movq lapic_map, %rdx /* pointer to local APIC */ movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */ 2: bsrl %eax, %eax /* index of highest set bit in ISR */ jz 3f addl $(32 * \index),%eax movq %rsp, %rsi movl %eax, %edi /* pass the IRQ */ call lapic_handle_intr 3: MEXITCOUNT jmp doreti .endm /* * Handle "spurious INTerrupts". * Notes: * This is different than the "spurious INTerrupt" generated by an * 8259 PIC for missing INTs. See the APIC documentation for details. * This routine should NOT do an 'EOI' cycle. */ .text SUPERALIGN_TEXT IDTVEC(spuriousint) /* No EOI cycle used here */ jmp doreti_iret ISR_VEC 1, apic_isr1 ISR_VEC 2, apic_isr2 ISR_VEC 3, apic_isr3 ISR_VEC 4, apic_isr4 ISR_VEC 5, apic_isr5 ISR_VEC 6, apic_isr6 ISR_VEC 7, apic_isr7 /* * Local APIC periodic timer handler. */ INTR_HANDLER timerint FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp, %rdi call lapic_handle_timer MEXITCOUNT jmp doreti /* * Local APIC CMCI handler. */ INTR_HANDLER cmcint FAKE_MCOUNT(TF_RIP(%rsp)) call lapic_handle_cmc MEXITCOUNT jmp doreti /* * Local APIC error interrupt handler. */ INTR_HANDLER errorint FAKE_MCOUNT(TF_RIP(%rsp)) call lapic_handle_error MEXITCOUNT jmp doreti #ifdef XENHVM /* * Xen event channel upcall interrupt handler. * Only used when the hypervisor supports direct vector callbacks. */ INTR_HANDLER xen_intr_upcall FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp, %rdi call xen_intr_handle_upcall MEXITCOUNT jmp doreti #endif #ifdef SMP /* * Global address space TLB shootdown. */ .text SUPERALIGN_TEXT invltlb_ret: call as_lapic_eoi jmp ld_regs SUPERALIGN_TEXT INTR_HANDLER invltlb call invltlb_handler jmp invltlb_ret INTR_HANDLER invltlb_pcid call invltlb_pcid_handler jmp invltlb_ret INTR_HANDLER invltlb_invpcid_nopti call invltlb_invpcid_handler jmp invltlb_ret INTR_HANDLER invltlb_invpcid_pti call invltlb_invpcid_pti_handler jmp invltlb_ret /* * Single page TLB shootdown */ INTR_HANDLER invlpg call invlpg_handler jmp invltlb_ret INTR_HANDLER invlpg_invpcid call invlpg_invpcid_handler jmp invltlb_ret INTR_HANDLER invlpg_pcid call invlpg_pcid_handler jmp invltlb_ret /* * Page range TLB shootdown. */ INTR_HANDLER invlrng call invlrng_handler jmp invltlb_ret INTR_HANDLER invlrng_invpcid call invlrng_invpcid_handler jmp invltlb_ret INTR_HANDLER invlrng_pcid call invlrng_pcid_handler jmp invltlb_ret /* * Invalidate cache. */ INTR_HANDLER invlcache call invlcache_handler jmp invltlb_ret /* * Handler for IPIs sent via the per-cpu IPI bitmap. */ INTR_HANDLER ipi_intr_bitmap_handler call as_lapic_eoi FAKE_MCOUNT(TF_RIP(%rsp)) call ipi_bitmap_handler MEXITCOUNT jmp doreti /* * Executed by a CPU when it receives an IPI_STOP from another CPU. */ INTR_HANDLER cpustop call as_lapic_eoi call cpustop_handler jmp doreti /* * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. */ INTR_HANDLER cpususpend call cpususpend_handler call as_lapic_eoi jmp doreti /* * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. * * - Calls the generic rendezvous action function. */ INTR_HANDLER rendezvous #ifdef COUNT_IPIS movl PCPU(CPUID), %eax movq ipi_rendezvous_counts(,%rax,8), %rax incq (%rax) #endif call smp_rendezvous_action call as_lapic_eoi jmp doreti /* * IPI handler whose purpose is to interrupt the CPU with minimum overhead. * This is used by bhyve to force a host cpu executing in guest context to * trap into the hypervisor. * * This handler is different from other IPI handlers in the following aspects: * * 1. It doesn't push a trapframe on the stack. * * This implies that a DDB backtrace involving 'justreturn' will skip the * function that was interrupted by this handler. * * 2. It doesn't 'swapgs' when userspace is interrupted. * * The 'justreturn' handler does not access any pcpu data so it is not an * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI * whose handler already doesn't trust GS.base when kernel code is interrupted. */ .text SUPERALIGN_TEXT IDTVEC(justreturn) pushq %rax pushq %rcx pushq %rdx call as_lapic_eoi popq %rdx popq %rcx popq %rax jmp doreti_iret INTR_HANDLER justreturn1 call as_lapic_eoi jmp doreti #endif /* SMP */ Index: head/sys/amd64/amd64/atpic_vector.S =================================================================== --- head/sys/amd64/amd64/atpic_vector.S (revision 331253) +++ head/sys/amd64/amd64/atpic_vector.S (revision 331254) @@ -1,70 +1,70 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD$ */ /* * Interrupt entry points for external interrupts triggered by the 8259A * master and slave interrupt controllers. */ -#include "assym.s" +#include "assym.inc" #include /* * Macros for interrupt entry, call to handler, and exit. */ .macro INTR irq_num, vec_name INTR_HANDLER \vec_name FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp, %rsi movl $\irq_num, %edi /* pass the IRQ */ call atpic_handle_intr MEXITCOUNT jmp doreti .endm INTR 0, atpic_intr0 INTR 1, atpic_intr1 INTR 2, atpic_intr2 INTR 3, atpic_intr3 INTR 4, atpic_intr4 INTR 5, atpic_intr5 INTR 6, atpic_intr6 INTR 7, atpic_intr7 INTR 8, atpic_intr8 INTR 9, atpic_intr9 INTR 10, atpic_intr10 INTR 11, atpic_intr11 INTR 12, atpic_intr12 INTR 13, atpic_intr13 INTR 14, atpic_intr14 INTR 15, atpic_intr15 Index: head/sys/amd64/amd64/cpu_switch.S =================================================================== --- head/sys/amd64/amd64/cpu_switch.S (revision 331253) +++ head/sys/amd64/amd64/cpu_switch.S (revision 331254) @@ -1,506 +1,506 @@ /*- * Copyright (c) 2003 Peter Wemm. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include -#include "assym.s" +#include "assym.inc" #include "opt_sched.h" /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ .text #ifdef SMP #define LK lock ; #else #define LK #endif #if defined(SCHED_ULE) && defined(SMP) #define SETLK xchgq #else #define SETLK movq #endif /* * cpu_throw() * * This is the second half of cpu_switch(). It is used when the current * thread is either a dummy or slated to die, and we no longer care * about its state. This is only a slight optimization and is probably * not worth it anymore. Note that we need to clear the pm_active bits so * we do need the old proc if it still exists. * %rdi = oldtd * %rsi = newtd */ ENTRY(cpu_throw) movq %rsi,%r12 movq %rsi,%rdi call pmap_activate_sw jmp sw1 END(cpu_throw) /* * cpu_switch(old, new, mtx) * * Save the current thread state, then select the next thread to run * and load its state. * %rdi = oldtd * %rsi = newtd * %rdx = mtx */ ENTRY(cpu_switch) /* Switch to new thread. First, save context. */ movq TD_PCB(%rdi),%r8 movq (%rsp),%rax /* Hardware registers */ movq %r15,PCB_R15(%r8) movq %r14,PCB_R14(%r8) movq %r13,PCB_R13(%r8) movq %r12,PCB_R12(%r8) movq %rbp,PCB_RBP(%r8) movq %rsp,PCB_RSP(%r8) movq %rbx,PCB_RBX(%r8) movq %rax,PCB_RIP(%r8) testl $PCB_FULL_IRET,PCB_FLAGS(%r8) jnz 2f orl $PCB_FULL_IRET,PCB_FLAGS(%r8) testl $TDP_KTHREAD,TD_PFLAGS(%rdi) jnz 2f testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) jz 2f movl %fs,%eax cmpl $KUF32SEL,%eax jne 1f rdfsbase %rax movq %rax,PCB_FSBASE(%r8) 1: movl %gs,%eax cmpl $KUG32SEL,%eax jne 2f movq %rdx,%r12 movl $MSR_KGSBASE,%ecx /* Read user gs base */ rdmsr shlq $32,%rdx orq %rdx,%rax movq %rax,PCB_GSBASE(%r8) movq %r12,%rdx 2: testl $PCB_DBREGS,PCB_FLAGS(%r8) jnz store_dr /* static predict not taken */ done_store_dr: /* have we used fp, and need a save? */ cmpq %rdi,PCPU(FPCURTHREAD) jne 3f movq PCB_SAVEFPU(%r8),%r8 clts cmpl $0,use_xsave jne 1f fxsave (%r8) jmp 2f 1: movq %rdx,%rcx movl xsave_mask,%eax movl xsave_mask+4,%edx .globl ctx_switch_xsave ctx_switch_xsave: /* This is patched to xsaveopt if supported, see fpuinit_bsp1() */ xsave (%r8) movq %rcx,%rdx 2: smsw %ax orb $CR0_TS,%al lmsw %ax xorl %eax,%eax movq %rax,PCPU(FPCURTHREAD) 3: /* Save is done. Now fire up new thread. Leave old vmspace. */ movq %rsi,%r12 movq %rdi,%r13 movq %rdx,%r15 movq %rsi,%rdi callq pmap_activate_sw SETLK %r15,TD_LOCK(%r13) /* Release the old thread */ sw1: movq TD_PCB(%r12),%r8 #if defined(SCHED_ULE) && defined(SMP) /* Wait for the new thread to become unblocked */ movq $blocked_lock, %rdx 1: movq TD_LOCK(%r12),%rcx cmpq %rcx, %rdx pause je 1b #endif /* * At this point, we've switched address spaces and are ready * to load up the rest of the next context. */ /* Skip loading LDT and user fsbase/gsbase for kthreads */ testl $TDP_KTHREAD,TD_PFLAGS(%r12) jnz do_kthread /* * Load ldt register */ movq TD_PROC(%r12),%rcx cmpq $0, P_MD+MD_LDT(%rcx) jne do_ldt xorl %eax,%eax ld_ldt: lldt %ax /* Restore fs base in GDT */ movl PCB_FSBASE(%r8),%eax movq PCPU(FS32P),%rdx movw %ax,2(%rdx) shrl $16,%eax movb %al,4(%rdx) shrl $8,%eax movb %al,7(%rdx) /* Restore gs base in GDT */ movl PCB_GSBASE(%r8),%eax movq PCPU(GS32P),%rdx movw %ax,2(%rdx) shrl $16,%eax movb %al,4(%rdx) shrl $8,%eax movb %al,7(%rdx) do_kthread: /* Do we need to reload tss ? */ movq PCPU(TSSP),%rax movq PCB_TSSP(%r8),%rdx testq %rdx,%rdx cmovzq PCPU(COMMONTSSP),%rdx cmpq %rax,%rdx jne do_tss done_tss: movq %r8,PCPU(RSP0) movq %r8,PCPU(CURPCB) /* Update the TSS_RSP0 pointer for the next interrupt */ cmpb $0,pti(%rip) jne 1f movq %r8,TSS_RSP0(%rdx) 1: movq %r12,PCPU(CURTHREAD) /* into next thread */ /* Test if debug registers should be restored. */ testl $PCB_DBREGS,PCB_FLAGS(%r8) jnz load_dr /* static predict not taken */ done_load_dr: /* Restore context. */ movq PCB_R15(%r8),%r15 movq PCB_R14(%r8),%r14 movq PCB_R13(%r8),%r13 movq PCB_R12(%r8),%r12 movq PCB_RBP(%r8),%rbp movq PCB_RSP(%r8),%rsp movq PCB_RBX(%r8),%rbx movq PCB_RIP(%r8),%rax movq %rax,(%rsp) ret /* * We order these strangely for several reasons. * 1: I wanted to use static branch prediction hints * 2: Most athlon64/opteron cpus don't have them. They define * a forward branch as 'predict not taken'. Intel cores have * the 'rep' prefix to invert this. * So, to make it work on both forms of cpu we do the detour. * We use jumps rather than call in order to avoid the stack. */ store_dr: movq %dr7,%rax /* yes, do the save */ movq %dr0,%r15 movq %dr1,%r14 movq %dr2,%r13 movq %dr3,%r12 movq %dr6,%r11 movq %r15,PCB_DR0(%r8) movq %r14,PCB_DR1(%r8) movq %r13,PCB_DR2(%r8) movq %r12,PCB_DR3(%r8) movq %r11,PCB_DR6(%r8) movq %rax,PCB_DR7(%r8) andq $0x0000fc00, %rax /* disable all watchpoints */ movq %rax,%dr7 jmp done_store_dr load_dr: movq %dr7,%rax movq PCB_DR0(%r8),%r15 movq PCB_DR1(%r8),%r14 movq PCB_DR2(%r8),%r13 movq PCB_DR3(%r8),%r12 movq PCB_DR6(%r8),%r11 movq PCB_DR7(%r8),%rcx movq %r15,%dr0 movq %r14,%dr1 /* Preserve reserved bits in %dr7 */ andq $0x0000fc00,%rax andq $~0x0000fc00,%rcx movq %r13,%dr2 movq %r12,%dr3 orq %rcx,%rax movq %r11,%dr6 movq %rax,%dr7 jmp done_load_dr do_tss: movq %rdx,PCPU(TSSP) movq %rdx,%rcx movq PCPU(TSS),%rax movw %cx,2(%rax) shrq $16,%rcx movb %cl,4(%rax) shrq $8,%rcx movb %cl,7(%rax) shrq $8,%rcx movl %ecx,8(%rax) movb $0x89,5(%rax) /* unset busy */ cmpb $0,pti(%rip) je 1f movq PCPU(PRVSPACE),%rax addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax movq %rax,TSS_RSP0(%rdx) 1: movl $TSSSEL,%eax ltr %ax jmp done_tss do_ldt: movq PCPU(LDT),%rax movq P_MD+MD_LDT_SD(%rcx),%rdx movq %rdx,(%rax) movq P_MD+MD_LDT_SD+8(%rcx),%rdx movq %rdx,8(%rax) movl $LDTSEL,%eax jmp ld_ldt END(cpu_switch) /* * savectx(pcb) * Update pcb, saving current processor state. */ ENTRY(savectx) /* Save caller's return address. */ movq (%rsp),%rax movq %rax,PCB_RIP(%rdi) movq %rbx,PCB_RBX(%rdi) movq %rsp,PCB_RSP(%rdi) movq %rbp,PCB_RBP(%rdi) movq %r12,PCB_R12(%rdi) movq %r13,PCB_R13(%rdi) movq %r14,PCB_R14(%rdi) movq %r15,PCB_R15(%rdi) movq %cr0,%rax movq %rax,PCB_CR0(%rdi) movq %cr2,%rax movq %rax,PCB_CR2(%rdi) movq %cr3,%rax movq %rax,PCB_CR3(%rdi) movq %cr4,%rax movq %rax,PCB_CR4(%rdi) movq %dr0,%rax movq %rax,PCB_DR0(%rdi) movq %dr1,%rax movq %rax,PCB_DR1(%rdi) movq %dr2,%rax movq %rax,PCB_DR2(%rdi) movq %dr3,%rax movq %rax,PCB_DR3(%rdi) movq %dr6,%rax movq %rax,PCB_DR6(%rdi) movq %dr7,%rax movq %rax,PCB_DR7(%rdi) movl $MSR_FSBASE,%ecx rdmsr movl %eax,PCB_FSBASE(%rdi) movl %edx,PCB_FSBASE+4(%rdi) movl $MSR_GSBASE,%ecx rdmsr movl %eax,PCB_GSBASE(%rdi) movl %edx,PCB_GSBASE+4(%rdi) movl $MSR_KGSBASE,%ecx rdmsr movl %eax,PCB_KGSBASE(%rdi) movl %edx,PCB_KGSBASE+4(%rdi) movl $MSR_EFER,%ecx rdmsr movl %eax,PCB_EFER(%rdi) movl %edx,PCB_EFER+4(%rdi) movl $MSR_STAR,%ecx rdmsr movl %eax,PCB_STAR(%rdi) movl %edx,PCB_STAR+4(%rdi) movl $MSR_LSTAR,%ecx rdmsr movl %eax,PCB_LSTAR(%rdi) movl %edx,PCB_LSTAR+4(%rdi) movl $MSR_CSTAR,%ecx rdmsr movl %eax,PCB_CSTAR(%rdi) movl %edx,PCB_CSTAR+4(%rdi) movl $MSR_SF_MASK,%ecx rdmsr movl %eax,PCB_SFMASK(%rdi) movl %edx,PCB_SFMASK+4(%rdi) sgdt PCB_GDT(%rdi) sidt PCB_IDT(%rdi) sldt PCB_LDT(%rdi) str PCB_TR(%rdi) movl $1,%eax ret END(savectx) /* * resumectx(pcb) * Resuming processor state from pcb. */ ENTRY(resumectx) /* Switch to KPML4phys. */ movq KPML4phys,%rax movq %rax,%cr3 /* Force kernel segment registers. */ movl $KDSEL,%eax movw %ax,%ds movw %ax,%es movw %ax,%ss movl $KUF32SEL,%eax movw %ax,%fs movl $KUG32SEL,%eax movw %ax,%gs movl $MSR_FSBASE,%ecx movl PCB_FSBASE(%rdi),%eax movl 4 + PCB_FSBASE(%rdi),%edx wrmsr movl $MSR_GSBASE,%ecx movl PCB_GSBASE(%rdi),%eax movl 4 + PCB_GSBASE(%rdi),%edx wrmsr movl $MSR_KGSBASE,%ecx movl PCB_KGSBASE(%rdi),%eax movl 4 + PCB_KGSBASE(%rdi),%edx wrmsr /* Restore EFER one more time. */ movl $MSR_EFER,%ecx movl PCB_EFER(%rdi),%eax wrmsr /* Restore fast syscall stuff. */ movl $MSR_STAR,%ecx movl PCB_STAR(%rdi),%eax movl 4 + PCB_STAR(%rdi),%edx wrmsr movl $MSR_LSTAR,%ecx movl PCB_LSTAR(%rdi),%eax movl 4 + PCB_LSTAR(%rdi),%edx wrmsr movl $MSR_CSTAR,%ecx movl PCB_CSTAR(%rdi),%eax movl 4 + PCB_CSTAR(%rdi),%edx wrmsr movl $MSR_SF_MASK,%ecx movl PCB_SFMASK(%rdi),%eax wrmsr /* Restore CR0, CR2, CR4 and CR3. */ movq PCB_CR0(%rdi),%rax movq %rax,%cr0 movq PCB_CR2(%rdi),%rax movq %rax,%cr2 movq PCB_CR4(%rdi),%rax movq %rax,%cr4 movq PCB_CR3(%rdi),%rax movq %rax,%cr3 /* Restore descriptor tables. */ lidt PCB_IDT(%rdi) lldt PCB_LDT(%rdi) #define SDT_SYSTSS 9 #define SDT_SYSBSY 11 /* Clear "task busy" bit and reload TR. */ movq PCPU(TSS),%rax andb $(~SDT_SYSBSY | SDT_SYSTSS),5(%rax) movw PCB_TR(%rdi),%ax ltr %ax #undef SDT_SYSTSS #undef SDT_SYSBSY /* Restore debug registers. */ movq PCB_DR0(%rdi),%rax movq %rax,%dr0 movq PCB_DR1(%rdi),%rax movq %rax,%dr1 movq PCB_DR2(%rdi),%rax movq %rax,%dr2 movq PCB_DR3(%rdi),%rax movq %rax,%dr3 movq PCB_DR6(%rdi),%rax movq %rax,%dr6 movq PCB_DR7(%rdi),%rax movq %rax,%dr7 /* Restore other callee saved registers. */ movq PCB_R15(%rdi),%r15 movq PCB_R14(%rdi),%r14 movq PCB_R13(%rdi),%r13 movq PCB_R12(%rdi),%r12 movq PCB_RBP(%rdi),%rbp movq PCB_RSP(%rdi),%rsp movq PCB_RBX(%rdi),%rbx /* Restore return address. */ movq PCB_RIP(%rdi),%rax movq %rax,(%rsp) xorl %eax,%eax ret END(resumectx) Index: head/sys/amd64/amd64/exception.S =================================================================== --- head/sys/amd64/amd64/exception.S (revision 331253) +++ head/sys/amd64/amd64/exception.S (revision 331254) @@ -1,1172 +1,1172 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * Copyright (c) 2007-2018 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Portions of this software were developed by * Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_atpic.h" #include "opt_compat.h" #include "opt_hwpmc_hooks.h" -#include "assym.s" +#include "assym.inc" #include #include #include #include #ifdef KDTRACE_HOOKS .bss .globl dtrace_invop_jump_addr .align 8 .type dtrace_invop_jump_addr,@object .size dtrace_invop_jump_addr,8 dtrace_invop_jump_addr: .zero 8 .globl dtrace_invop_calltrap_addr .align 8 .type dtrace_invop_calltrap_addr,@object .size dtrace_invop_calltrap_addr,8 dtrace_invop_calltrap_addr: .zero 8 #endif .text #ifdef HWPMC_HOOKS ENTRY(start_exceptions) #endif /*****************************************************************************/ /* Trap handling */ /*****************************************************************************/ /* * Trap and fault vector routines. * * All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes * state on the stack but also disables interrupts. This is important for * us for the use of the swapgs instruction. We cannot be interrupted * until the GS.base value is correct. For most traps, we automatically * then enable interrupts if the interrupted context had them enabled. * This is equivalent to the i386 port's use of SDT_SYS386TGT. * * The cpu will push a certain amount of state onto the kernel stack for * the current process. See amd64/include/frame.h. * This includes the current RFLAGS (status register, which includes * the interrupt disable state prior to the trap), the code segment register, * and the return instruction pointer are pushed by the cpu. The cpu * will also push an 'error' code for certain traps. We push a dummy * error code for those traps where the cpu doesn't in order to maintain * a consistent frame. We also push a contrived 'trap number'. * * The CPU does not push the general registers, so we must do that, and we * must restore them prior to calling 'iret'. The CPU adjusts %cs and %ss * but does not mess with %ds, %es, %gs or %fs. We swap the %gs base for * for the kernel mode operation shortly, without changes to the selector * loaded. Since superuser long mode works with any selectors loaded into * segment registers other then %cs, which makes them mostly unused in long * mode, and kernel does not reference %fs, leave them alone. The segment * registers are reloaded on return to the usermode. */ MCOUNT_LABEL(user) MCOUNT_LABEL(btrap) /* Traps that we leave interrupts disabled for. */ .macro TRAP_NOEN l, trapno PTI_ENTRY \l,X\l .globl X\l .type X\l,@function X\l: subq $TF_RIP,%rsp movl $\trapno,TF_TRAPNO(%rsp) movq $0,TF_ADDR(%rsp) movq $0,TF_ERR(%rsp) jmp alltraps_noen .endm TRAP_NOEN dbg, T_TRCTRAP TRAP_NOEN bpt, T_BPTFLT #ifdef KDTRACE_HOOKS TRAP_NOEN dtrace_ret, T_DTRACE_RET #endif /* Regular traps; The cpu does not supply tf_err for these. */ .macro TRAP l, trapno PTI_ENTRY \l,X\l .globl X\l .type X\l,@function X\l: subq $TF_RIP,%rsp movl $\trapno,TF_TRAPNO(%rsp) movq $0,TF_ADDR(%rsp) movq $0,TF_ERR(%rsp) jmp alltraps .endm TRAP div, T_DIVIDE TRAP ofl, T_OFLOW TRAP bnd, T_BOUND TRAP ill, T_PRIVINFLT TRAP dna, T_DNA TRAP fpusegm, T_FPOPFLT TRAP rsvd, T_RESERVED TRAP fpu, T_ARITHTRAP TRAP xmm, T_XMMFLT /* This group of traps have tf_err already pushed by the cpu. */ .macro TRAP_ERR l, trapno PTI_ENTRY \l,X\l,has_err=1 .globl X\l .type X\l,@function X\l: subq $TF_ERR,%rsp movl $\trapno,TF_TRAPNO(%rsp) movq $0,TF_ADDR(%rsp) jmp alltraps .endm TRAP_ERR tss, T_TSSFLT TRAP_ERR align, T_ALIGNFLT /* * alltraps entry point. Use swapgs if this is the first time in the * kernel from userland. Reenable interrupts if they were enabled * before the trap. This approximates SDT_SYS386TGT on the i386 port. */ SUPERALIGN_TEXT .globl alltraps .type alltraps,@function alltraps: movq %rdi,TF_RDI(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 1: SAVE_SEGS movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) movq %rcx,TF_RCX(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) jz 2f call handle_ibrs_entry 2: testl $PSL_I,TF_RFLAGS(%rsp) jz alltraps_pushregs_no_rax sti alltraps_pushregs_no_rax: movq %rsi,TF_RSI(%rsp) movq %r8,TF_R8(%rsp) movq %r9,TF_R9(%rsp) movq %rbx,TF_RBX(%rsp) movq %rbp,TF_RBP(%rsp) movq %r10,TF_R10(%rsp) movq %r11,TF_R11(%rsp) movq %r12,TF_R12(%rsp) movq %r13,TF_R13(%rsp) movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) movl $TF_HASSEGS,TF_FLAGS(%rsp) cld FAKE_MCOUNT(TF_RIP(%rsp)) #ifdef KDTRACE_HOOKS /* * DTrace Function Boundary Trace (fbt) probes are triggered * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint * interrupt. For all other trap types, just handle them in * the usual way. */ testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jnz calltrap /* ignore userland traps */ cmpl $T_BPTFLT,TF_TRAPNO(%rsp) jne calltrap /* Check if there is no DTrace hook registered. */ cmpq $0,dtrace_invop_jump_addr je calltrap /* * Set our jump address for the jump back in the event that * the breakpoint wasn't caused by DTrace at all. */ movq $calltrap,dtrace_invop_calltrap_addr(%rip) /* Jump to the code hooked in by DTrace. */ jmpq *dtrace_invop_jump_addr #endif .globl calltrap .type calltrap,@function calltrap: movq %rsp,%rdi call trap_check MEXITCOUNT jmp doreti /* Handle any pending ASTs */ /* * alltraps_noen entry point. Unlike alltraps above, we want to * leave the interrupts disabled. This corresponds to * SDT_SYS386IGT on the i386 port. */ SUPERALIGN_TEXT .globl alltraps_noen .type alltraps_noen,@function alltraps_noen: movq %rdi,TF_RDI(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 1: SAVE_SEGS movq %rdx,TF_RDX(%rsp) movq %rax,TF_RAX(%rsp) movq %rcx,TF_RCX(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) jz alltraps_pushregs_no_rax call handle_ibrs_entry jmp alltraps_pushregs_no_rax IDTVEC(dblfault) subq $TF_ERR,%rsp movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) movq $0,TF_ADDR(%rsp) movq $0,TF_ERR(%rsp) movq %rdi,TF_RDI(%rsp) movq %rsi,TF_RSI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) movq %r8,TF_R8(%rsp) movq %r9,TF_R9(%rsp) movq %rax,TF_RAX(%rsp) movq %rbx,TF_RBX(%rsp) movq %rbp,TF_RBP(%rsp) movq %r10,TF_R10(%rsp) movq %r11,TF_R11(%rsp) movq %r12,TF_R12(%rsp) movq %r13,TF_R13(%rsp) movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) SAVE_SEGS movl $TF_HASSEGS,TF_FLAGS(%rsp) cld testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 1f /* already running with kernel GS.base */ swapgs 1: movq PCPU(KCR3),%rax cmpq $~0,%rax je 2f movq %rax,%cr3 2: movq %rsp,%rdi call dblfault_handler 3: hlt jmp 3b ALIGN_TEXT IDTVEC(page_pti) testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) jz Xpage swapgs pushq %rax pushq %rdx movq %cr3,%rax movq %rax,PCPU(SAVED_UCR3) PTI_UUENTRY has_err=1 subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) jmp page_u IDTVEC(page) subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) /* free up GP registers */ movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz page_cr2 /* already running with kernel GS.base */ swapgs page_u: movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) movq PCPU(SAVED_UCR3),%rax movq %rax,PCB_SAVED_UCR3(%rdi) call handle_ibrs_entry page_cr2: movq %cr2,%rdi /* preserve %cr2 before .. */ movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ SAVE_SEGS movl $T_PAGEFLT,TF_TRAPNO(%rsp) testl $PSL_I,TF_RFLAGS(%rsp) jz alltraps_pushregs_no_rax sti jmp alltraps_pushregs_no_rax /* * We have to special-case this one. If we get a trap in doreti() at * the iretq stage, we'll reenter with the wrong gs state. We'll have * to do a special the swapgs in this case even coming from the kernel. * XXX linux has a trap handler for their equivalent of load_gs(). * * On the stack, we have the hardware interrupt frame to return * to usermode (faulted) and another frame with error code, for * fault. For PTI, copy both frames to the main thread stack. */ .macro PROTF_ENTRY name,trapno \name\()_pti_doreti: pushq %rax pushq %rdx swapgs movq PCPU(KCR3),%rax movq %rax,%cr3 movq PCPU(RSP0),%rax subq $2*PTI_SIZE-3*8,%rax /* no err, %rax, %rdx in faulted frame */ MOVE_STACKS (PTI_SIZE / 4 - 3) movq %rax,%rsp popq %rdx popq %rax swapgs jmp X\name IDTVEC(\name\()_pti) cmpq $doreti_iret,PTI_RIP-2*8(%rsp) je \name\()_pti_doreti testb $SEL_RPL_MASK,PTI_CS-2*8(%rsp) /* %rax, %rdx not yet pushed */ jz X\name PTI_UENTRY has_err=1 swapgs IDTVEC(\name) subq $TF_ERR,%rsp movl $\trapno,TF_TRAPNO(%rsp) jmp prot_addrf .endm PROTF_ENTRY missing, T_SEGNPFLT PROTF_ENTRY stk, T_STKFLT PROTF_ENTRY prot, T_PROTFLT prot_addrf: movq $0,TF_ADDR(%rsp) movq %rdi,TF_RDI(%rsp) /* free up a GP register */ movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) movw %fs,TF_FS(%rsp) movw %gs,TF_GS(%rsp) leaq doreti_iret(%rip),%rdi cmpq %rdi,TF_RIP(%rsp) je 5f /* kernel but with user gsbase!! */ testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 6f /* already running with kernel GS.base */ testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) jz 2f cmpw $KUF32SEL,TF_FS(%rsp) jne 1f rdfsbase %rax 1: cmpw $KUG32SEL,TF_GS(%rsp) jne 2f rdgsbase %rdx 2: swapgs movq PCPU(CURPCB),%rdi testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) jz 4f cmpw $KUF32SEL,TF_FS(%rsp) jne 3f movq %rax,PCB_FSBASE(%rdi) 3: cmpw $KUG32SEL,TF_GS(%rsp) jne 4f movq %rdx,PCB_GSBASE(%rdi) 4: call handle_ibrs_entry orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ movw %es,TF_ES(%rsp) movw %ds,TF_DS(%rsp) testl $PSL_I,TF_RFLAGS(%rsp) jz alltraps_pushregs_no_rax sti jmp alltraps_pushregs_no_rax 5: swapgs 6: movq PCPU(CURPCB),%rdi jmp 4b /* * Fast syscall entry point. We enter here with just our new %cs/%ss set, * and the new privilige level. We are still running on the old user stack * pointer. We have to juggle a few things around to find our stack etc. * swapgs gives us access to our PCPU space only. * * We do not support invoking this from a custom segment registers, * esp. %cs, %ss, %fs, %gs, e.g. using entries from an LDT. */ SUPERALIGN_TEXT IDTVEC(fast_syscall_pti) swapgs movq %rax,PCPU(SCRATCH_RAX) movq PCPU(KCR3),%rax movq %rax,%cr3 jmp fast_syscall_common SUPERALIGN_TEXT IDTVEC(fast_syscall) swapgs movq %rax,PCPU(SCRATCH_RAX) fast_syscall_common: movq %rsp,PCPU(SCRATCH_RSP) movq PCPU(RSP0),%rsp /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ subq $TF_SIZE,%rsp /* defer TF_RSP till we have a spare register */ movq %r11,TF_RFLAGS(%rsp) movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ movq %r11,TF_RSP(%rsp) /* user stack pointer */ movq PCPU(SCRATCH_RAX),%rax movq %rax,TF_RAX(%rsp) /* syscall number */ movq %rdx,TF_RDX(%rsp) /* arg 3 */ SAVE_SEGS call handle_ibrs_entry movq PCPU(CURPCB),%r11 andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) sti movq $KUDSEL,TF_SS(%rsp) movq $KUCSEL,TF_CS(%rsp) movq $2,TF_ERR(%rsp) movq %rdi,TF_RDI(%rsp) /* arg 1 */ movq %rsi,TF_RSI(%rsp) /* arg 2 */ movq %r10,TF_RCX(%rsp) /* arg 4 */ movq %r8,TF_R8(%rsp) /* arg 5 */ movq %r9,TF_R9(%rsp) /* arg 6 */ movq %rbx,TF_RBX(%rsp) /* C preserved */ movq %rbp,TF_RBP(%rsp) /* C preserved */ movq %r12,TF_R12(%rsp) /* C preserved */ movq %r13,TF_R13(%rsp) /* C preserved */ movq %r14,TF_R14(%rsp) /* C preserved */ movq %r15,TF_R15(%rsp) /* C preserved */ movl $TF_HASSEGS,TF_FLAGS(%rsp) FAKE_MCOUNT(TF_RIP(%rsp)) movq PCPU(CURTHREAD),%rdi movq %rsp,TD_FRAME(%rdi) movl TF_RFLAGS(%rsp),%esi andl $PSL_T,%esi call amd64_syscall 1: movq PCPU(CURPCB),%rax /* Disable interrupts before testing PCB_FULL_IRET. */ cli testl $PCB_FULL_IRET,PCB_FLAGS(%rax) jnz 4f /* Check for and handle AST's on return to userland. */ movq PCPU(CURTHREAD),%rax testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) jne 3f call handle_ibrs_exit /* Restore preserved registers. */ MEXITCOUNT movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ movq TF_RDX(%rsp),%rdx /* return value 2 */ movq TF_RAX(%rsp),%rax /* return value 1 */ movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ movq TF_RIP(%rsp),%rcx /* original %rip */ movq TF_RSP(%rsp),%rsp /* user stack pointer */ cmpb $0,pti je 2f movq PCPU(UCR3),%r9 movq %r9,%cr3 xorl %r9d,%r9d 2: swapgs sysretq 3: /* AST scheduled. */ sti movq %rsp,%rdi call ast jmp 1b 4: /* Requested full context restore, use doreti for that. */ MEXITCOUNT jmp doreti /* * Here for CYA insurance, in case a "syscall" instruction gets * issued from 32 bit compatibility mode. MSR_CSTAR has to point * to *something* if EFER_SCE is enabled. */ IDTVEC(fast_syscall32) sysret /* * NMI handling is special. * * First, NMIs do not respect the state of the processor's RFLAGS.IF * bit. The NMI handler may be entered at any time, including when * the processor is in a critical section with RFLAGS.IF == 0. * The processor's GS.base value could be invalid on entry to the * handler. * * Second, the processor treats NMIs specially, blocking further NMIs * until an 'iretq' instruction is executed. We thus need to execute * the NMI handler with interrupts disabled, to prevent a nested interrupt * from executing an 'iretq' instruction and inadvertently taking the * processor out of NMI mode. * * Third, the NMI handler runs on its own stack (tss_ist2). The canonical * GS.base value for the processor is stored just above the bottom of its * NMI stack. For NMIs taken from kernel mode, the current value in * the processor's GS.base is saved at entry to C-preserved register %r12, * the canonical value for GS.base is then loaded into the processor, and * the saved value is restored at exit time. For NMIs taken from user mode, * the cheaper 'SWAPGS' instructions are used for swapping GS.base. */ IDTVEC(nmi) subq $TF_RIP,%rsp movl $(T_NMI),TF_TRAPNO(%rsp) movq $0,TF_ADDR(%rsp) movq $0,TF_ERR(%rsp) movq %rdi,TF_RDI(%rsp) movq %rsi,TF_RSI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) movq %r8,TF_R8(%rsp) movq %r9,TF_R9(%rsp) movq %rax,TF_RAX(%rsp) movq %rbx,TF_RBX(%rsp) movq %rbp,TF_RBP(%rsp) movq %r10,TF_R10(%rsp) movq %r11,TF_R11(%rsp) movq %r12,TF_R12(%rsp) movq %r13,TF_R13(%rsp) movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) SAVE_SEGS movl $TF_HASSEGS,TF_FLAGS(%rsp) cld xorl %ebx,%ebx testb $SEL_RPL_MASK,TF_CS(%rsp) jnz nmi_fromuserspace /* * We've interrupted the kernel. Preserve GS.base in %r12, * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. */ movl $MSR_GSBASE,%ecx rdmsr movq %rax,%r12 shlq $32,%rdx orq %rdx,%r12 /* Retrieve and load the canonical value for GS.base. */ movq TF_SIZE(%rsp),%rdx movl %edx,%eax shrq $32,%rdx wrmsr movq %cr3,%r13 movq PCPU(KCR3),%rax cmpq $~0,%rax je 1f movq %rax,%cr3 1: testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) je nmi_calltrap movl $MSR_IA32_SPEC_CTRL,%ecx rdmsr movl %eax,%r14d call handle_ibrs_entry jmp nmi_calltrap nmi_fromuserspace: incl %ebx swapgs movq %cr3,%r13 movq PCPU(KCR3),%rax cmpq $~0,%rax je 1f movq %rax,%cr3 1: call handle_ibrs_entry movq PCPU(CURPCB),%rdi testq %rdi,%rdi jz 3f orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) testb $CPUID_STDEXT_FSGSBASE,cpu_stdext_feature(%rip) jz 3f cmpw $KUF32SEL,TF_FS(%rsp) jne 2f rdfsbase %rax movq %rax,PCB_FSBASE(%rdi) 2: cmpw $KUG32SEL,TF_GS(%rsp) jne 3f movl $MSR_KGSBASE,%ecx rdmsr shlq $32,%rdx orq %rdx,%rax movq %rax,PCB_GSBASE(%rdi) 3: /* Note: this label is also used by ddb and gdb: */ nmi_calltrap: FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp,%rdi call trap MEXITCOUNT #ifdef HWPMC_HOOKS /* * Capture a userspace callchain if needed. * * - Check if the current trap was from user mode. * - Check if the current thread is valid. * - Check if the thread requires a user call chain to be * captured. * * We are still in NMI mode at this point. */ testl %ebx,%ebx jz nocallchain /* not from userspace */ movq PCPU(CURTHREAD),%rax orq %rax,%rax /* curthread present? */ jz nocallchain /* * Move execution to the regular kernel stack, because we * committed to return through doreti. */ movq %rsp,%rsi /* source stack pointer */ movq $TF_SIZE,%rcx movq PCPU(RSP0),%rdx subq %rcx,%rdx movq %rdx,%rdi /* destination stack pointer */ shrq $3,%rcx /* trap frame size in long words */ cld rep movsq /* copy trapframe */ movq %rdx,%rsp /* we are on the regular kstack */ testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ jz nocallchain /* * A user callchain is to be captured, so: * - Take the processor out of "NMI" mode by faking an "iret", * to allow for nested NMI interrupts. * - Enable interrupts, so that copyin() can work. */ movl %ss,%eax pushq %rax /* tf_ss */ pushq %rdx /* tf_rsp (on kernel stack) */ pushfq /* tf_rflags */ movl %cs,%eax pushq %rax /* tf_cs */ pushq $outofnmi /* tf_rip */ iretq outofnmi: /* * At this point the processor has exited NMI mode and is running * with interrupts turned off on the normal kernel stack. * * If a pending NMI gets recognized at or after this point, it * will cause a kernel callchain to be traced. * * We turn interrupts back on, and call the user callchain capture hook. */ movq pmc_hook,%rax orq %rax,%rax jz nocallchain movq PCPU(CURTHREAD),%rdi /* thread */ movq $PMC_FN_USER_CALLCHAIN,%rsi /* command */ movq %rsp,%rdx /* frame */ sti call *%rax cli nocallchain: #endif testl %ebx,%ebx /* %ebx == 0 => return to userland */ jnz doreti_exit /* * Restore speculation control MSR, if preserved. */ testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) je 1f movl %r14d,%eax xorl %edx,%edx movl $MSR_IA32_SPEC_CTRL,%ecx wrmsr /* * Put back the preserved MSR_GSBASE value. */ 1: movl $MSR_GSBASE,%ecx movq %r12,%rdx movl %edx,%eax shrq $32,%rdx wrmsr movq %r13,%cr3 RESTORE_REGS addq $TF_RIP,%rsp jmp doreti_iret /* * MC# handling is similar to NMI. * * As with NMIs, machine check exceptions do not respect RFLAGS.IF and * can occur at any time with a GS.base value that does not correspond * to the privilege level in CS. * * Machine checks are not unblocked by iretq, but it is best to run * the handler with interrupts disabled since the exception may have * interrupted a critical section. * * The MC# handler runs on its own stack (tss_ist3). The canonical * GS.base value for the processor is stored just above the bottom of * its MC# stack. For exceptions taken from kernel mode, the current * value in the processor's GS.base is saved at entry to C-preserved * register %r12, the canonical value for GS.base is then loaded into * the processor, and the saved value is restored at exit time. For * exceptions taken from user mode, the cheaper 'SWAPGS' instructions * are used for swapping GS.base. */ IDTVEC(mchk) subq $TF_RIP,%rsp movl $(T_MCHK),TF_TRAPNO(%rsp) movq $0,TF_ADDR(%rsp) movq $0,TF_ERR(%rsp) movq %rdi,TF_RDI(%rsp) movq %rsi,TF_RSI(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) movq %r8,TF_R8(%rsp) movq %r9,TF_R9(%rsp) movq %rax,TF_RAX(%rsp) movq %rbx,TF_RBX(%rsp) movq %rbp,TF_RBP(%rsp) movq %r10,TF_R10(%rsp) movq %r11,TF_R11(%rsp) movq %r12,TF_R12(%rsp) movq %r13,TF_R13(%rsp) movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) SAVE_SEGS movl $TF_HASSEGS,TF_FLAGS(%rsp) cld xorl %ebx,%ebx testb $SEL_RPL_MASK,TF_CS(%rsp) jnz mchk_fromuserspace /* * We've interrupted the kernel. Preserve GS.base in %r12, * %cr3 in %r13, and possibly lower half of MSR_IA32_SPEC_CTL in %r14d. */ movl $MSR_GSBASE,%ecx rdmsr movq %rax,%r12 shlq $32,%rdx orq %rdx,%r12 /* Retrieve and load the canonical value for GS.base. */ movq TF_SIZE(%rsp),%rdx movl %edx,%eax shrq $32,%rdx wrmsr movq %cr3,%r13 movq PCPU(KCR3),%rax cmpq $~0,%rax je 1f movq %rax,%cr3 1: testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) je mchk_calltrap movl $MSR_IA32_SPEC_CTRL,%ecx rdmsr movl %eax,%r14d call handle_ibrs_entry jmp mchk_calltrap mchk_fromuserspace: incl %ebx swapgs movq %cr3,%r13 movq PCPU(KCR3),%rax cmpq $~0,%rax je 1f movq %rax,%cr3 1: call handle_ibrs_entry /* Note: this label is also used by ddb and gdb: */ mchk_calltrap: FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp,%rdi call mca_intr MEXITCOUNT testl %ebx,%ebx /* %ebx == 0 => return to userland */ jnz doreti_exit /* * Restore speculation control MSR, if preserved. */ testl $CPUID_STDEXT3_IBPB,cpu_stdext_feature3(%rip) je 1f movl %r14d,%eax xorl %edx,%edx movl $MSR_IA32_SPEC_CTRL,%ecx wrmsr /* * Put back the preserved MSR_GSBASE value. */ 1: movl $MSR_GSBASE,%ecx movq %r12,%rdx movl %edx,%eax shrq $32,%rdx wrmsr movq %r13,%cr3 RESTORE_REGS addq $TF_RIP,%rsp jmp doreti_iret ENTRY(fork_trampoline) movq %r12,%rdi /* function */ movq %rbx,%rsi /* arg1 */ movq %rsp,%rdx /* trapframe pointer */ call fork_exit MEXITCOUNT jmp doreti /* Handle any ASTs */ /* * To efficiently implement classification of trap and interrupt handlers * for profiling, there must be only trap handlers between the labels btrap * and bintr, and only interrupt handlers between the labels bintr and * eintr. This is implemented (partly) by including files that contain * some of the handlers. Before including the files, set up a normal asm * environment so that the included files doen't need to know that they are * included. */ #ifdef COMPAT_FREEBSD32 .data .p2align 4 .text SUPERALIGN_TEXT #include #endif .data .p2align 4 .text SUPERALIGN_TEXT MCOUNT_LABEL(bintr) #include #ifdef DEV_ATPIC .data .p2align 4 .text SUPERALIGN_TEXT #include #endif .text MCOUNT_LABEL(eintr) /* * void doreti(struct trapframe) * * Handle return from interrupts, traps and syscalls. */ .text SUPERALIGN_TEXT .type doreti,@function .globl doreti doreti: FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ /* * Check if ASTs can be handled now. */ testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */ jz doreti_exit /* can't handle ASTs now if not */ doreti_ast: /* * Check for ASTs atomically with returning. Disabling CPU * interrupts provides sufficient locking even in the SMP case, * since we will be informed of any new ASTs by an IPI. */ cli movq PCPU(CURTHREAD),%rax testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) je doreti_exit sti movq %rsp,%rdi /* pass a pointer to the trapframe */ call ast jmp doreti_ast /* * doreti_exit: pop registers, iret. * * The segment register pop is a special case, since it may * fault if (for example) a sigreturn specifies bad segment * registers. The fault is handled in trap.c. */ doreti_exit: MEXITCOUNT movq PCPU(CURPCB),%r8 /* * Do not reload segment registers for kernel. * Since we do not reload segments registers with sane * values on kernel entry, descriptors referenced by * segments registers might be not valid. This is fatal * for user mode, but is not a problem for the kernel. */ testb $SEL_RPL_MASK,TF_CS(%rsp) jz ld_regs testl $PCB_FULL_IRET,PCB_FLAGS(%r8) jz ld_regs andl $~PCB_FULL_IRET,PCB_FLAGS(%r8) testl $TF_HASSEGS,TF_FLAGS(%rsp) je set_segs do_segs: /* Restore %fs and fsbase */ movw TF_FS(%rsp),%ax .globl ld_fs ld_fs: movw %ax,%fs cmpw $KUF32SEL,%ax jne 1f movl $MSR_FSBASE,%ecx movl PCB_FSBASE(%r8),%eax movl PCB_FSBASE+4(%r8),%edx .globl ld_fsbase ld_fsbase: wrmsr 1: /* Restore %gs and gsbase */ movw TF_GS(%rsp),%si pushfq cli movl $MSR_GSBASE,%ecx /* Save current kernel %gs base into %r12d:%r13d */ rdmsr movl %eax,%r12d movl %edx,%r13d .globl ld_gs ld_gs: movw %si,%gs /* Save user %gs base into %r14d:%r15d */ rdmsr movl %eax,%r14d movl %edx,%r15d /* Restore kernel %gs base */ movl %r12d,%eax movl %r13d,%edx wrmsr popfq /* * Restore user %gs base, either from PCB if used for TLS, or * from the previously saved msr read. */ movl $MSR_KGSBASE,%ecx cmpw $KUG32SEL,%si jne 1f movl PCB_GSBASE(%r8),%eax movl PCB_GSBASE+4(%r8),%edx jmp ld_gsbase 1: movl %r14d,%eax movl %r15d,%edx .globl ld_gsbase ld_gsbase: wrmsr /* May trap if non-canonical, but only for TLS. */ .globl ld_es ld_es: movw TF_ES(%rsp),%es .globl ld_ds ld_ds: movw TF_DS(%rsp),%ds ld_regs: RESTORE_REGS testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ jz 2f /* keep running with kernel GS.base */ cli call handle_ibrs_exit_rs cmpb $0,pti je 1f pushq %rdx movq PCPU(PRVSPACE),%rdx addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx movq %rax,PTI_RAX(%rdx) popq %rax movq %rax,PTI_RDX(%rdx) movq TF_RIP(%rsp),%rax movq %rax,PTI_RIP(%rdx) movq TF_CS(%rsp),%rax movq %rax,PTI_CS(%rdx) movq TF_RFLAGS(%rsp),%rax movq %rax,PTI_RFLAGS(%rdx) movq TF_RSP(%rsp),%rax movq %rax,PTI_RSP(%rdx) movq TF_SS(%rsp),%rax movq %rax,PTI_SS(%rdx) movq PCPU(UCR3),%rax swapgs movq %rdx,%rsp movq %rax,%cr3 popq %rdx popq %rax addq $8,%rsp jmp doreti_iret 1: swapgs 2: addq $TF_RIP,%rsp .globl doreti_iret doreti_iret: iretq set_segs: movw $KUDSEL,%ax movw %ax,TF_DS(%rsp) movw %ax,TF_ES(%rsp) movw $KUF32SEL,TF_FS(%rsp) movw $KUG32SEL,TF_GS(%rsp) jmp do_segs /* * doreti_iret_fault. Alternative return code for * the case where we get a fault in the doreti_exit code * above. trap() (amd64/amd64/trap.c) catches this specific * case, sends the process a signal and continues in the * corresponding place in the code below. */ ALIGN_TEXT .globl doreti_iret_fault doreti_iret_fault: subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) call handle_ibrs_entry testb $SEL_RPL_MASK,TF_CS(%rsp) jz 1f sti 1: SAVE_SEGS movl $TF_HASSEGS,TF_FLAGS(%rsp) movq %rdi,TF_RDI(%rsp) movq %rsi,TF_RSI(%rsp) movq %r8,TF_R8(%rsp) movq %r9,TF_R9(%rsp) movq %rbx,TF_RBX(%rsp) movq %rbp,TF_RBP(%rsp) movq %r10,TF_R10(%rsp) movq %r11,TF_R11(%rsp) movq %r12,TF_R12(%rsp) movq %r13,TF_R13(%rsp) movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) movl $T_PROTFLT,TF_TRAPNO(%rsp) movq $0,TF_ERR(%rsp) /* XXX should be the error code */ movq $0,TF_ADDR(%rsp) FAKE_MCOUNT(TF_RIP(%rsp)) jmp calltrap ALIGN_TEXT .globl ds_load_fault ds_load_fault: movl $T_PROTFLT,TF_TRAPNO(%rsp) testb $SEL_RPL_MASK,TF_CS(%rsp) jz 1f sti 1: movq %rsp,%rdi call trap movw $KUDSEL,TF_DS(%rsp) jmp doreti ALIGN_TEXT .globl es_load_fault es_load_fault: movl $T_PROTFLT,TF_TRAPNO(%rsp) testl $PSL_I,TF_RFLAGS(%rsp) jz 1f sti 1: movq %rsp,%rdi call trap movw $KUDSEL,TF_ES(%rsp) jmp doreti ALIGN_TEXT .globl fs_load_fault fs_load_fault: testl $PSL_I,TF_RFLAGS(%rsp) jz 1f sti 1: movl $T_PROTFLT,TF_TRAPNO(%rsp) movq %rsp,%rdi call trap movw $KUF32SEL,TF_FS(%rsp) jmp doreti ALIGN_TEXT .globl gs_load_fault gs_load_fault: popfq movl $T_PROTFLT,TF_TRAPNO(%rsp) testl $PSL_I,TF_RFLAGS(%rsp) jz 1f sti 1: movq %rsp,%rdi call trap movw $KUG32SEL,TF_GS(%rsp) jmp doreti ALIGN_TEXT .globl fsbase_load_fault fsbase_load_fault: movl $T_PROTFLT,TF_TRAPNO(%rsp) testl $PSL_I,TF_RFLAGS(%rsp) jz 1f sti 1: movq %rsp,%rdi call trap movq PCPU(CURTHREAD),%r8 movq TD_PCB(%r8),%r8 movq $0,PCB_FSBASE(%r8) jmp doreti ALIGN_TEXT .globl gsbase_load_fault gsbase_load_fault: movl $T_PROTFLT,TF_TRAPNO(%rsp) testl $PSL_I,TF_RFLAGS(%rsp) jz 1f sti 1: movq %rsp,%rdi call trap movq PCPU(CURTHREAD),%r8 movq TD_PCB(%r8),%r8 movq $0,PCB_GSBASE(%r8) jmp doreti #ifdef HWPMC_HOOKS ENTRY(end_exceptions) #endif Index: head/sys/amd64/amd64/locore.S =================================================================== --- head/sys/amd64/amd64/locore.S (revision 331253) +++ head/sys/amd64/amd64/locore.S (revision 331254) @@ -1,89 +1,89 @@ /*- * Copyright (c) 2003 Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include -#include "assym.s" +#include "assym.inc" /* * Compiled KERNBASE location */ .globl kernbase,loc_PTmap,loc_PDmap,loc_PDPmap,loc_PML4map,loc_PML4pml4e,dmapbase,dmapend .set kernbase,KERNBASE .set loc_PTmap,addr_PTmap .set loc_PDmap,addr_PDmap .set loc_PDPmap,addr_PDPmap .set loc_PML4map,addr_PML4map .set loc_PML4pml4e,addr_PML4pml4e .set dmapbase,DMAP_MIN_ADDRESS .set dmapend,DMAP_MAX_ADDRESS .text /********************************************************************** * * This is where the loader trampoline start us, set the ball rolling... * * We are called with the stack looking like this: * 0(%rsp) = 32 bit return address (cannot be used) * 4(%rsp) = 32 bit modulep * 8(%rsp) = 32 bit kernend * * We are already in long mode, on a 64 bit %cs and running at KERNBASE. */ NON_GPROF_ENTRY(btext) /* Tell the bios to warmboot next time */ movw $0x1234,0x472 /* Don't trust what the loader gives for rflags. */ pushq $PSL_KERNEL popfq /* Find the metadata pointers before we lose them */ movq %rsp, %rbp movl 4(%rbp),%edi /* modulep (arg 1) */ movl 8(%rbp),%esi /* kernend (arg 2) */ /* Get onto a stack that we can trust - there is no going back now. */ movq $bootstack,%rsp xorl %ebp, %ebp call hammer_time /* set up cpu for unix operation */ movq %rax,%rsp /* set up kstack for mi_startup() */ call mi_startup /* autoconfiguration, mountroot etc */ 0: hlt jmp 0b .bss ALIGN_DATA /* just to be sure */ .globl bootstack .space 0x1000 /* space for bootstack - temporary stack */ bootstack: Index: head/sys/amd64/amd64/mpboot.S =================================================================== --- head/sys/amd64/amd64/mpboot.S (revision 331253) +++ head/sys/amd64/amd64/mpboot.S (revision 331254) @@ -1,252 +1,252 @@ /*- * Copyright (c) 2003 Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include /* miscellaneous asm macros */ #include -#include "assym.s" +#include "assym.inc" .data /* So we can modify it */ .p2align 4,0 .globl mptramp_start mptramp_start: .code16 /* * The AP enters here in response to the startup IPI. * We are in real mode. %cs is the only segment register set. */ cli /* make sure no interrupts */ mov %cs, %ax /* copy %cs to %ds. Remember these */ mov %ax, %ds /* are offsets rather than selectors */ mov %ax, %ss /* * Find relocation base and patch the gdt descript and ljmp targets */ xorl %ebx,%ebx mov %cs, %bx sall $4, %ebx /* %ebx is now our relocation base */ orl %ebx, lgdt_desc-mptramp_start+2 orl %ebx, jmp_32-mptramp_start+2 orl %ebx, jmp_64-mptramp_start+1 /* * Load the descriptor table pointer. We'll need it when running * in 16 bit protected mode. */ lgdt lgdt_desc-mptramp_start /* Enable protected mode */ movl $CR0_PE, %eax mov %eax, %cr0 /* * Now execute a far jump to turn on protected mode. This * causes the segment registers to turn into selectors and causes * %cs to be loaded from the gdt. * * The following instruction is: * ljmpl $bootcode-gdt, $protmode-mptramp_start * but gas cannot assemble that. And besides, we patch the targets * in early startup and its a little clearer what we are patching. */ jmp_32: .byte 0x66 /* size override to 32 bits */ .byte 0xea /* opcode for far jump */ .long protmode-mptramp_start /* offset in segment */ .word bootcode-gdt /* index in gdt for 32 bit code */ /* * At this point, we are running in 32 bit legacy protected mode. */ .code32 protmode: mov $bootdata-gdt, %eax mov %ax, %ds /* Turn on the PAE bit for when paging is enabled */ mov %cr4, %eax orl $CR4_PAE, %eax mov %eax, %cr4 /* * Enable EFER.LME so that we get long mode when all the prereqs are * in place. In this case, it turns on when CR0_PG is finally enabled. * Pick up a few other EFER bits that we'll use need we're here. */ movl $MSR_EFER, %ecx rdmsr orl $EFER_LME | EFER_SCE, %eax wrmsr /* * Point to the embedded page tables for startup. Note that this * only gets accessed after we're actually in 64 bit mode, however * we can only set the bottom 32 bits of %cr3 in this state. This * means we are required to use a temporary page table that is below * the 4GB limit. %ebx is still our relocation base. We could just * subtract 3 * PAGE_SIZE, but that would be too easy. */ leal mptramp_pagetables-mptramp_start(%ebx),%eax movl (%eax), %eax mov %eax, %cr3 /* * Finally, switch to long bit mode by enabling paging. We have * to be very careful here because all the segmentation disappears * out from underneath us. The spec says we can depend on the * subsequent pipelined branch to execute, but *only if* everything * is still identity mapped. If any mappings change, the pipeline * will flush. */ mov %cr0, %eax orl $CR0_PG, %eax mov %eax, %cr0 /* * At this point paging is enabled, and we are in "compatibility" mode. * We do another far jump to reload %cs with the 64 bit selector. * %cr3 points to a 4-level page table page. * We cannot yet jump all the way to the kernel because we can only * specify a 32 bit linear address. So, yet another trampoline. * * The following instruction is: * ljmp $kernelcode-gdt, $tramp_64-mptramp_start * but gas cannot assemble that. And besides, we patch the targets * in early startup and its a little clearer what we are patching. */ jmp_64: .byte 0xea /* opcode for far jump */ .long tramp_64-mptramp_start /* offset in segment */ .word kernelcode-gdt /* index in gdt for 64 bit code */ /* * Yeehar! We're running in 64 bit mode! We can mostly ignore our * segment registers, and get on with it. * Note that we are running at the correct virtual address, but with * a 1:1 1GB mirrored mapping over entire address space. We had better * switch to a real %cr3 promptly so that we can get to the direct map * space. Remember that jmp is relative and that we've been relocated, * so use an indirect jump. */ .code64 tramp_64: movabsq $entry_64,%rax /* 64 bit immediate load */ jmp *%rax .p2align 4,0 gdt: /* * All segment descriptor tables start with a null descriptor */ .long 0x00000000 .long 0x00000000 /* * This is the 64 bit long mode code descriptor. There is no * 64 bit data descriptor. */ kernelcode: .long 0x00000000 .long 0x00209800 /* * This is the descriptor for the 32 bit boot code. * %cs: +A, +R, -C, DPL=0, +P, +D, +G * Accessed, Readable, Present, 32 bit, 4G granularity */ bootcode: .long 0x0000ffff .long 0x00cf9b00 /* * This is the descriptor for the 32 bit boot data. * We load it into %ds and %ss. The bits for each selector * are interpreted slightly differently. * %ds: +A, +W, -E, DPL=0, +P, +D, +G * %ss: +A, +W, -E, DPL=0, +P, +B, +G * Accessed, Writeable, Expand up, Present, 32 bit, 4GB * For %ds, +D means 'default operand size is 32 bit'. * For %ss, +B means the stack register is %esp rather than %sp. */ bootdata: .long 0x0000ffff .long 0x00cf9300 gdtend: /* * The address of our page table pages that the boot code * uses to trampoline up to kernel address space. */ .globl mptramp_pagetables mptramp_pagetables: .long 0 /* * The pseudo descriptor for lgdt to use. */ lgdt_desc: .word gdtend-gdt /* Length */ .long gdt-mptramp_start /* Offset plus %ds << 4 */ .globl mptramp_end mptramp_end: /* * From here on down is executed in the kernel .text section. */ .text .code64 .p2align 4,0 entry_64: /* * If the BSP reported NXE support, enable EFER.NXE for all APs * prior to loading %cr3. This avoids page faults if the AP * encounters memory marked with the NX bit prior to detecting and * enabling NXE support. */ movq pg_nx, %rbx testq %rbx, %rbx je 1f movl $MSR_EFER, %ecx rdmsr orl $EFER_NXE, %eax wrmsr 1: /* * Load a real %cr3 that has all the direct map stuff and switches * off the 1GB replicated mirror. Load a stack pointer and jump * into AP startup code in C. */ movq KPML4phys, %rax movq %rax, %cr3 movq bootSTK, %rsp jmp init_secondary Index: head/sys/amd64/amd64/sigtramp.S =================================================================== --- head/sys/amd64/amd64/sigtramp.S (revision 331253) +++ head/sys/amd64/amd64/sigtramp.S (revision 331254) @@ -1,56 +1,56 @@ /*- * Copyright (c) 2003 Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include -#include "assym.s" +#include "assym.inc" .text /********************************************************************** * * Signal trampoline, copied to top of user stack * */ NON_GPROF_ENTRY(sigcode) call *SIGF_HANDLER(%rsp) /* call signal handler */ lea SIGF_UC(%rsp),%rdi /* get ucontext_t */ pushq $0 /* junk to fake return addr. */ movq $SYS_sigreturn,%rax syscall /* enter kernel with args */ 0: hlt /* trap priviliged instruction */ jmp 0b ALIGN_TEXT esigcode: .data .globl szsigcode szsigcode: .long esigcode-sigcode Index: head/sys/amd64/amd64/support.S =================================================================== --- head/sys/amd64/amd64/support.S (revision 331253) +++ head/sys/amd64/amd64/support.S (revision 331254) @@ -1,917 +1,917 @@ /*- * Copyright (c) 2003 Peter Wemm. * Copyright (c) 1993 The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_ddb.h" #include #include #include -#include "assym.s" +#include "assym.inc" .text /* * bcopy family * void bzero(void *buf, u_int len) */ /* done */ ENTRY(bzero) PUSH_FRAME_POINTER movq %rsi,%rcx xorl %eax,%eax shrq $3,%rcx rep stosq movq %rsi,%rcx andq $7,%rcx rep stosb POP_FRAME_POINTER ret END(bzero) /* Address: %rdi */ ENTRY(pagezero) PUSH_FRAME_POINTER movq $PAGE_SIZE/8,%rcx xorl %eax,%eax rep stosq POP_FRAME_POINTER ret END(pagezero) /* Address: %rdi */ ENTRY(sse2_pagezero) PUSH_FRAME_POINTER movq $-PAGE_SIZE,%rdx subq %rdx,%rdi xorl %eax,%eax jmp 1f /* * The loop takes 29 bytes. Ensure that it doesn't cross a 32-byte * cache line. */ .p2align 5,0x90 1: movnti %rax,(%rdi,%rdx) movnti %rax,8(%rdi,%rdx) movnti %rax,16(%rdi,%rdx) movnti %rax,24(%rdi,%rdx) addq $32,%rdx jne 1b sfence POP_FRAME_POINTER ret END(sse2_pagezero) ENTRY(bcmp) PUSH_FRAME_POINTER movq %rdx,%rcx shrq $3,%rcx repe cmpsq jne 1f movq %rdx,%rcx andq $7,%rcx repe cmpsb 1: setne %al movsbl %al,%eax POP_FRAME_POINTER ret END(bcmp) /* * bcopy(src, dst, cnt) * rdi, rsi, rdx * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 */ ENTRY(bcopy) PUSH_FRAME_POINTER xchgq %rsi,%rdi movq %rdx,%rcx movq %rdi,%rax subq %rsi,%rax cmpq %rcx,%rax /* overlapping && src < dst? */ jb 1f shrq $3,%rcx /* copy by 64-bit words */ rep movsq movq %rdx,%rcx andq $7,%rcx /* any bytes left? */ rep movsb POP_FRAME_POINTER ret /* ALIGN_TEXT */ 1: addq %rcx,%rdi /* copy backwards */ addq %rcx,%rsi decq %rdi decq %rsi andq $7,%rcx /* any fractional bytes? */ std rep movsb movq %rdx,%rcx /* copy remainder by 32-bit words */ shrq $3,%rcx subq $7,%rsi subq $7,%rdi rep movsq cld POP_FRAME_POINTER ret END(bcopy) /* * Note: memcpy does not support overlapping copies */ ENTRY(memcpy) PUSH_FRAME_POINTER movq %rdi,%rax movq %rdx,%rcx shrq $3,%rcx /* copy by 64-bit words */ rep movsq movq %rdx,%rcx andq $7,%rcx /* any bytes left? */ rep movsb POP_FRAME_POINTER ret END(memcpy) /* * pagecopy(%rdi=from, %rsi=to) */ ENTRY(pagecopy) PUSH_FRAME_POINTER movq $-PAGE_SIZE,%rax movq %rax,%rdx subq %rax,%rdi subq %rax,%rsi 1: prefetchnta (%rdi,%rax) addq $64,%rax jne 1b 2: movq (%rdi,%rdx),%rax movnti %rax,(%rsi,%rdx) movq 8(%rdi,%rdx),%rax movnti %rax,8(%rsi,%rdx) movq 16(%rdi,%rdx),%rax movnti %rax,16(%rsi,%rdx) movq 24(%rdi,%rdx),%rax movnti %rax,24(%rsi,%rdx) addq $32,%rdx jne 2b sfence POP_FRAME_POINTER ret END(pagecopy) /* fillw(pat, base, cnt) */ /* %rdi,%rsi, %rdx */ ENTRY(fillw) PUSH_FRAME_POINTER movq %rdi,%rax movq %rsi,%rdi movq %rdx,%rcx rep stosw POP_FRAME_POINTER ret END(fillw) /*****************************************************************************/ /* copyout and fubyte family */ /*****************************************************************************/ /* * Access user memory from inside the kernel. These routines should be * the only places that do this. * * These routines set curpcb->pcb_onfault for the time they execute. When a * protection violation occurs inside the functions, the trap handler * returns to *curpcb->pcb_onfault instead of the function. */ /* * copyout(from_kernel, to_user, len) * %rdi, %rsi, %rdx */ ENTRY(copyout) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rax movq $copyout_fault,PCB_ONFAULT(%rax) testq %rdx,%rdx /* anything to do? */ jz done_copyout /* * Check explicitly for non-user addresses. This check is essential * because it prevents usermode from writing into the kernel. We do * not verify anywhere else that the user did not specify a rogue * address. */ /* * First, prevent address wrapping. */ movq %rsi,%rax addq %rdx,%rax jc copyout_fault /* * XXX STOP USING VM_MAXUSER_ADDRESS. * It is an end address, not a max, so every time it is used correctly it * looks like there is an off by one error, and of course it caused an off * by one error in several places. */ movq $VM_MAXUSER_ADDRESS,%rcx cmpq %rcx,%rax ja copyout_fault xchgq %rdi,%rsi /* bcopy(%rsi, %rdi, %rdx) */ movq %rdx,%rcx shrq $3,%rcx rep movsq movb %dl,%cl andb $7,%cl rep movsb done_copyout: xorl %eax,%eax movq PCPU(CURPCB),%rdx movq %rax,PCB_ONFAULT(%rdx) POP_FRAME_POINTER ret ALIGN_TEXT copyout_fault: movq PCPU(CURPCB),%rdx movq $0,PCB_ONFAULT(%rdx) movq $EFAULT,%rax POP_FRAME_POINTER ret END(copyout) /* * copyin(from_user, to_kernel, len) * %rdi, %rsi, %rdx */ ENTRY(copyin) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rax movq $copyin_fault,PCB_ONFAULT(%rax) testq %rdx,%rdx /* anything to do? */ jz done_copyin /* * make sure address is valid */ movq %rdi,%rax addq %rdx,%rax jc copyin_fault movq $VM_MAXUSER_ADDRESS,%rcx cmpq %rcx,%rax ja copyin_fault xchgq %rdi,%rsi movq %rdx,%rcx movb %cl,%al shrq $3,%rcx /* copy longword-wise */ rep movsq movb %al,%cl andb $7,%cl /* copy remaining bytes */ rep movsb done_copyin: xorl %eax,%eax movq PCPU(CURPCB),%rdx movq %rax,PCB_ONFAULT(%rdx) POP_FRAME_POINTER ret ALIGN_TEXT copyin_fault: movq PCPU(CURPCB),%rdx movq $0,PCB_ONFAULT(%rdx) movq $EFAULT,%rax POP_FRAME_POINTER ret END(copyin) /* * casueword32. Compare and set user integer. Returns -1 on fault, * 0 if access was successful. Old value is written to *oldp. * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx */ ENTRY(casueword32) PUSH_FRAME_POINTER movq PCPU(CURPCB),%r8 movq $fusufault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rdi /* verify address is valid */ ja fusufault movl %esi,%eax /* old */ #ifdef SMP lock #endif cmpxchgl %ecx,(%rdi) /* new = %ecx */ /* * The old value is in %eax. If the store succeeded it will be the * value we expected (old) from before the store, otherwise it will * be the current value. Save %eax into %esi to prepare the return * value. */ movl %eax,%esi xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) /* * Access the oldp after the pcb_onfault is cleared, to correctly * catch corrupted pointer. */ movl %esi,(%rdx) /* oldp = %rdx */ POP_FRAME_POINTER ret END(casueword32) /* * casueword. Compare and set user long. Returns -1 on fault, * 0 if access was successful. Old value is written to *oldp. * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx */ ENTRY(casueword) PUSH_FRAME_POINTER movq PCPU(CURPCB),%r8 movq $fusufault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rdi /* verify address is valid */ ja fusufault movq %rsi,%rax /* old */ #ifdef SMP lock #endif cmpxchgq %rcx,(%rdi) /* new = %rcx */ /* * The old value is in %rax. If the store succeeded it will be the * value we expected (old) from before the store, otherwise it will * be the current value. */ movq %rax,%rsi xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) movq %rsi,(%rdx) POP_FRAME_POINTER ret END(casueword) /* * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit * byte from user memory. * addr = %rdi, valp = %rsi */ ALTENTRY(fueword64) ENTRY(fueword) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-8,%rax cmpq %rax,%rdi /* verify address is valid */ ja fusufault xorl %eax,%eax movq (%rdi),%r11 movq %rax,PCB_ONFAULT(%rcx) movq %r11,(%rsi) POP_FRAME_POINTER ret END(fueword64) END(fueword) ENTRY(fueword32) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rdi /* verify address is valid */ ja fusufault xorl %eax,%eax movl (%rdi),%r11d movq %rax,PCB_ONFAULT(%rcx) movl %r11d,(%rsi) POP_FRAME_POINTER ret END(fueword32) /* * fuswintr() and suswintr() are specialized variants of fuword16() and * suword16(), respectively. They are called from the profiling code, * potentially at interrupt time. If they fail, that's okay; good things * will happen later. They always fail for now, until the trap code is * able to deal with this. */ ALTENTRY(suswintr) ENTRY(fuswintr) movq $-1,%rax ret END(suswintr) END(fuswintr) ENTRY(fuword16) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-2,%rax cmpq %rax,%rdi ja fusufault movzwl (%rdi),%eax movq $0,PCB_ONFAULT(%rcx) POP_FRAME_POINTER ret END(fuword16) ENTRY(fubyte) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-1,%rax cmpq %rax,%rdi ja fusufault movzbl (%rdi),%eax movq $0,PCB_ONFAULT(%rcx) POP_FRAME_POINTER ret END(fubyte) ALIGN_TEXT fusufault: movq PCPU(CURPCB),%rcx xorl %eax,%eax movq %rax,PCB_ONFAULT(%rcx) decq %rax POP_FRAME_POINTER ret /* * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to * user memory. * addr = %rdi, value = %rsi */ ALTENTRY(suword64) ENTRY(suword) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-8,%rax cmpq %rax,%rdi /* verify address validity */ ja fusufault movq %rsi,(%rdi) xorl %eax,%eax movq PCPU(CURPCB),%rcx movq %rax,PCB_ONFAULT(%rcx) POP_FRAME_POINTER ret END(suword64) END(suword) ENTRY(suword32) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rdi /* verify address validity */ ja fusufault movl %esi,(%rdi) xorl %eax,%eax movq PCPU(CURPCB),%rcx movq %rax,PCB_ONFAULT(%rcx) POP_FRAME_POINTER ret END(suword32) ENTRY(suword16) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-2,%rax cmpq %rax,%rdi /* verify address validity */ ja fusufault movw %si,(%rdi) xorl %eax,%eax movq PCPU(CURPCB),%rcx /* restore trashed register */ movq %rax,PCB_ONFAULT(%rcx) POP_FRAME_POINTER ret END(suword16) ENTRY(subyte) PUSH_FRAME_POINTER movq PCPU(CURPCB),%rcx movq $fusufault,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS-1,%rax cmpq %rax,%rdi /* verify address validity */ ja fusufault movl %esi,%eax movb %al,(%rdi) xorl %eax,%eax movq PCPU(CURPCB),%rcx /* restore trashed register */ movq %rax,PCB_ONFAULT(%rcx) POP_FRAME_POINTER ret END(subyte) /* * copyinstr(from, to, maxlen, int *lencopied) * %rdi, %rsi, %rdx, %rcx * * copy a string from 'from' to 'to', stop when a 0 character is reached. * return ENAMETOOLONG if string is longer than maxlen, and * EFAULT on protection violations. If lencopied is non-zero, * return the actual length in *lencopied. */ ENTRY(copyinstr) PUSH_FRAME_POINTER movq %rdx,%r8 /* %r8 = maxlen */ movq %rcx,%r9 /* %r9 = *len */ xchgq %rdi,%rsi /* %rdi = from, %rsi = to */ movq PCPU(CURPCB),%rcx movq $cpystrflt,PCB_ONFAULT(%rcx) movq $VM_MAXUSER_ADDRESS,%rax /* make sure 'from' is within bounds */ subq %rsi,%rax jbe cpystrflt /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */ cmpq %rdx,%rax jae 1f movq %rax,%rdx movq %rax,%r8 1: incq %rdx 2: decq %rdx jz 3f lodsb stosb orb %al,%al jnz 2b /* Success -- 0 byte reached */ decq %rdx xorl %eax,%eax jmp cpystrflt_x 3: /* rdx is zero - return ENAMETOOLONG or EFAULT */ movq $VM_MAXUSER_ADDRESS,%rax cmpq %rax,%rsi jae cpystrflt 4: movq $ENAMETOOLONG,%rax jmp cpystrflt_x cpystrflt: movq $EFAULT,%rax cpystrflt_x: /* set *lencopied and return %eax */ movq PCPU(CURPCB),%rcx movq $0,PCB_ONFAULT(%rcx) testq %r9,%r9 jz 1f subq %rdx,%r8 movq %r8,(%r9) 1: POP_FRAME_POINTER ret END(copyinstr) /* * copystr(from, to, maxlen, int *lencopied) * %rdi, %rsi, %rdx, %rcx */ ENTRY(copystr) PUSH_FRAME_POINTER movq %rdx,%r8 /* %r8 = maxlen */ xchgq %rdi,%rsi incq %rdx 1: decq %rdx jz 4f lodsb stosb orb %al,%al jnz 1b /* Success -- 0 byte reached */ decq %rdx xorl %eax,%eax jmp 6f 4: /* rdx is zero -- return ENAMETOOLONG */ movq $ENAMETOOLONG,%rax 6: testq %rcx,%rcx jz 7f /* set *lencopied and return %rax */ subq %rdx,%r8 movq %r8,(%rcx) 7: POP_FRAME_POINTER ret END(copystr) /* * Handling of special amd64 registers and descriptor tables etc */ /* void lgdt(struct region_descriptor *rdp); */ ENTRY(lgdt) /* reload the descriptor table */ lgdt (%rdi) /* flush the prefetch q */ jmp 1f nop 1: movl $KDSEL,%eax movl %eax,%ds movl %eax,%es movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */ movl %eax,%gs movl %eax,%ss /* reload code selector by turning return into intersegmental return */ popq %rax pushq $KCSEL pushq %rax MEXITCOUNT lretq END(lgdt) /*****************************************************************************/ /* setjump, longjump */ /*****************************************************************************/ ENTRY(setjmp) movq %rbx,0(%rdi) /* save rbx */ movq %rsp,8(%rdi) /* save rsp */ movq %rbp,16(%rdi) /* save rbp */ movq %r12,24(%rdi) /* save r12 */ movq %r13,32(%rdi) /* save r13 */ movq %r14,40(%rdi) /* save r14 */ movq %r15,48(%rdi) /* save r15 */ movq 0(%rsp),%rdx /* get rta */ movq %rdx,56(%rdi) /* save rip */ xorl %eax,%eax /* return(0); */ ret END(setjmp) ENTRY(longjmp) movq 0(%rdi),%rbx /* restore rbx */ movq 8(%rdi),%rsp /* restore rsp */ movq 16(%rdi),%rbp /* restore rbp */ movq 24(%rdi),%r12 /* restore r12 */ movq 32(%rdi),%r13 /* restore r13 */ movq 40(%rdi),%r14 /* restore r14 */ movq 48(%rdi),%r15 /* restore r15 */ movq 56(%rdi),%rdx /* get rta */ movq %rdx,0(%rsp) /* put in return frame */ xorl %eax,%eax /* return(1); */ incl %eax ret END(longjmp) /* * Support for reading MSRs in the safe manner. (Instead of panic on #gp, * return an error.) */ ENTRY(rdmsr_safe) /* int rdmsr_safe(u_int msr, uint64_t *data) */ PUSH_FRAME_POINTER movq PCPU(CURPCB),%r8 movq $msr_onfault,PCB_ONFAULT(%r8) movl %edi,%ecx rdmsr /* Read MSR pointed by %ecx. Returns hi byte in edx, lo in %eax */ salq $32,%rdx /* sign-shift %rdx left */ movl %eax,%eax /* zero-extend %eax -> %rax */ orq %rdx,%rax movq %rax,(%rsi) xorq %rax,%rax movq %rax,PCB_ONFAULT(%r8) POP_FRAME_POINTER ret /* * Support for writing MSRs in the safe manner. (Instead of panic on #gp, * return an error.) */ ENTRY(wrmsr_safe) /* int wrmsr_safe(u_int msr, uint64_t data) */ PUSH_FRAME_POINTER movq PCPU(CURPCB),%r8 movq $msr_onfault,PCB_ONFAULT(%r8) movl %edi,%ecx movl %esi,%eax sarq $32,%rsi movl %esi,%edx wrmsr /* Write MSR pointed by %ecx. Accepts hi byte in edx, lo in %eax. */ xorq %rax,%rax movq %rax,PCB_ONFAULT(%r8) POP_FRAME_POINTER ret /* * MSR operations fault handler */ ALIGN_TEXT msr_onfault: movq $0,PCB_ONFAULT(%r8) movl $EFAULT,%eax POP_FRAME_POINTER ret /* * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3); * Invalidates address space addressed by ucr3, then returns to kcr3. * Done in assembler to ensure no other memory accesses happen while * on ucr3. */ ALIGN_TEXT ENTRY(pmap_pti_pcid_invalidate) pushfq cli movq %rdi,%cr3 /* to user page table */ movq %rsi,%cr3 /* back to kernel */ popfq retq /* * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va); * Invalidates virtual address va in address space ucr3, then returns to kcr3. */ ALIGN_TEXT ENTRY(pmap_pti_pcid_invlpg) pushfq cli movq %rdi,%cr3 /* to user page table */ invlpg (%rdx) movq %rsi,%cr3 /* back to kernel */ popfq retq /* * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva, * vm_offset_t eva); * Invalidates virtual addresses between sva and eva in address space ucr3, * then returns to kcr3. */ ALIGN_TEXT ENTRY(pmap_pti_pcid_invlrng) pushfq cli movq %rdi,%cr3 /* to user page table */ 1: invlpg (%rdx) addq $PAGE_SIZE,%rdx cmpq %rdx,%rcx ja 1b movq %rsi,%cr3 /* back to kernel */ popfq retq .altmacro .macro ibrs_seq_label l handle_ibrs_\l: .endm .macro ibrs_call_label l call handle_ibrs_\l .endm .macro ibrs_seq count ll=1 .rept \count ibrs_call_label %(ll) nop ibrs_seq_label %(ll) addq $8,%rsp ll=ll+1 .endr .endm /* all callers already saved %rax, %rdx, and %rcx */ ENTRY(handle_ibrs_entry) cmpb $0,hw_ibrs_active(%rip) je 1f movl $MSR_IA32_SPEC_CTRL,%ecx movl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax movl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx wrmsr movb $1,PCPU(IBPB_SET) testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip) jne 1f ibrs_seq 32 1: ret END(handle_ibrs_entry) ENTRY(handle_ibrs_exit) cmpb $0,PCPU(IBPB_SET) je 1f movl $MSR_IA32_SPEC_CTRL,%ecx xorl %eax,%eax xorl %edx,%edx wrmsr movb $0,PCPU(IBPB_SET) 1: ret END(handle_ibrs_exit) /* registers-neutral version, but needs stack */ ENTRY(handle_ibrs_exit_rs) cmpb $0,PCPU(IBPB_SET) je 1f pushq %rax pushq %rdx pushq %rcx movl $MSR_IA32_SPEC_CTRL,%ecx xorl %eax,%eax xorl %edx,%edx wrmsr popq %rcx popq %rdx popq %rax movb $0,PCPU(IBPB_SET) 1: ret END(handle_ibrs_exit_rs) .noaltmacro Index: head/sys/amd64/amd64/xen-locore.S =================================================================== --- head/sys/amd64/amd64/xen-locore.S (revision 331253) +++ head/sys/amd64/amd64/xen-locore.S (revision 331254) @@ -1,87 +1,87 @@ /*- * Copyright (c) 2003 Peter Wemm * Copyright (c) 2011-2012 Spectra Logic Corporation * Copyright (c) 2013 Roger Pau Monne * All rights reserved. * * This software was developed by Cherry G. Mathew * under sponsorship from Spectra Logic Corporation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #define __ASSEMBLY__ #include -#include "assym.s" +#include "assym.inc" .section __xen_guest ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz, "FreeBSD") ELFNOTE(Xen, XEN_ELFNOTE_GUEST_VERSION, .asciz, __XSTRING(__FreeBSD_version)) ELFNOTE(Xen, XEN_ELFNOTE_XEN_VERSION, .asciz, "xen-3.0") ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE, .quad, KERNBASE) ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET, .quad, 0) ELFNOTE(Xen, XEN_ELFNOTE_ENTRY, .quad, xen_start) ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, .quad, hypercall_page) ELFNOTE(Xen, XEN_ELFNOTE_HV_START_LOW, .quad, HYPERVISOR_VIRT_START) ELFNOTE(Xen, XEN_ELFNOTE_FEATURES, .asciz, "writable_descriptor_tables|auto_translated_physmap|supervisor_mode_kernel|hvm_callback_vector") ELFNOTE(Xen, XEN_ELFNOTE_PAE_MODE, .asciz, "yes") ELFNOTE(Xen, XEN_ELFNOTE_L1_MFN_VALID, .long, PG_V, PG_V) ELFNOTE(Xen, XEN_ELFNOTE_LOADER, .asciz, "generic") ELFNOTE(Xen, XEN_ELFNOTE_SUSPEND_CANCEL, .long, 0) ELFNOTE(Xen, XEN_ELFNOTE_BSD_SYMTAB, .asciz, "yes") .text .p2align PAGE_SHIFT, 0x90 /* Hypercall_page needs to be PAGE aligned */ NON_GPROF_ENTRY(hypercall_page) .skip 0x1000, 0x90 /* Fill with "nop"s */ NON_GPROF_ENTRY(xen_start) /* Don't trust what the loader gives for rflags. */ pushq $PSL_KERNEL popfq /* Parameters for the xen init function */ movq %rsi, %rdi /* shared_info (arg 1) */ movq %rsp, %rsi /* xenstack (arg 2) */ /* Use our own stack */ movq $bootstack,%rsp xorl %ebp, %ebp /* u_int64_t hammer_time_xen(start_info_t *si, u_int64_t xenstack); */ call hammer_time_xen movq %rax, %rsp /* set up kstack for mi_startup() */ call mi_startup /* autoconfiguration, mountroot etc */ /* NOTREACHED */ 0: hlt jmp 0b Index: head/sys/amd64/ia32/ia32_exception.S =================================================================== --- head/sys/amd64/ia32/ia32_exception.S (revision 331253) +++ head/sys/amd64/ia32/ia32_exception.S (revision 331254) @@ -1,78 +1,78 @@ /*- * Copyright (c) 2003 Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include -#include "assym.s" +#include "assym.inc" .text /* * Call gate entry for FreeBSD ELF and Linux/NetBSD syscall (int 0x80) * * This is a SDT_SYSIDT entry point (unlike the i386 port) so that we * can do a swapgs before enabling interrupts. This is critical because * if we took an interrupt before swapgs, the interrupt code would see * that it originated in supervisor mode and skip the swapgs. */ SUPERALIGN_TEXT IDTVEC(int0x80_syscall_pti) PTI_UENTRY has_err=0 jmp int0x80_syscall_common SUPERALIGN_TEXT IDTVEC(int0x80_syscall) swapgs int0x80_syscall_common: pushq $2 /* sizeof "int 0x80" */ subq $TF_ERR,%rsp /* skip over tf_trapno */ movq %rdi,TF_RDI(%rsp) movq PCPU(CURPCB),%rdi andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) SAVE_SEGS movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) movq %rcx,TF_RCX(%rsp) call handle_ibrs_entry sti movq %rsi,TF_RSI(%rsp) movq %r8,TF_R8(%rsp) movq %r9,TF_R9(%rsp) movq %rbx,TF_RBX(%rsp) movq %rbp,TF_RBP(%rsp) movq %r10,TF_R10(%rsp) movq %r11,TF_R11(%rsp) movq %r12,TF_R12(%rsp) movq %r13,TF_R13(%rsp) movq %r14,TF_R14(%rsp) movq %r15,TF_R15(%rsp) movl $TF_HASSEGS,TF_FLAGS(%rsp) cld FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp, %rdi call ia32_syscall MEXITCOUNT jmp doreti Index: head/sys/amd64/linux/linux_support.s =================================================================== --- head/sys/amd64/linux/linux_support.s (revision 331253) +++ head/sys/amd64/linux/linux_support.s (revision 331254) @@ -1,123 +1,123 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2007 Konstantin Belousov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "linux_assym.h" /* system definitions */ #include /* miscellaneous asm macros */ -#include "assym.s" +#include "assym.inc" futex_fault: movq $0,PCB_ONFAULT(%r8) movl $-EFAULT,%eax ret ENTRY(futex_xchgl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault xchgl %edi,(%rsi) movl %edi,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_addl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault #ifdef SMP lock #endif xaddl %edi,(%rsi) movl %edi,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_orl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx orl %edi,%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_andl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx andl %edi,%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_xorl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx xorl %edi,%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret Index: head/sys/amd64/linux32/linux32_support.s =================================================================== --- head/sys/amd64/linux32/linux32_support.s (revision 331253) +++ head/sys/amd64/linux32/linux32_support.s (revision 331254) @@ -1,123 +1,123 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2007 Konstantin Belousov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "linux32_assym.h" /* system definitions */ #include /* miscellaneous asm macros */ -#include "assym.s" +#include "assym.inc" futex_fault: movq $0,PCB_ONFAULT(%r8) movl $-EFAULT,%eax ret ENTRY(futex_xchgl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault xchgl %edi,(%rsi) movl %edi,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_addl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault #ifdef SMP lock #endif xaddl %edi,(%rsi) movl %edi,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_orl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx orl %edi,%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_andl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx andl %edi,%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret ENTRY(futex_xorl) movq PCPU(CURPCB),%r8 movq $futex_fault,PCB_ONFAULT(%r8) movq $VM_MAXUSER_ADDRESS-4,%rax cmpq %rax,%rsi ja futex_fault movl (%rsi),%eax 1: movl %eax,%ecx xorl %edi,%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%rsi) jnz 1b movl %eax,(%rdx) xorl %eax,%eax movq %rax,PCB_ONFAULT(%r8) ret Index: head/sys/amd64/sgx/sgx_support.S =================================================================== --- head/sys/amd64/sgx/sgx_support.S (revision 331253) +++ head/sys/amd64/sgx/sgx_support.S (revision 331254) @@ -1,70 +1,70 @@ /*- * Copyright (c) 2017 Ruslan Bukin * All rights reserved. * * This software was developed by BAE Systems, the University of Cambridge * Computer Laboratory, and Memorial University under DARPA/AFRL contract * FA8650-15-C-7558 ("CADETS"), as part of the DARPA Transparent Computing * (TC) research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include -#include "assym.s" +#include "assym.inc" .text /* * int * sgx_encls(uint32_t eax, uint64_t rbx, uint64_t rcx, uint64_t rdx); * %edi, %rsi, %rdx, %rcx */ ENTRY(sgx_encls) PUSH_FRAME_POINTER pushq %rbx movq PCPU(CURPCB),%r8 movl %edi,%eax movq %rsi,%rbx xchgq %rdx,%rcx movq $sgx_onfault,PCB_ONFAULT(%r8) .byte 0x0f, 0x01, 0xcf movq $0,PCB_ONFAULT(%r8) popq %rbx POP_FRAME_POINTER ret END(sgx_encls) /* * SGX operations fault handler */ ALIGN_TEXT sgx_onfault: movq $0,PCB_ONFAULT(%r8) movl $SGX_EFAULT,%eax popq %rbx POP_FRAME_POINTER ret Index: head/sys/arm/arm/bcopy_page.S =================================================================== --- head/sys/arm/arm/bcopy_page.S (revision 331253) +++ head/sys/arm/arm/bcopy_page.S (revision 331254) @@ -1,282 +1,282 @@ /* $NetBSD: bcopy_page.S,v 1.7 2003/10/13 21:03:13 scw Exp $ */ /*- * Copyright (c) 1995 Scott Stevens * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Scott Stevens. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * RiscBSD kernel project * * bcopy_page.S * * page optimised bcopy and bzero routines * * Created : 08/04/95 */ #include __FBSDID("$FreeBSD$"); -#include "assym.s" +#include "assym.inc" #ifndef _ARM_ARCH_5E /* #define BIG_LOOPS */ /* * bcopy_page(src, dest) * * Optimised copy page routine. * * On entry: * r0 - src address * r1 - dest address * * Requires: * number of bytes per page (PAGE_SIZE) is a multiple of 512 (BIG_LOOPS), 128 * otherwise. */ #define CHUNK_SIZE 32 #define PREFETCH_FIRST_CHUNK /* nothing */ #define PREFETCH_NEXT_CHUNK /* nothing */ #ifndef COPY_CHUNK #define COPY_CHUNK \ PREFETCH_NEXT_CHUNK ; \ ldmia r0!, {r3-r8,ip,lr} ; \ stmia r1!, {r3-r8,ip,lr} #endif /* ! COPY_CHUNK */ #ifndef SAVE_REGS #define SAVE_REGS stmfd sp!, {r4-r8, lr}; _SAVE({r4-r8, lr}) #define RESTORE_REGS ldmfd sp!, {r4-r8, pc} #endif ENTRY(bcopy_page) PREFETCH_FIRST_CHUNK SAVE_REGS #ifdef BIG_LOOPS mov r2, #(PAGE_SIZE >> 9) #else mov r2, #(PAGE_SIZE >> 7) #endif 1: COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK #ifdef BIG_LOOPS /* There is little point making the loop any larger; unless we are running with the cache off, the load/store overheads will completely dominate this loop. */ COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK COPY_CHUNK #endif subs r2, r2, #1 bne 1b RESTORE_REGS /* ...and return. */ END(bcopy_page) /* * bzero_page(dest) * * Optimised zero page routine. * * On entry: * r0 - dest address * * Requires: * number of bytes per page (PAGE_SIZE) is a multiple of 512 (BIG_LOOPS), 128 * otherwise */ ENTRY(bzero_page) stmfd sp!, {r4-r8, lr} _SAVE({r4-r8, lr}) #ifdef BIG_LOOPS mov r2, #(PAGE_SIZE >> 9) #else mov r2, #(PAGE_SIZE >> 7) #endif mov r3, #0 mov r4, #0 mov r5, #0 mov r6, #0 mov r7, #0 mov r8, #0 mov ip, #0 mov lr, #0 1: stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} #ifdef BIG_LOOPS /* There is little point making the loop any larger; unless we are running with the cache off, the load/store overheads will completely dominate this loop. */ stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} stmia r0!, {r3-r8,ip,lr} #endif subs r2, r2, #1 bne 1b ldmfd sp!, {r4-r8, pc} END(bzero_page) #else /* _ARM_ARCH_5E */ /* * armv5e version of bcopy_page */ ENTRY(bcopy_page) pld [r0] stmfd sp!, {r4, r5} _SAVE({r4, r5}) mov ip, #32 ldr r2, [r0], #0x04 /* 0x00 */ ldr r3, [r0], #0x04 /* 0x04 */ 1: pld [r0, #0x18] /* Prefetch 0x20 */ ldr r4, [r0], #0x04 /* 0x08 */ ldr r5, [r0], #0x04 /* 0x0c */ strd r2, [r1], #0x08 ldr r2, [r0], #0x04 /* 0x10 */ ldr r3, [r0], #0x04 /* 0x14 */ strd r4, [r1], #0x08 ldr r4, [r0], #0x04 /* 0x18 */ ldr r5, [r0], #0x04 /* 0x1c */ strd r2, [r1], #0x08 ldr r2, [r0], #0x04 /* 0x20 */ ldr r3, [r0], #0x04 /* 0x24 */ pld [r0, #0x18] /* Prefetch 0x40 */ strd r4, [r1], #0x08 ldr r4, [r0], #0x04 /* 0x28 */ ldr r5, [r0], #0x04 /* 0x2c */ strd r2, [r1], #0x08 ldr r2, [r0], #0x04 /* 0x30 */ ldr r3, [r0], #0x04 /* 0x34 */ strd r4, [r1], #0x08 ldr r4, [r0], #0x04 /* 0x38 */ ldr r5, [r0], #0x04 /* 0x3c */ strd r2, [r1], #0x08 ldr r2, [r0], #0x04 /* 0x40 */ ldr r3, [r0], #0x04 /* 0x44 */ pld [r0, #0x18] /* Prefetch 0x60 */ strd r4, [r1], #0x08 ldr r4, [r0], #0x04 /* 0x48 */ ldr r5, [r0], #0x04 /* 0x4c */ strd r2, [r1], #0x08 ldr r2, [r0], #0x04 /* 0x50 */ ldr r3, [r0], #0x04 /* 0x54 */ strd r4, [r1], #0x08 ldr r4, [r0], #0x04 /* 0x58 */ ldr r5, [r0], #0x04 /* 0x5c */ strd r2, [r1], #0x08 ldr r2, [r0], #0x04 /* 0x60 */ ldr r3, [r0], #0x04 /* 0x64 */ pld [r0, #0x18] /* Prefetch 0x80 */ strd r4, [r1], #0x08 ldr r4, [r0], #0x04 /* 0x68 */ ldr r5, [r0], #0x04 /* 0x6c */ strd r2, [r1], #0x08 ldr r2, [r0], #0x04 /* 0x70 */ ldr r3, [r0], #0x04 /* 0x74 */ strd r4, [r1], #0x08 ldr r4, [r0], #0x04 /* 0x78 */ ldr r5, [r0], #0x04 /* 0x7c */ strd r2, [r1], #0x08 subs ip, ip, #0x01 ldrgt r2, [r0], #0x04 /* 0x80 */ ldrgt r3, [r0], #0x04 /* 0x84 */ strd r4, [r1], #0x08 bgt 1b ldmfd sp!, {r4, r5} RET END(bcopy_page) /* * armv5e version of bzero_page */ ENTRY(bzero_page) mov r1, #PAGE_SIZE mov r2, #0 mov r3, #0 1: strd r2, [r0], #8 /* 32 */ strd r2, [r0], #8 strd r2, [r0], #8 strd r2, [r0], #8 strd r2, [r0], #8 /* 64 */ strd r2, [r0], #8 strd r2, [r0], #8 strd r2, [r0], #8 strd r2, [r0], #8 /* 96 */ strd r2, [r0], #8 strd r2, [r0], #8 strd r2, [r0], #8 strd r2, [r0], #8 /* 128 */ strd r2, [r0], #8 strd r2, [r0], #8 strd r2, [r0], #8 subs r1, r1, #128 bne 1b RET END(bzero_page) #endif /* _ARM_ARCH_5E */ Index: head/sys/arm/arm/bcopyinout.S =================================================================== --- head/sys/arm/arm/bcopyinout.S (revision 331253) +++ head/sys/arm/arm/bcopyinout.S (revision 331254) @@ -1,641 +1,641 @@ /* $NetBSD: bcopyinout.S,v 1.11 2003/10/13 21:22:40 scw Exp $ */ /*- * Copyright (c) 2002 Wasabi Systems, Inc. * All rights reserved. * * Written by Allen Briggs for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -#include "assym.s" +#include "assym.inc" #include #include .L_arm_memcpy: .word _C_LABEL(_arm_memcpy) .L_min_memcpy_size: .word _C_LABEL(_min_memcpy_size) __FBSDID("$FreeBSD$"); #ifdef _ARM_ARCH_5E #include #else .text .align 2 #if __ARM_ARCH >= 6 #define GET_PCB(tmp) \ mrc p15, 0, tmp, c13, c0, 4; \ add tmp, tmp, #(TD_PCB) #else .Lcurpcb: .word _C_LABEL(__pcpu) + PC_CURPCB #define GET_PCB(tmp) \ ldr tmp, .Lcurpcb #endif #define SAVE_REGS stmfd sp!, {r4-r11}; _SAVE({r4-r11}) #define RESTORE_REGS ldmfd sp!, {r4-r11} #if defined(_ARM_ARCH_5E) #define HELLOCPP # #define PREFETCH(rx,o) pld [ rx , HELLOCPP (o) ] #else #define PREFETCH(rx,o) #endif /* * r0 = user space address * r1 = kernel space address * r2 = length * * Copies bytes from user space to kernel space * * We save/restore r4-r11: * r4-r11 are scratch */ ENTRY(copyin) /* Quick exit if length is zero */ teq r2, #0 moveq r0, #0 RETeq adds r3, r0, r2 movcs r0, #EFAULT RETc(cs) ldr r12, =(VM_MAXUSER_ADDRESS + 1) cmp r3, r12 movcs r0, #EFAULT RETc(cs) ldr r3, .L_arm_memcpy ldr r3, [r3] cmp r3, #0 beq .Lnormal ldr r3, .L_min_memcpy_size ldr r3, [r3] cmp r2, r3 blt .Lnormal stmfd sp!, {r0-r2, r4, lr} mov r3, r0 mov r0, r1 mov r1, r3 mov r3, #2 /* SRC_IS_USER */ ldr r4, .L_arm_memcpy mov lr, pc ldr pc, [r4] cmp r0, #0 ldmfd sp!, {r0-r2, r4, lr} moveq r0, #0 RETeq .Lnormal: SAVE_REGS GET_PCB(r4) ldr r4, [r4] ldr r5, [r4, #PCB_ONFAULT] adr r3, .Lcopyfault str r3, [r4, #PCB_ONFAULT] PREFETCH(r0, 0) PREFETCH(r1, 0) /* * If not too many bytes, take the slow path. */ cmp r2, #0x08 blt .Licleanup /* * Align destination to word boundary. */ and r6, r1, #0x3 ldr pc, [pc, r6, lsl #2] b .Lialend .word .Lialend .word .Lial3 .word .Lial2 .word .Lial1 .Lial3: ldrbt r6, [r0], #1 sub r2, r2, #1 strb r6, [r1], #1 .Lial2: ldrbt r7, [r0], #1 sub r2, r2, #1 strb r7, [r1], #1 .Lial1: ldrbt r6, [r0], #1 sub r2, r2, #1 strb r6, [r1], #1 .Lialend: /* * If few bytes left, finish slow. */ cmp r2, #0x08 blt .Licleanup /* * If source is not aligned, finish slow. */ ands r3, r0, #0x03 bne .Licleanup cmp r2, #0x60 /* Must be > 0x5f for unrolled cacheline */ blt .Licleanup8 /* * Align destination to cacheline boundary. * If source and destination are nicely aligned, this can be a big * win. If not, it's still cheaper to copy in groups of 32 even if * we don't get the nice cacheline alignment. */ and r6, r1, #0x1f ldr pc, [pc, r6] b .Licaligned .word .Licaligned .word .Lical28 .word .Lical24 .word .Lical20 .word .Lical16 .word .Lical12 .word .Lical8 .word .Lical4 .Lical28:ldrt r6, [r0], #4 sub r2, r2, #4 str r6, [r1], #4 .Lical24:ldrt r7, [r0], #4 sub r2, r2, #4 str r7, [r1], #4 .Lical20:ldrt r6, [r0], #4 sub r2, r2, #4 str r6, [r1], #4 .Lical16:ldrt r7, [r0], #4 sub r2, r2, #4 str r7, [r1], #4 .Lical12:ldrt r6, [r0], #4 sub r2, r2, #4 str r6, [r1], #4 .Lical8:ldrt r7, [r0], #4 sub r2, r2, #4 str r7, [r1], #4 .Lical4:ldrt r6, [r0], #4 sub r2, r2, #4 str r6, [r1], #4 /* * We start with > 0x40 bytes to copy (>= 0x60 got us into this * part of the code, and we may have knocked that down by as much * as 0x1c getting aligned). * * This loop basically works out to: * do { * prefetch-next-cacheline(s) * bytes -= 0x20; * copy cacheline * } while (bytes >= 0x40); * bytes -= 0x20; * copy cacheline */ .Licaligned: PREFETCH(r0, 32) PREFETCH(r1, 32) sub r2, r2, #0x20 /* Copy a cacheline */ ldrt r10, [r0], #4 ldrt r11, [r0], #4 ldrt r6, [r0], #4 ldrt r7, [r0], #4 ldrt r8, [r0], #4 ldrt r9, [r0], #4 stmia r1!, {r10-r11} ldrt r10, [r0], #4 ldrt r11, [r0], #4 stmia r1!, {r6-r11} cmp r2, #0x40 bge .Licaligned sub r2, r2, #0x20 /* Copy a cacheline */ ldrt r10, [r0], #4 ldrt r11, [r0], #4 ldrt r6, [r0], #4 ldrt r7, [r0], #4 ldrt r8, [r0], #4 ldrt r9, [r0], #4 stmia r1!, {r10-r11} ldrt r10, [r0], #4 ldrt r11, [r0], #4 stmia r1!, {r6-r11} cmp r2, #0x08 blt .Liprecleanup .Licleanup8: ldrt r8, [r0], #4 ldrt r9, [r0], #4 sub r2, r2, #8 stmia r1!, {r8, r9} cmp r2, #8 bge .Licleanup8 .Liprecleanup: /* * If we're done, bail. */ cmp r2, #0 beq .Lout .Licleanup: and r6, r2, #0x3 ldr pc, [pc, r6, lsl #2] b .Licend .word .Lic4 .word .Lic1 .word .Lic2 .word .Lic3 .Lic4: ldrbt r6, [r0], #1 sub r2, r2, #1 strb r6, [r1], #1 .Lic3: ldrbt r7, [r0], #1 sub r2, r2, #1 strb r7, [r1], #1 .Lic2: ldrbt r6, [r0], #1 sub r2, r2, #1 strb r6, [r1], #1 .Lic1: ldrbt r7, [r0], #1 subs r2, r2, #1 strb r7, [r1], #1 .Licend: bne .Licleanup .Liout: mov r0, #0 str r5, [r4, #PCB_ONFAULT] RESTORE_REGS RET .Lcopyfault: ldr r0, =EFAULT str r5, [r4, #PCB_ONFAULT] RESTORE_REGS RET END(copyin) /* * r0 = kernel space address * r1 = user space address * r2 = length * * Copies bytes from kernel space to user space * * We save/restore r4-r11: * r4-r11 are scratch */ ENTRY(copyout) /* Quick exit if length is zero */ teq r2, #0 moveq r0, #0 RETeq adds r3, r1, r2 movcs r0, #EFAULT RETc(cs) ldr r12, =(VM_MAXUSER_ADDRESS + 1) cmp r3, r12 movcs r0, #EFAULT RETc(cs) ldr r3, .L_arm_memcpy ldr r3, [r3] cmp r3, #0 beq .Lnormale ldr r3, .L_min_memcpy_size ldr r3, [r3] cmp r2, r3 blt .Lnormale stmfd sp!, {r0-r2, r4, lr} _SAVE({r0-r2, r4, lr}) mov r3, r0 mov r0, r1 mov r1, r3 mov r3, #1 /* DST_IS_USER */ ldr r4, .L_arm_memcpy mov lr, pc ldr pc, [r4] cmp r0, #0 ldmfd sp!, {r0-r2, r4, lr} moveq r0, #0 RETeq .Lnormale: SAVE_REGS GET_PCB(r4) ldr r4, [r4] ldr r5, [r4, #PCB_ONFAULT] adr r3, .Lcopyfault str r3, [r4, #PCB_ONFAULT] PREFETCH(r0, 0) PREFETCH(r1, 0) /* * If not too many bytes, take the slow path. */ cmp r2, #0x08 blt .Lcleanup /* * Align destination to word boundary. */ and r6, r1, #0x3 ldr pc, [pc, r6, lsl #2] b .Lalend .word .Lalend .word .Lal3 .word .Lal2 .word .Lal1 .Lal3: ldrb r6, [r0], #1 sub r2, r2, #1 strbt r6, [r1], #1 .Lal2: ldrb r7, [r0], #1 sub r2, r2, #1 strbt r7, [r1], #1 .Lal1: ldrb r6, [r0], #1 sub r2, r2, #1 strbt r6, [r1], #1 .Lalend: /* * If few bytes left, finish slow. */ cmp r2, #0x08 blt .Lcleanup /* * If source is not aligned, finish slow. */ ands r3, r0, #0x03 bne .Lcleanup cmp r2, #0x60 /* Must be > 0x5f for unrolled cacheline */ blt .Lcleanup8 /* * Align source & destination to cacheline boundary. */ and r6, r1, #0x1f ldr pc, [pc, r6] b .Lcaligned .word .Lcaligned .word .Lcal28 .word .Lcal24 .word .Lcal20 .word .Lcal16 .word .Lcal12 .word .Lcal8 .word .Lcal4 .Lcal28:ldr r6, [r0], #4 sub r2, r2, #4 strt r6, [r1], #4 .Lcal24:ldr r7, [r0], #4 sub r2, r2, #4 strt r7, [r1], #4 .Lcal20:ldr r6, [r0], #4 sub r2, r2, #4 strt r6, [r1], #4 .Lcal16:ldr r7, [r0], #4 sub r2, r2, #4 strt r7, [r1], #4 .Lcal12:ldr r6, [r0], #4 sub r2, r2, #4 strt r6, [r1], #4 .Lcal8: ldr r7, [r0], #4 sub r2, r2, #4 strt r7, [r1], #4 .Lcal4: ldr r6, [r0], #4 sub r2, r2, #4 strt r6, [r1], #4 /* * We start with > 0x40 bytes to copy (>= 0x60 got us into this * part of the code, and we may have knocked that down by as much * as 0x1c getting aligned). * * This loop basically works out to: * do { * prefetch-next-cacheline(s) * bytes -= 0x20; * copy cacheline * } while (bytes >= 0x40); * bytes -= 0x20; * copy cacheline */ .Lcaligned: PREFETCH(r0, 32) PREFETCH(r1, 32) sub r2, r2, #0x20 /* Copy a cacheline */ ldmia r0!, {r6-r11} strt r6, [r1], #4 strt r7, [r1], #4 ldmia r0!, {r6-r7} strt r8, [r1], #4 strt r9, [r1], #4 strt r10, [r1], #4 strt r11, [r1], #4 strt r6, [r1], #4 strt r7, [r1], #4 cmp r2, #0x40 bge .Lcaligned sub r2, r2, #0x20 /* Copy a cacheline */ ldmia r0!, {r6-r11} strt r6, [r1], #4 strt r7, [r1], #4 ldmia r0!, {r6-r7} strt r8, [r1], #4 strt r9, [r1], #4 strt r10, [r1], #4 strt r11, [r1], #4 strt r6, [r1], #4 strt r7, [r1], #4 cmp r2, #0x08 blt .Lprecleanup .Lcleanup8: ldmia r0!, {r8-r9} sub r2, r2, #8 strt r8, [r1], #4 strt r9, [r1], #4 cmp r2, #8 bge .Lcleanup8 .Lprecleanup: /* * If we're done, bail. */ cmp r2, #0 beq .Lout .Lcleanup: and r6, r2, #0x3 ldr pc, [pc, r6, lsl #2] b .Lcend .word .Lc4 .word .Lc1 .word .Lc2 .word .Lc3 .Lc4: ldrb r6, [r0], #1 sub r2, r2, #1 strbt r6, [r1], #1 .Lc3: ldrb r7, [r0], #1 sub r2, r2, #1 strbt r7, [r1], #1 .Lc2: ldrb r6, [r0], #1 sub r2, r2, #1 strbt r6, [r1], #1 .Lc1: ldrb r7, [r0], #1 subs r2, r2, #1 strbt r7, [r1], #1 .Lcend: bne .Lcleanup .Lout: mov r0, #0 str r5, [r4, #PCB_ONFAULT] RESTORE_REGS RET END(copyout) #endif /* * int badaddr_read_1(const uint8_t *src, uint8_t *dest) * * Copies a single 8-bit value from src to dest, returning 0 on success, * else EFAULT if a page fault occurred. */ ENTRY(badaddr_read_1) GET_PCB(r2) ldr r2, [r2] ldr ip, [r2, #PCB_ONFAULT] adr r3, 1f str r3, [r2, #PCB_ONFAULT] nop nop nop ldrb r3, [r0] nop nop nop strb r3, [r1] mov r0, #0 /* No fault */ 1: str ip, [r2, #PCB_ONFAULT] RET END(badaddr_read_1) /* * int badaddr_read_2(const uint16_t *src, uint16_t *dest) * * Copies a single 16-bit value from src to dest, returning 0 on success, * else EFAULT if a page fault occurred. */ ENTRY(badaddr_read_2) GET_PCB(r2) ldr r2, [r2] ldr ip, [r2, #PCB_ONFAULT] adr r3, 1f str r3, [r2, #PCB_ONFAULT] nop nop nop ldrh r3, [r0] nop nop nop strh r3, [r1] mov r0, #0 /* No fault */ 1: str ip, [r2, #PCB_ONFAULT] RET END(badaddr_read_2) /* * int badaddr_read_4(const uint32_t *src, uint32_t *dest) * * Copies a single 32-bit value from src to dest, returning 0 on success, * else EFAULT if a page fault occurred. */ ENTRY(badaddr_read_4) GET_PCB(r2) ldr r2, [r2] ldr ip, [r2, #PCB_ONFAULT] adr r3, 1f str r3, [r2, #PCB_ONFAULT] nop nop nop ldr r3, [r0] nop nop nop str r3, [r1] mov r0, #0 /* No fault */ 1: str ip, [r2, #PCB_ONFAULT] RET END(badaddr_read_4) Index: head/sys/arm/arm/copystr.S =================================================================== --- head/sys/arm/arm/copystr.S (revision 331253) +++ head/sys/arm/arm/copystr.S (revision 331254) @@ -1,224 +1,224 @@ /* $NetBSD: copystr.S,v 1.8 2002/10/13 14:54:48 bjh21 Exp $ */ /*- * Copyright (c) 1995 Mark Brinicombe. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * copystr.S * * optimised and fault protected copystr functions * * Created : 16/05/95 */ -#include "assym.s" +#include "assym.inc" #include #include __FBSDID("$FreeBSD$"); #include .text .align 2 #if __ARM_ARCH >= 6 #define GET_PCB(tmp) \ mrc p15, 0, tmp, c13, c0, 4; \ add tmp, tmp, #(TD_PCB) #else .Lpcb: .word _C_LABEL(__pcpu) + PC_CURPCB #define GET_PCB(tmp) \ ldr tmp, .Lpcb #endif /* * r0 - from * r1 - to * r2 - maxlens * r3 - lencopied * * Copy string from r0 to r1 */ ENTRY(copystr) stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ teq r2, #0x00000000 mov r5, #0x00000000 moveq r0, #ENAMETOOLONG beq 2f 1: ldrb r4, [r0], #0x0001 add r5, r5, #0x00000001 teq r4, #0x00000000 strb r4, [r1], #0x0001 teqne r5, r2 bne 1b teq r4, #0x00000000 moveq r0, #0x00000000 movne r0, #ENAMETOOLONG 2: teq r3, #0x00000000 strne r5, [r3] ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ RET END(copystr) #define SAVE_REGS stmfd sp!, {r4-r6} #define RESTORE_REGS ldmfd sp!, {r4-r6} /* * r0 - user space address * r1 - kernel space address * r2 - maxlens * r3 - lencopied * * Copy string from user space to kernel space */ ENTRY(copyinstr) SAVE_REGS teq r2, #0x00000000 mov r6, #0x00000000 moveq r0, #ENAMETOOLONG beq 2f ldr r12, =VM_MAXUSER_ADDRESS GET_PCB(r4) ldr r4, [r4] #ifdef DIAGNOSTIC teq r4, #0x00000000 beq .Lcopystrpcbfault #endif adr r5, .Lcopystrfault str r5, [r4, #PCB_ONFAULT] 1: cmp r0, r12 bcs .Lcopystrfault ldrbt r5, [r0], #0x0001 add r6, r6, #0x00000001 teq r5, #0x00000000 strb r5, [r1], #0x0001 teqne r6, r2 bne 1b mov r0, #0x00000000 str r0, [r4, #PCB_ONFAULT] teq r5, #0x00000000 moveq r0, #0x00000000 movne r0, #ENAMETOOLONG 2: teq r3, #0x00000000 strne r6, [r3] RESTORE_REGS RET END(copyinstr) /* * r0 - kernel space address * r1 - user space address * r2 - maxlens * r3 - lencopied * * Copy string from kernel space to user space */ ENTRY(copyoutstr) SAVE_REGS teq r2, #0x00000000 mov r6, #0x00000000 moveq r0, #ENAMETOOLONG beq 2f ldr r12, =VM_MAXUSER_ADDRESS GET_PCB(r4) ldr r4, [r4] #ifdef DIAGNOSTIC teq r4, #0x00000000 beq .Lcopystrpcbfault #endif adr r5, .Lcopystrfault str r5, [r4, #PCB_ONFAULT] 1: cmp r0, r12 bcs .Lcopystrfault ldrb r5, [r0], #0x0001 add r6, r6, #0x00000001 teq r5, #0x00000000 strbt r5, [r1], #0x0001 teqne r6, r2 bne 1b mov r0, #0x00000000 str r0, [r4, #PCB_ONFAULT] teq r5, #0x00000000 moveq r0, #0x00000000 movne r0, #ENAMETOOLONG 2: teq r3, #0x00000000 strne r6, [r3] RESTORE_REGS RET END(copyoutstr) /* A fault occurred during the copy */ .Lcopystrfault: mov r1, #0x00000000 str r1, [r4, #PCB_ONFAULT] mov r0, #EFAULT RESTORE_REGS RET #ifdef DIAGNOSTIC .Lcopystrpcbfault: mov r2, r1 mov r1, r0 adr r0, Lcopystrpcbfaulttext bic sp, sp, #7 /* align stack to 8 bytes */ b _C_LABEL(panic) Lcopystrpcbfaulttext: .asciz "No valid PCB during copyinoutstr() addr1=%08x addr2=%08x\n" .align 2 #endif Index: head/sys/arm/arm/cpu_asm-v6.S =================================================================== --- head/sys/arm/arm/cpu_asm-v6.S (revision 331253) +++ head/sys/arm/arm/cpu_asm-v6.S (revision 331254) @@ -1,270 +1,270 @@ /*- * Copyright 2014 Svatopluk Kraus * Copyright 2014 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#include "assym.s" +#include "assym.inc" #include #include #include #include #define GET_PCB(tmp) \ mrc CP15_TPIDRPRW(tmp); \ add tmp, tmp, #(TD_PCB) /* * Define cache functions used by startup code, which counts on the fact that * only r0-r3,r12 (ip) are modified and no stack space is used. These functions * must be called with interrupts disabled. Moreover, these work only with * caches integrated to CPU (accessible via CP15); systems with an external L2 * cache controller such as a PL310 need separate calls to that device driver * to affect L2 caches. This is not a factor during early kernel startup, as * any external L2 cache controller has not been enabled yet. */ /* Invalidate D cache to PoC. (aka all cache levels)*/ ASENTRY_NP(dcache_inv_poc_all) #if __ARM_ARCH == 6 mcr CP15_DCIALL DSB bx lr #else mrc CP15_CLIDR(r0) ands r0, r0, #0x07000000 mov r0, r0, lsr #23 /* Get LoC 'naturally' aligned for */ beq 4f /* use in the CSSELR register below */ 1: sub r0, #2 mcr CP15_CSSELR(r0) /* set cache level */ isb mrc CP15_CCSIDR(r0) /* read CCSIDR */ ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */ ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */ clz r1, r3 /* number of bits to MSB of way */ lsl r3, r3, r1 /* shift into position */ mov ip, #1 lsl ip, ip, r1 /* ip now contains the way decr */ ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */ add r0, r0, #4 /* apply bias */ lsl r2, r2, r0 /* shift sets by log2(linesize) */ add r3, r3, r2 /* merge numsets - 1 with numways - 1 */ sub ip, ip, r2 /* subtract numsets - 1 from way decr */ mov r1, #1 lsl r1, r1, r0 /* r1 now contains the set decr */ mov r2, ip /* r2 now contains set way decr */ /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */ 2: mcr CP15_DCISW(r3) /* invalidate line */ movs r0, r3 /* get current way/set */ beq 3f /* at 0 means we are done */ movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/ subne r3, r3, r1 /* non-zero?, decrement set */ subeq r3, r3, r2 /* zero?, decrement way and restore set count */ b 2b 3: mrc CP15_CSSELR(r0) /* get cache level */ teq r0, #0 bne 1b 4: dsb /* wait for stores to finish */ mov r0, #0 mcr CP15_CSSELR(r0) isb bx lr #endif /* __ARM_ARCH == 6 */ END(dcache_inv_poc_all) /* Invalidate D cache to PoU. (aka L1 cache only)*/ ASENTRY_NP(dcache_inv_pou_all) #if __ARM_ARCH == 6 mcr CP15_DCIALL DSB bx lr #else mrc CP15_CLIDR(r0) ands r0, r0, #0x38000000 mov r0, r0, lsr #26 /* Get LoUU (naturally aligned) */ beq 4f 1: sub r0, #2 mcr CP15_CSSELR(r0) /* set cache level */ isb mrc CP15_CCSIDR(r0) /* read CCSIDR */ ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */ ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */ clz r1, r3 /* number of bits to MSB of way */ lsl r3, r3, r1 /* shift into position */ mov ip, #1 lsl ip, ip, r1 /* ip now contains the way decr */ ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */ add r0, r0, #4 /* apply bias */ lsl r2, r2, r0 /* shift sets by log2(linesize) */ add r3, r3, r2 /* merge numsets - 1 with numways - 1 */ sub ip, ip, r2 /* subtract numsets - 1 from way decr */ mov r1, #1 lsl r1, r1, r0 /* r1 now contains the set decr */ mov r2, ip /* r2 now contains set way decr */ /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */ 2: mcr CP15_DCISW(r3) /* invalidate line */ movs r0, r3 /* get current way/set */ beq 3f /* at 0 means we are done */ movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/ subne r3, r3, r1 /* non-zero?, decrement set */ subeq r3, r3, r2 /* zero?, decrement way and restore set count */ b 2b 3: mrc CP15_CSSELR(r0) /* get cache level */ teq r0, #0 bne 1b 4: dsb /* wait for stores to finish */ mov r0, #0 mcr CP15_CSSELR(r0) bx lr #endif END(dcache_inv_pou_all) /* Write back and Invalidate D cache to PoC. */ ASENTRY_NP(dcache_wbinv_poc_all) #if __ARM_ARCH == 6 mcr CP15_DCCIALL DSB bx lr #else mrc CP15_CLIDR(r0) ands r0, r0, #0x07000000 beq 4f mov r0, #0 /* Clean from inner to outer levels */ 1: mcr CP15_CSSELR(r0) /* set cache level */ isb mrc CP15_CCSIDR(r0) /* read CCSIDR */ ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */ ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */ clz r1, r3 /* number of bits to MSB of way */ lsl r3, r3, r1 /* shift into position */ mov ip, #1 lsl ip, ip, r1 /* ip now contains the way decr */ ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */ add r0, r0, #4 /* apply bias */ lsl r2, r2, r0 /* shift sets by log2(linesize) */ add r3, r3, r2 /* merge numsets - 1 with numways - 1 */ sub ip, ip, r2 /* subtract numsets - 1 from way decr */ mov r1, #1 lsl r1, r1, r0 /* r1 now contains the set decr */ mov r2, ip /* r2 now contains set way decr */ /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */ 2: mcr CP15_DCCISW(r3) /* clean and invalidate line */ movs r0, r3 /* get current way/set */ beq 3f /* at 0 means we are done */ movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/ subne r3, r3, r1 /* non-zero?, decrement set */ subeq r3, r3, r2 /* zero?, decrement way and restore set count */ b 2b 3: mrc CP15_CSSELR(r0) /* get cache level */ add r0, r0, #2 /* next level */ mrc CP15_CLIDR(r1) ands r1, r1, #0x07000000 mov r1, r1, lsr #23 /* Get LoC (naturally aligned) */ cmp r1, r0 bne 1b 4: dsb /* wait for stores to finish */ mov r0, #0 mcr CP15_CSSELR(r0) bx lr #endif /* __ARM_ARCH == 6 */ END(dcache_wbinv_poc_all) ASENTRY_NP(dcache_wb_pou_checked) ldr ip, .Lcpuinfo ldr ip, [ip, #DCACHE_LINE_SIZE] GET_PCB(r2) ldr r2, [r2] adr r3, _C_LABEL(cachebailout) str r3, [r2, #PCB_ONFAULT] 1: mcr CP15_DCCMVAC(r0) add r0, r0, ip subs r1, r1, ip bhi 1b DSB mov r0, #0 str r0, [r2, #PCB_ONFAULT] mov r0, #1 /* cannot be faulting address */ RET .Lcpuinfo: .word cpuinfo END(dcache_wb_pou_checked) ASENTRY_NP(icache_inv_pou_checked) ldr ip, .Lcpuinfo ldr ip, [ip, #ICACHE_LINE_SIZE] GET_PCB(r2) ldr r2, [r2] adr r3, _C_LABEL(cachebailout) str r3, [r2, #PCB_ONFAULT] 1: mcr CP15_ICIMVAU(r0) add r0, r0, ip subs r1, r1, ip bhi 1b DSB ISB mov r0, #0 str r0, [r2, #PCB_ONFAULT] mov r0, #1 /* cannot be faulting address */ RET END(icache_inv_pou_checked) /* label must be global as trap-v6.c references it */ .global _C_LABEL(cachebailout) _C_LABEL(cachebailout): DSB ISB mov r1, #0 str r1, [r2, #PCB_ONFAULT] RET Index: head/sys/arm/arm/exception.S =================================================================== --- head/sys/arm/arm/exception.S (revision 331253) +++ head/sys/arm/arm/exception.S (revision 331254) @@ -1,503 +1,503 @@ /* $NetBSD: exception.S,v 1.13 2003/10/31 16:30:15 scw Exp $ */ /*- * Copyright (c) 1994-1997 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * exception.S * * Low level handlers for exception vectors * * Created : 24/09/94 * * Based on kate/display/abort.s * */ -#include "assym.s" +#include "assym.inc" #include #include #include #include __FBSDID("$FreeBSD$"); #ifdef KDTRACE_HOOKS .bss .align 4 .global _C_LABEL(dtrace_invop_jump_addr) _C_LABEL(dtrace_invop_jump_addr): .word 0 .word 0 #endif .text .align 2 /* * ASM macros for pushing and pulling trapframes from the stack * * These macros are used to handle the irqframe and trapframe structures * defined above. */ /* * PUSHFRAME - macro to push a trap frame on the stack in the current mode * Since the current mode is used, the SVC lr field is not defined. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #if __ARM_ARCH < 6 #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; \ ldr r0, =ARM_RAS_START; \ mov r1, #0; \ str r1, [r0]; \ mov r1, #0xffffffff; \ str r1, [r0, #4]; #else #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; #endif /* * PULLFRAME - macro to pull a trap frame from the stack in the current mode * Since the current mode is used, the SVC lr field is ignored. */ #if __ARM_ARCH < 6 #define PULLFRAME \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #else #define PULLFRAME \ ldr r0, [sp], #4 ; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #endif /* * PUSHFRAMEINSVC - macro to push a trap frame on the stack in SVC32 mode * This should only be used if the processor is not currently in SVC32 * mode. The processor mode is switched to SVC mode and the trap frame is * stored. The SVC lr field is used to store the previous value of * lr in SVC mode. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #if __ARM_ARCH < 6 #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ /* and for dtrace to emulate push/pop */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ ldr r5, =ARM_RAS_START; /* Check if there's any RAS */ \ ldr r4, [r5, #4]; /* reset it to point at the */ \ cmp r4, #0xffffffff; /* end of memory if necessary; */ \ movne r1, #0xffffffff; /* leave value in r4 for later */ \ strne r1, [r5, #4]; /* comparison against PC. */ \ ldr r3, [r5]; /* Retrieve global RAS_START */ \ cmp r3, #0; /* and reset it if non-zero. */ \ movne r1, #0; /* If non-zero RAS_START and */ \ strne r1, [r5]; /* PC was lower than RAS_END, */ \ ldrne r1, [r0, #16]; /* adjust the saved PC so that */ \ cmpne r4, r1; /* execution later resumes at */ \ strhi r3, [r0, #16]; /* the RAS_START location. */ \ mrs r0, spsr; \ str r0, [sp, #-4]! #else #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ /* and for dtrace to emulate push/pop */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]! #endif /* * PULLFRAMEFROMSVCANDEXIT - macro to pull a trap frame from the stack * in SVC32 mode and restore the saved processor mode and PC. * This should be used when the SVC lr register needs to be restored on * exit. */ #if __ARM_ARCH < 6 #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #else #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #endif /* * Unwind hints so we can unwind past functions that use * PULLFRAMEFROMSVCANDEXIT. They are run in reverse order. * As the last thing we do is restore the stack pointer * we can ignore the padding at the end of struct trapframe. */ #define UNWINDSVCFRAME \ .save {r13-r15}; /* Restore sp, lr, pc */ \ .pad #(2*4); /* Skip user sp and lr */ \ .save {r0-r12}; /* Restore r0-r12 */ \ .pad #(4) /* Skip spsr */ #define DO_AST \ ldr r0, [sp]; /* Get the SPSR from stack */ \ mrs r4, cpsr; /* save CPSR */ \ orr r1, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r1; /* Disable interrupts */ \ and r0, r0, #(PSR_MODE); /* Returning to USR mode? */ \ teq r0, #(PSR_USR32_MODE); \ bne 2f; /* Nope, get out now */ \ bic r4, r4, #(PSR_I|PSR_F); \ 1: GET_CURTHREAD_PTR(r5); \ ldr r1, [r5, #(TD_FLAGS)]; \ and r1, r1, #(TDF_ASTPENDING|TDF_NEEDRESCHED); \ teq r1, #0; \ beq 2f; /* Nope. Just bail */ \ msr cpsr_c, r4; /* Restore interrupts */ \ mov r0, sp; \ bl _C_LABEL(ast); /* ast(frame) */ \ orr r0, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r0; \ b 1b; \ 2: /* * Entry point for a Software Interrupt (SWI). * * The hardware switches to svc32 mode on a swi, so we're already on the * right stack; just build a trapframe and call the handler. */ ASENTRY_NP(swi_entry) PUSHFRAME /* Build the trapframe on the */ mov r0, sp /* scv32 stack, pass it to the */ bl _C_LABEL(swi_handler) /* swi handler. */ /* * The fork_trampoline() code in swtch.S aranges for the MI fork_exit() * to return to swi_exit here, to return to userland. The net effect is * that a newly created thread appears to return from a SWI just like * the parent thread that created it. */ ASEENTRY_NP(swi_exit) DO_AST /* Handle pending signals. */ PULLFRAME /* Deallocate trapframe. */ movs pc, lr /* Return to userland. */ STOP_UNWINDING /* Don't unwind into user mode. */ EEND(swi_exit) END(swi_entry) /* * Standard exception exit handler. * * This is used to return from all exceptions except SWI. It uses DO_AST and * PULLFRAMEFROMSVCANDEXIT and can only be called if the exception entry code * used PUSHFRAMEINSVC. * * If the return is to user mode, this uses DO_AST to deliver any pending * signals and/or handle TDF_NEEDRESCHED first. */ ASENTRY_NP(exception_exit) DO_AST /* Handle pending signals. */ PULLFRAMEFROMSVCANDEXIT /* Return. */ UNWINDSVCFRAME /* Special unwinding for exceptions. */ END(exception_exit) /* * Entry point for a Prefetch Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(prefetch_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ mov r1, #1 /* Type flag */ b _C_LABEL(abort_handler) END(prefetch_abort_entry) /* * Entry point for a Data Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(data_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #8 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Exception exit routine */ mov r0, sp /* Trapframe to the handler */ mov r1, #0 /* Type flag */ b _C_LABEL(abort_handler) END(data_abort_entry) /* * Entry point for an Undefined Instruction exception. * * The hardware switches to the undefined mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the undefined mode stack). */ ASENTRY_NP(undefined_entry) PUSHFRAMEINSVC /* mode stack, build trapframe there. */ mov r4, r0 /* R0 contains SPSR */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. pass frame */ ldr r2, [sp, #(TF_PC)] /* load pc */ #if __ARM_ARCH >= 7 tst r4, #(PSR_T) /* test if PSR_T */ subne r2, r2, #(THUMB_INSN_SIZE) subeq r2, r2, #(INSN_SIZE) #else sub r2, r2, #(INSN_SIZE) /* fix pc */ #endif str r2, [sp, #TF_PC] /* store pc */ #ifdef KDTRACE_HOOKS /* Check if dtrace is enabled */ ldr r1, =_C_LABEL(dtrace_invop_jump_addr) ldr r3, [r1] cmp r3, #0 beq undefinedinstruction and r4, r4, #(PSR_MODE) /* Mask out unneeded bits */ cmp r4, #(PSR_USR32_MODE) /* Check if we came from usermode */ beq undefinedinstruction ldr r4, [r2] /* load instrution */ ldr r1, =FBT_BREAKPOINT /* load fbt inv op */ cmp r1, r4 bne undefinedinstruction bx r3 /* call invop_jump_addr */ #endif b undefinedinstruction /* call stadnard handler */ END(undefined_entry) /* * Entry point for a normal IRQ. * * The hardware switches to the IRQ mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the IRQ mode stack). */ ASENTRY_NP(irq_entry) sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ b _C_LABEL(intr_irq_handler)/* trapframe to the handler. */ END(irq_entry) /* * Entry point for an FIQ interrupt. * * We don't currently support FIQ handlers very much. Something can * install itself in the FIQ vector using code (that may or may not work * these days) in fiq.c. If nobody does that and an FIQ happens, this * default handler just disables FIQs and otherwise ignores it. */ ASENTRY_NP(fiq_entry) mrs r8, cpsr /* FIQ handling isn't supported, */ bic r8, #(PSR_F) /* just disable FIQ and return. */ msr cpsr_c, r8 /* The r8 we trash here is the */ subs pc, lr, #4 /* banked FIQ-mode r8. */ END(fiq_entry) /* * Entry point for an Address Exception exception. * This is an arm26 exception that should never happen. */ ASENTRY_NP(addr_exception_entry) mov r3, lr mrs r2, spsr mrs r1, cpsr adr r0, Laddr_exception_msg b _C_LABEL(panic) Laddr_exception_msg: .asciz "Address Exception CPSR=0x%08x SPSR=0x%08x LR=0x%08x\n" .balign 4 END(addr_exception_entry) /* * Entry point for the system Reset vector. * This should never happen, so panic. */ ASENTRY_NP(reset_entry) mov r1, lr adr r0, Lreset_panicmsg b _C_LABEL(panic) /* NOTREACHED */ Lreset_panicmsg: .asciz "Reset vector called, LR = 0x%08x" .balign 4 END(reset_entry) /* * page0 and page0_data -- An image of the ARM vectors which is copied to * the ARM vectors page (high or low) as part of CPU initialization. The * code that does the copy assumes that page0_data holds one 32-bit word * of data for each of the predefined ARM vectors. It also assumes that * page0_data follows the vectors in page0, but other stuff can appear * between the two. We currently leave room between the two for some fiq * handler code to be copied in. */ .global _C_LABEL(page0), _C_LABEL(page0_data) _C_LABEL(page0): ldr pc, .Lreset_entry ldr pc, .Lundefined_entry ldr pc, .Lswi_entry ldr pc, .Lprefetch_abort_entry ldr pc, .Ldata_abort_entry ldr pc, .Laddr_exception_entry ldr pc, .Lirq_entry .fiqv: ldr pc, .Lfiq_entry .space 256 /* room for some fiq handler code */ _C_LABEL(page0_data): .Lreset_entry: .word reset_entry .Lundefined_entry: .word undefined_entry .Lswi_entry: .word swi_entry .Lprefetch_abort_entry: .word prefetch_abort_entry .Ldata_abort_entry: .word data_abort_entry .Laddr_exception_entry: .word addr_exception_entry .Lirq_entry: .word irq_entry .Lfiq_entry: .word fiq_entry /* * These items are used by the code in fiq.c to install what it calls the * "null" handler. It's actually our default vector entry that just jumps * to the default handler which just disables FIQs and returns. */ .global _C_LABEL(fiq_nullhandler_code), _C_LABEL(fiq_nullhandler_size) _C_LABEL(fiq_nullhandler_code): .word .fiqv _C_LABEL(fiq_nullhandler_size): .word 4 Index: head/sys/arm/arm/fusu.S =================================================================== --- head/sys/arm/arm/fusu.S (revision 331253) +++ head/sys/arm/arm/fusu.S (revision 331254) @@ -1,349 +1,349 @@ /* $NetBSD: fusu.S,v 1.10 2003/12/01 13:34:44 rearnsha Exp $ */ /*- * Copyright (c) 1996-1998 Mark Brinicombe. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include -#include "assym.s" +#include "assym.inc" __FBSDID("$FreeBSD$"); .syntax unified #if __ARM_ARCH >= 6 #define GET_PCB(tmp) \ mrc p15, 0, tmp, c13, c0, 4; \ add tmp, tmp, #(TD_PCB) #else .Lcurpcb: .word _C_LABEL(__pcpu) + PC_CURPCB #define GET_PCB(tmp) \ ldr tmp, .Lcurpcb #endif /* * casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, * uint32_t newval); */ ENTRY(casueword) EENTRY_NP(casueword32) stmfd sp!, {r4, r5, r6} ldr r4, =(VM_MAXUSER_ADDRESS-3) cmp r0, r4 mvncs r0, #0 bcs 2f GET_PCB(r6) ldr r6, [r6] #ifdef DIAGNOSTIC teq r6, #0x00000000 ldmfdeq sp!, {r4, r5, r6} beq .Lfusupcbfault #endif adr r4, .Lcasuwordfault str r4, [r6, #PCB_ONFAULT] #if __ARM_ARCH >= 6 1: ldrex r4, [r0] cmp r4, r1 strexeq r5, r3, [r0] cmpeq r5, #1 beq 1b #else ldrt r4, [r0] cmp r4, r1 strteq r3, [r0] #endif str r4, [r2] mov r0, #0 str r0, [r6, #PCB_ONFAULT] 2: ldmfd sp!, {r4, r5, r6} RET EEND(casueword32) END(casueword) /* * Handle faults from casuword. Clean up and return -1. */ .Lcasuwordfault: mov r0, #0x00000000 str r0, [r6, #PCB_ONFAULT] mvn r0, #0 ldmfd sp!, {r4, r5, r6} RET /* * fueword(caddr_t uaddr, long *val); * Fetch an int from the user's address space. */ ENTRY(fueword) EENTRY_NP(fueword32) ldr r3, =(VM_MAXUSER_ADDRESS-3) cmp r0, r3 mvncs r0, #0 RETc(cs) GET_PCB(r2) ldr r2, [r2] #ifdef DIAGNOSTIC teq r2, #0x00000000 beq .Lfusupcbfault #endif adr r3, .Lfusufault str r3, [r2, #PCB_ONFAULT] ldrt r3, [r0] str r3, [r1] mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET EEND(fueword32) END(fueword) /* * fusword(caddr_t uaddr); * Fetch a short from the user's address space. */ ENTRY(fusword) ldr r3, =(VM_MAXUSER_ADDRESS-1) cmp r0, r3 mvncs r0, #0 RETc(cs) GET_PCB(r2) ldr r2, [r2] #ifdef DIAGNOSTIC teq r2, #0x00000000 beq .Lfusupcbfault #endif adr r1, .Lfusufault str r1, [r2, #PCB_ONFAULT] ldrbt r3, [r0], #1 ldrbt ip, [r0] #ifdef __ARMEB__ orr r0, ip, r3, asl #8 #else orr r0, r3, ip, asl #8 #endif mov r1, #0x00000000 str r1, [r2, #PCB_ONFAULT] RET END(fusword) /* * fuswintr(caddr_t uaddr); * Fetch a short from the user's address space. Can be called during an * interrupt. */ ENTRY(fuswintr) mov r0, #-1 RET END(fuswintr) /* * fubyte(caddr_t uaddr); * Fetch a byte from the user's address space. */ ENTRY(fubyte) ldr r3, =VM_MAXUSER_ADDRESS cmp r0, r3 mvncs r0, #0 RETc(cs) GET_PCB(r2) ldr r2, [r2] #ifdef DIAGNOSTIC teq r2, #0x00000000 beq .Lfusupcbfault #endif adr r1, .Lfusufault str r1, [r2, #PCB_ONFAULT] ldrbt r3, [r0] mov r1, #0x00000000 str r1, [r2, #PCB_ONFAULT] mov r0, r3 RET END(fubyte) /* * Handle faults from [fs]u*(). Clean up and return -1. */ .Lfusufault: mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] mvn r0, #0x00000000 RET #ifdef DIAGNOSTIC /* * Handle earlier faults from [fs]u*(), due to no pcb */ .Lfusupcbfault: mov r1, r0 adr r0, fusupcbfaulttext b _C_LABEL(panic) fusupcbfaulttext: .asciz "Yikes - no valid PCB during fusuxxx() addr=%08x\n" .align 2 #endif /* * suword(caddr_t uaddr, int x); * Store an int in the user's address space. */ ENTRY(suword) EENTRY_NP(suword32) ldr r3, =(VM_MAXUSER_ADDRESS-3) cmp r0, r3 mvncs r0, #0 RETc(cs) GET_PCB(r2) ldr r2, [r2] #ifdef DIAGNOSTIC teq r2, #0x00000000 beq .Lfusupcbfault #endif adr r3, .Lfusufault str r3, [r2, #PCB_ONFAULT] strt r1, [r0] mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET EEND(suword32) END(suword) /* * suswintr(caddr_t uaddr, short x); * Store a short in the user's address space. Can be called during an * interrupt. */ ENTRY(suswintr) mov r0, #-1 RET END(suswintr) /* * susword(caddr_t uaddr, short x); * Store a short in the user's address space. */ ENTRY(susword) ldr r3, =(VM_MAXUSER_ADDRESS-1) cmp r0, r3 mvncs r0, #0 RETc(cs) GET_PCB(r2) ldr r2, [r2] #ifdef DIAGNOSTIC teq r2, #0x00000000 beq .Lfusupcbfault #endif adr r3, .Lfusufault str r3, [r2, #PCB_ONFAULT] #ifdef __ARMEB__ mov ip, r1, lsr #8 strbt ip, [r0], #1 #else strbt r1, [r0], #1 mov r1, r1, lsr #8 #endif strbt r1, [r0] mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET END(susword) /* * subyte(caddr_t uaddr, char x); * Store a byte in the user's address space. */ ENTRY(subyte) ldr r3, =VM_MAXUSER_ADDRESS cmp r0, r3 mvncs r0, #0 RETc(cs) GET_PCB(r2) ldr r2, [r2] #ifdef DIAGNOSTIC teq r2, #0x00000000 beq .Lfusupcbfault #endif adr r3, .Lfusufault str r3, [r2, #PCB_ONFAULT] strbt r1, [r0] mov r0, #0x00000000 str r0, [r2, #PCB_ONFAULT] RET END(subyte) Index: head/sys/arm/arm/hypervisor-stub.S =================================================================== --- head/sys/arm/arm/hypervisor-stub.S (revision 331253) +++ head/sys/arm/arm/hypervisor-stub.S (revision 331254) @@ -1,87 +1,87 @@ /* * Copyright (C) 2015 Mihai Carabas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include "assym.s" +#include "assym.inc" #include #include #include #include __FBSDID("$FreeBSD$"); #if __ARM_ARCH >= 7 #if defined(__ARM_ARCH_7VE__) || defined(__clang__) .arch_extension virt #endif ASENTRY_NP(hypervisor_stub_vect_install) /* Install hypervisor stub vectors. */ adr r0, hypervisor_stub_vect mcr CP15_HVBAR(r0) /* Disable all the traps in the hypervisor. */ mov r0, #0 mcr CP15_HCR(r0) mcr CP15_HCPTR(r0) mcr CP15_HSTR(r0) mcr CP15_HSCTLR(r0) /* Don't disable access to perf-mon from PL0,1 and preserve HPMN. */ mrc CP15_HDCR(r0) and r0, #(ARM_CP15_HDCR_HPMN) /* Caller implicit instruction barrier in the ERET. */ mcr CP15_HDCR(r0) RET END(hypervisor_stub_vect_install) ASENTRY_NP(hypervisor_stub_trap) /* * If the first parameter is -1 than return the * exception vector (HVBAR), otherwise set it to * the value of it. */ cmp r0, #-1 mrceq CP15_HVBAR(r0) mcrne CP15_HVBAR(r0) ERET END(hypervisor_stub_trap) .globl hypervisor_stub_vect .align 5 _C_LABEL(hypervisor_stub_vect): .word 0 /* Reset */ .word 0 /* undev */ .word 0 /* SMC */ .word 0 /* PABT */ .word 0 /* DABT */ b hypervisor_stub_trap /* HYP-Mode */ .word 0 /* FIQ */ .word 0 /* IRQ */ #endif /* __ARM_ARCH >= 7 */ Index: head/sys/arm/arm/in_cksum_arm.S =================================================================== --- head/sys/arm/arm/in_cksum_arm.S (revision 331253) +++ head/sys/arm/arm/in_cksum_arm.S (revision 331254) @@ -1,344 +1,344 @@ /* $NetBSD: in_cksum_arm.S,v 1.2 2003/09/23 10:01:36 scw Exp $ */ /*- * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * */ /* * Hand-optimised in_cksum() and in4_cksum() implementations for ARM/armv5e */ #include "opt_inet.h" #include -#include "assym.s" +#include "assym.inc" __FBSDID("$FreeBSD$"); .syntax unified /* * int in_cksum(struct mbuf *m, int len) * * Entry: * r0 m * r1 len * * NOTE: Assumes 'm' is *never* NULL. */ /* LINTSTUB: Func: int in_cksum(struct mbuf *, int) */ ENTRY(in_cksum) stmfd sp!, {r4-r11,lr} mov r8, #0x00 mov r9, r1 mov r10, #0x00 mov ip, r0 .Lin_cksum_loop: ldr r1, [ip, #(M_LEN)] ldr r0, [ip, #(M_DATA)] ldr ip, [ip, #(M_NEXT)] .Lin_cksum_entry4: cmp r9, r1 movlt r1, r9 sub r9, r9, r1 eor r11, r10, r0 add r10, r10, r1 adds r2, r1, #0x00 blne _ASM_LABEL(L_cksumdata) tst r11, #0x01 movne r2, r2, ror #8 adds r8, r8, r2 adc r8, r8, #0x00 cmp ip, #0x00 bne .Lin_cksum_loop mov r1, #0xff orr r1, r1, #0xff00 and r0, r8, r1 add r0, r0, r8, lsr #16 add r0, r0, r0, lsr #16 and r0, r0, r1 eor r0, r0, r1 ldmfd sp!, {r4-r11,pc} END(in_cksum) ENTRY(do_cksum) stmfd sp!, {r4-r7, lr} bl L_cksumdata mov r0, r2 ldmfd sp!, {r4-r7, pc} END(do_cksum) /* * The main in*_cksum() workhorse... * * Entry parameters: * r0 Pointer to buffer * r1 Buffer length * lr Return address * * Returns: * r2 Accumulated 32-bit sum * * Clobbers: * r0-r7 */ /* LINTSTUB: Ignore */ ASENTRY_NP(L_cksumdata) #ifdef _ARM_ARCH_5E pld [r0] /* Pre-fetch the start of the buffer */ #endif mov r2, #0 /* We first have to word-align the buffer. */ ands r7, r0, #0x03 beq .Lcksumdata_wordaligned rsb r7, r7, #0x04 cmp r1, r7 /* Enough bytes left to make it? */ blt .Lcksumdata_endgame cmp r7, #0x02 ldrb r4, [r0], #0x01 /* Fetch 1st byte */ ldrbge r5, [r0], #0x01 /* Fetch 2nd byte */ movlt r5, #0x00 ldrbgt r6, [r0], #0x01 /* Fetch 3rd byte */ movle r6, #0x00 /* Combine the three bytes depending on endianness and alignment */ #ifdef __ARMEB__ orreq r2, r5, r4, lsl #8 orreq r2, r2, r6, lsl #24 orrne r2, r4, r5, lsl #8 orrne r2, r2, r6, lsl #16 #else orreq r2, r4, r5, lsl #8 orreq r2, r2, r6, lsl #16 orrne r2, r5, r4, lsl #8 orrne r2, r2, r6, lsl #24 #endif subs r1, r1, r7 /* Update length */ RETeq /* All done? */ /* Buffer is now word aligned */ .Lcksumdata_wordaligned: #ifdef _ARM_ARCH_5E cmp r1, #0x04 /* Less than 4 bytes left? */ blt .Lcksumdata_endgame /* Yup */ /* Now quad-align, if necessary */ ands r7, r0, #0x04 ldrne r7, [r0], #0x04 subne r1, r1, #0x04 subs r1, r1, #0x40 blt .Lcksumdata_bigloop_end /* Note: C flag clear if branch taken */ /* * Buffer is now quad aligned. Sum 64 bytes at a time. * Note: First ldrd is hoisted above the loop, together with * setting r6 to zero to avoid stalling for results in the * loop. (r7 is live, from above). */ ldrd r4, [r0], #0x08 mov r6, #0x00 .Lcksumdata_bigloop: pld [r0, #0x18] adds r2, r2, r6 adcs r2, r2, r7 ldrd r6, [r0], #0x08 adcs r2, r2, r4 adcs r2, r2, r5 ldrd r4, [r0], #0x08 adcs r2, r2, r6 adcs r2, r2, r7 ldrd r6, [r0], #0x08 adcs r2, r2, r4 adcs r2, r2, r5 ldrd r4, [r0], #0x08 adcs r2, r2, r6 adcs r2, r2, r7 pld [r0, #0x18] ldrd r6, [r0], #0x08 adcs r2, r2, r4 adcs r2, r2, r5 ldrd r4, [r0], #0x08 adcs r2, r2, r6 adcs r2, r2, r7 ldrd r6, [r0], #0x08 adcs r2, r2, r4 adcs r2, r2, r5 adc r2, r2, #0x00 subs r1, r1, #0x40 ldrdge r4, [r0], #0x08 bge .Lcksumdata_bigloop adds r2, r2, r6 /* r6/r7 still need summing */ .Lcksumdata_bigloop_end: adcs r2, r2, r7 adc r2, r2, #0x00 #else /* !_ARM_ARCH_5E */ subs r1, r1, #0x40 blt .Lcksumdata_bigloop_end .Lcksumdata_bigloop: ldmia r0!, {r3, r4, r5, r6} adds r2, r2, r3 adcs r2, r2, r4 adcs r2, r2, r5 ldmia r0!, {r3, r4, r5, r7} adcs r2, r2, r6 adcs r2, r2, r3 adcs r2, r2, r4 adcs r2, r2, r5 ldmia r0!, {r3, r4, r5, r6} adcs r2, r2, r7 adcs r2, r2, r3 adcs r2, r2, r4 adcs r2, r2, r5 ldmia r0!, {r3, r4, r5, r7} adcs r2, r2, r6 adcs r2, r2, r3 adcs r2, r2, r4 adcs r2, r2, r5 adcs r2, r2, r7 adc r2, r2, #0x00 subs r1, r1, #0x40 bge .Lcksumdata_bigloop .Lcksumdata_bigloop_end: #endif adds r1, r1, #0x40 RETeq cmp r1, #0x20 #ifdef _ARM_ARCH_5E ldrdge r4, [r0], #0x08 /* Avoid stalling pld and result */ blt .Lcksumdata_less_than_32 pld [r0, #0x18] ldrd r6, [r0], #0x08 adds r2, r2, r4 adcs r2, r2, r5 ldrd r4, [r0], #0x08 adcs r2, r2, r6 adcs r2, r2, r7 ldrd r6, [r0], #0x08 adcs r2, r2, r4 adcs r2, r2, r5 adcs r2, r2, r6 /* XXX: Unavoidable result stall */ adcs r2, r2, r7 #else blt .Lcksumdata_less_than_32 ldmia r0!, {r3, r4, r5, r6} adds r2, r2, r3 adcs r2, r2, r4 adcs r2, r2, r5 ldmia r0!, {r3, r4, r5, r7} adcs r2, r2, r6 adcs r2, r2, r3 adcs r2, r2, r4 adcs r2, r2, r5 adcs r2, r2, r7 #endif adc r2, r2, #0x00 subs r1, r1, #0x20 RETeq .Lcksumdata_less_than_32: /* There are less than 32 bytes left */ and r3, r1, #0x18 rsb r4, r3, #0x18 sub r1, r1, r3 adds r4, r4, r4, lsr #1 /* Side effect: Clear carry flag */ addne pc, pc, r4 nop /* * Note: We use ldm here, even on armv5e, since the combined issue/result * latencies for ldm and ldrd are the same. Using ldm avoids needless #ifdefs. */ /* At least 24 bytes remaining... */ ldmia r0!, {r4, r5} adcs r2, r2, r4 adcs r2, r2, r5 /* At least 16 bytes remaining... */ ldmia r0!, {r4, r5} adcs r2, r2, r4 adcs r2, r2, r5 /* At least 8 bytes remaining... */ ldmia r0!, {r4, r5} adcs r2, r2, r4 adcs r2, r2, r5 /* Less than 8 bytes remaining... */ adc r2, r2, #0x00 subs r1, r1, #0x04 blt .Lcksumdata_lessthan4 ldr r4, [r0], #0x04 sub r1, r1, #0x04 adds r2, r2, r4 adc r2, r2, #0x00 /* Deal with < 4 bytes remaining */ .Lcksumdata_lessthan4: adds r1, r1, #0x04 RETeq /* Deal with 1 to 3 remaining bytes, possibly misaligned */ .Lcksumdata_endgame: ldrb r3, [r0] /* Fetch first byte */ cmp r1, #0x02 ldrbge r4, [r0, #0x01] /* Fetch 2nd and 3rd as necessary */ movlt r4, #0x00 ldrbgt r5, [r0, #0x02] movle r5, #0x00 /* Combine the three bytes depending on endianness and alignment */ tst r0, #0x01 #ifdef __ARMEB__ orreq r3, r4, r3, lsl #8 orreq r3, r3, r5, lsl #24 orrne r3, r3, r4, lsl #8 orrne r3, r3, r5, lsl #16 #else orreq r3, r3, r4, lsl #8 orreq r3, r3, r5, lsl #16 orrne r3, r4, r3, lsl #8 orrne r3, r3, r5, lsl #24 #endif adds r2, r2, r3 adc r2, r2, #0x00 RET END(L_cksumdata) Index: head/sys/arm/arm/locore-v4.S =================================================================== --- head/sys/arm/arm/locore-v4.S (revision 331253) +++ head/sys/arm/arm/locore-v4.S (revision 331254) @@ -1,492 +1,492 @@ /* $NetBSD: locore.S,v 1.14 2003/04/20 16:21:40 thorpej Exp $ */ /*- * Copyright 2011 Semihalf * Copyright (C) 1994-1997 Mark Brinicombe * Copyright (C) 1994 Brini * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of Brini may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ -#include "assym.s" +#include "assym.inc" #include #include #include #include __FBSDID("$FreeBSD$"); /* 2K initial stack is plenty, it is only used by initarm() */ #define INIT_ARM_STACK_SIZE 2048 #define CPWAIT_BRANCH \ sub pc, pc, #4 #define CPWAIT(tmp) \ mrc p15, 0, tmp, c2, c0, 0 /* arbitrary read of CP15 */ ;\ mov tmp, tmp /* wait for it to complete */ ;\ CPWAIT_BRANCH /* branch to next insn */ /* * This is for libkvm, and should be the address of the beginning * of the kernel text segment (not necessarily the same as kernbase). * * These are being phased out. Newer copies of libkvm don't need these * values as the information is added to the core file by inspecting * the running kernel. */ .text .align 2 #ifdef PHYSADDR .globl kernbase .set kernbase,KERNBASE .globl physaddr .set physaddr,PHYSADDR #endif /* * On entry for FreeBSD boot ABI: * r0 - metadata pointer or 0 (boothowto on AT91's boot2) * r1 - if (r0 == 0) then metadata pointer * On entry for Linux boot ABI: * r0 - 0 * r1 - machine type (passed as arg2 to initarm) * r2 - Pointer to a tagged list or dtb image (phys addr) (passed as arg1 initarm) * * For both types of boot we gather up the args, put them in a struct arm_boot_params * structure and pass that to initarm. */ .globl btext btext: ASENTRY_NP(_start) STOP_UNWINDING /* Can't unwind into the bootloader! */ mov r9, r0 /* 0 or boot mode from boot2 */ mov r8, r1 /* Save Machine type */ mov ip, r2 /* Save meta data */ mov fp, r3 /* Future expansion */ /* Make sure interrupts are disabled. */ mrs r7, cpsr orr r7, r7, #(PSR_I | PSR_F) msr cpsr_c, r7 #if defined (FLASHADDR) && defined(LOADERRAMADDR) /* * Sanity check the configuration. * FLASHADDR and LOADERRAMADDR depend on PHYSADDR in some cases. * ARMv4 and ARMv5 make assumptions on where they are loaded. * TODO: Fix the ARMv4/v5 case. */ #ifndef PHYSADDR #error PHYSADDR must be defined for this configuration #endif /* Check if we're running from flash. */ ldr r7, =FLASHADDR /* * If we're running with MMU disabled, test against the * physical address instead. */ mrc CP15_SCTLR(r2) ands r2, r2, #CPU_CONTROL_MMU_ENABLE ldreq r6, =PHYSADDR ldrne r6, =LOADERRAMADDR cmp r7, r6 bls flash_lower cmp r7, pc bhi from_ram b do_copy flash_lower: cmp r6, pc bls from_ram do_copy: ldr r7, =KERNBASE adr r1, _start ldr r0, Lreal_start ldr r2, Lend sub r2, r2, r0 sub r0, r0, r7 add r0, r0, r6 mov r4, r0 bl memcpy ldr r0, Lram_offset add pc, r4, r0 Lram_offset: .word from_ram-_C_LABEL(_start) from_ram: nop #endif disable_mmu: /* Disable MMU for a while */ mrc CP15_SCTLR(r2) bic r2, r2, #(CPU_CONTROL_MMU_ENABLE | CPU_CONTROL_DC_ENABLE |\ CPU_CONTROL_WBUF_ENABLE) bic r2, r2, #(CPU_CONTROL_IC_ENABLE) bic r2, r2, #(CPU_CONTROL_BPRD_ENABLE) mcr CP15_SCTLR(r2) nop nop nop CPWAIT(r0) Lunmapped: /* * Build page table from scratch. */ /* * Figure out the physical address we're loaded at by assuming this * entry point code is in the first L1 section and so if we clear the * offset bits of the pc that will give us the section-aligned load * address, which remains in r5 throughout all the following code. */ ldr r2, =(L1_S_OFFSET) bic r5, pc, r2 /* Find the delta between VA and PA, result stays in r0 throughout. */ adr r0, Lpagetable bl translate_va_to_pa /* * First map the entire 4GB address space as VA=PA. It's mapped as * normal (cached) memory because it's for things like accessing the * parameters passed in from the bootloader, which might be at any * physical address, different for every platform. */ mov r1, #0 mov r2, #0 mov r3, #4096 bl build_pagetables /* * Next we do 64MiB starting at the physical load address, mapped to * the VA the kernel is linked for. */ mov r1, r5 ldr r2, =(KERNVIRTADDR) mov r3, #64 bl build_pagetables #if defined(PHYSADDR) && (KERNVIRTADDR != KERNBASE) /* * If the kernel wasn't loaded at the beginning of the ram, map the memory * before the kernel too, as some ports use that for pagetables, stack, etc... */ ldr r1, =PHYSADDR ldr r2, =KERNBASE ldr r3, =((KERNVIRTADDR - KERNBASE) / L1_S_SIZE) bl build_pagetables #endif /* Create a device mapping for early_printf if specified. */ #if defined(SOCDEV_PA) && defined(SOCDEV_VA) ldr r1, =SOCDEV_PA ldr r2, =SOCDEV_VA mov r3, #1 bl build_device_pagetables #endif mcr p15, 0, r0, c2, c0, 0 /* Set TTB */ mcr p15, 0, r0, c8, c7, 0 /* Flush TLB */ /* Set the Domain Access register. Very important! */ mov r0, #((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT) mcr p15, 0, r0, c3, c0, 0 /* * Enable MMU. */ mrc CP15_SCTLR(r0) orr r0, r0, #(CPU_CONTROL_MMU_ENABLE) mcr CP15_SCTLR(r0) nop nop nop CPWAIT(r0) /* Transition the PC from physical to virtual addressing. */ ldr pc,=mmu_done mmu_done: nop adr r1, .Lstart ldmia r1, {r1, r2, sp} /* Set initial stack and */ sub r2, r2, r1 /* get zero init data */ mov r3, #0 .L1: str r3, [r1], #0x0004 /* get zero init data */ subs r2, r2, #4 bgt .L1 virt_done: mov r1, #28 /* loader info size is 28 bytes also second arg */ subs sp, sp, r1 /* allocate arm_boot_params struct on stack */ mov r0, sp /* loader info pointer is first arg */ bic sp, sp, #7 /* align stack to 8 bytes */ str r1, [r0] /* Store length of loader info */ str r9, [r0, #4] /* Store r0 from boot loader */ str r8, [r0, #8] /* Store r1 from boot loader */ str ip, [r0, #12] /* store r2 from boot loader */ str fp, [r0, #16] /* store r3 from boot loader */ str r5, [r0, #20] /* store the physical address */ adr r4, Lpagetable /* load the pagetable address */ ldr r5, [r4, #4] str r5, [r0, #24] /* store the pagetable address */ mov fp, #0 /* trace back starts here */ bl _C_LABEL(initarm) /* Off we go */ /* init arm will return the new stack pointer. */ mov sp, r0 bl _C_LABEL(mi_startup) /* call mi_startup()! */ adr r0, .Lmainreturned b _C_LABEL(panic) /* NOTREACHED */ END(_start) #define VA_TO_PA_POINTER(name, table) \ name: ;\ .word . ;\ .word table /* * Returns the physical address of a magic va to pa pointer. * r0 - The pagetable data pointer. This must be built using the * VA_TO_PA_POINTER macro. * e.g. * VA_TO_PA_POINTER(Lpagetable, pagetable) * ... * adr r0, Lpagetable * bl translate_va_to_pa * r0 will now contain the physical address of pagetable * r1, r2 - Trashed */ translate_va_to_pa: ldr r1, [r0] sub r2, r1, r0 /* At this point: r2 = VA - PA */ /* * Find the physical address of the table. After these two * instructions: * r1 = va(pagetable) * * r0 = va(pagetable) - (VA - PA) * = va(pagetable) - VA + PA * = pa(pagetable) */ ldr r1, [r0, #4] sub r0, r1, r2 RET /* * Builds the page table * r0 - The table base address * r1 - The physical address (trashed) * r2 - The virtual address (trashed) * r3 - The number of 1MiB sections * r4 - Trashed * * Addresses must be 1MiB aligned */ build_device_pagetables: ldr r4, =(L1_TYPE_S|L1_S_AP(AP_KRW)) b 1f build_pagetables: /* Set the required page attributed */ ldr r4, =(L1_TYPE_S|L1_S_C|L1_S_AP(AP_KRW)) 1: orr r1, r4 /* Move the virtual address to the correct bit location */ lsr r2, #(L1_S_SHIFT - 2) mov r4, r3 2: str r1, [r0, r2] add r2, r2, #4 add r1, r1, #(L1_S_SIZE) adds r4, r4, #-1 bhi 2b RET VA_TO_PA_POINTER(Lpagetable, pagetable) Lreal_start: .word _start Lend: .word _edata .Lstart: .word _edata .word _ebss .word svcstk + INIT_ARM_STACK_SIZE .Lvirt_done: .word virt_done .Lmainreturned: .asciz "main() returned" .align 2 .bss svcstk: .space INIT_ARM_STACK_SIZE /* * Memory for the initial pagetable. We are unable to place this in * the bss as this will be cleared after the table is loaded. */ .section ".init_pagetable", "aw", %nobits .align 14 /* 16KiB aligned */ pagetable: .space L1_TABLE_SIZE .text .align 2 .Lcpufuncs: .word _C_LABEL(cpufuncs) ENTRY_NP(cpu_halt) mrs r2, cpsr bic r2, r2, #(PSR_MODE) orr r2, r2, #(PSR_SVC32_MODE) orr r2, r2, #(PSR_I | PSR_F) msr cpsr_fsxc, r2 ldr r4, .Lcpu_reset_address ldr r4, [r4] ldr r0, .Lcpufuncs mov lr, pc ldr pc, [r0, #CF_IDCACHE_WBINV_ALL] mov lr, pc ldr pc, [r0, #CF_L2CACHE_WBINV_ALL] /* * Load the cpu_reset_needs_v4_MMU_disable flag to determine if it's * necessary. */ ldr r1, .Lcpu_reset_needs_v4_MMU_disable ldr r1, [r1] cmp r1, #0 mov r2, #0 /* * MMU & IDC off, 32 bit program & data space * Hurl ourselves into the ROM */ mov r0, #(CPU_CONTROL_32BP_ENABLE | CPU_CONTROL_32BD_ENABLE) mcr CP15_SCTLR(r0) mcrne p15, 0, r2, c8, c7, 0 /* nail I+D TLB on ARMv4 and greater */ mov pc, r4 /* * _cpu_reset_address contains the address to branch to, to complete * the cpu reset after turning the MMU off * This variable is provided by the hardware specific code */ .Lcpu_reset_address: .word _C_LABEL(cpu_reset_address) /* * cpu_reset_needs_v4_MMU_disable contains a flag that signals if the * v4 MMU disable instruction needs executing... it is an illegal instruction * on f.e. ARM6/7 that locks up the computer in an endless illegal * instruction / data-abort / reset loop. */ .Lcpu_reset_needs_v4_MMU_disable: .word _C_LABEL(cpu_reset_needs_v4_MMU_disable) END(cpu_halt) /* * setjump + longjmp */ ENTRY(setjmp) stmia r0, {r4-r14} mov r0, #0x00000000 RET END(setjmp) ENTRY(longjmp) ldmia r0, {r4-r14} mov r0, #0x00000001 RET END(longjmp) .data .global _C_LABEL(esym) _C_LABEL(esym): .word _C_LABEL(end) ENTRY_NP(abort) b _C_LABEL(abort) END(abort) ENTRY_NP(sigcode) mov r0, sp add r0, r0, #SIGF_UC /* * Call the sigreturn system call. * * We have to load r7 manually rather than using * "ldr r7, =SYS_sigreturn" to ensure the value of szsigcode is * correct. Using the alternative places esigcode at the address * of the data rather than the address one past the data. */ ldr r7, [pc, #12] /* Load SYS_sigreturn */ swi SYS_sigreturn /* Well if that failed we better exit quick ! */ ldr r7, [pc, #8] /* Load SYS_exit */ swi SYS_exit /* Branch back to retry SYS_sigreturn */ b . - 16 END(sigcode) .word SYS_sigreturn .word SYS_exit .align 2 .global _C_LABEL(esigcode) _C_LABEL(esigcode): .data .global szsigcode szsigcode: .long esigcode-sigcode /* End of locore.S */ Index: head/sys/arm/arm/locore-v6.S =================================================================== --- head/sys/arm/arm/locore-v6.S (revision 331253) +++ head/sys/arm/arm/locore-v6.S (revision 331254) @@ -1,595 +1,595 @@ /*- * Copyright 2004-2014 Olivier Houchard * Copyright 2012-2014 Ian Lepore * Copyright 2013-2014 Andrew Turner * Copyright 2014 Svatopluk Kraus * Copyright 2014 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include "assym.s" +#include "assym.inc" #include #include #include #include #include #include __FBSDID("$FreeBSD$"); -/* We map 64MB of kernel unless overridden in assym.s by the kernel option. */ +/* We map 64MB of kernel unless overridden in assym.inc by the kernel option. */ #ifndef LOCORE_MAP_MB #define LOCORE_MAP_MB 64 #endif #if __ARM_ARCH >= 7 #if defined(__ARM_ARCH_7VE__) || defined(__clang__) /* * HYP support is in bintuils >= 2.21 and gcc >= 4.9 defines __ARM_ARCH_7VE__ * when enabled. llvm >= 3.6 supports it too. */ .arch_extension virt #endif #endif /* __ARM_ARCH >= 7 */ /* A small statically-allocated stack used only during initarm() and AP startup. */ #define INIT_ARM_STACK_SIZE 2048 .text .align 2 #if __ARM_ARCH >= 7 #define HANDLE_HYP \ /* Leave HYP mode */ ;\ mrs r0, cpsr ;\ and r0, r0, #(PSR_MODE) /* Mode is in the low 5 bits of CPSR */ ;\ teq r0, #(PSR_HYP32_MODE) /* Hyp Mode? */ ;\ bne 1f ;\ /* Install Hypervisor Stub Exception Vector */ ;\ bl hypervisor_stub_vect_install ;\ mov r0, 0 ;\ adr r1, hypmode_enabled ;\ str r0, [r1] ;\ /* Ensure that IRQ, FIQ and Aborts will be disabled after eret */ ;\ mrs r0, cpsr ;\ bic r0, r0, #(PSR_MODE) ;\ orr r0, r0, #(PSR_SVC32_MODE) ;\ orr r0, r0, #(PSR_I | PSR_F | PSR_A) ;\ msr spsr_cxsf, r0 ;\ /* Exit hypervisor mode */ ;\ adr lr, 2f ;\ MSR_ELR_HYP(14) ;\ ERET ;\ 1: ;\ mov r0, -1 ;\ adr r1, hypmode_enabled ;\ str r0, [r1] ;\ 2: #else #define HANDLE_HYP #endif /* __ARM_ARCH >= 7 */ /* * On entry for FreeBSD boot ABI: * r0 - metadata pointer or 0 (boothowto on AT91's boot2) * r1 - if (r0 == 0) then metadata pointer * On entry for Linux boot ABI: * r0 - 0 * r1 - machine type (passed as arg2 to initarm) * r2 - Pointer to a tagged list or dtb image (phys addr) (passed as arg1 initarm) * * For both types of boot we gather up the args, put them in a struct arm_boot_params * structure and pass that to initarm. */ .globl btext btext: ASENTRY_NP(_start) STOP_UNWINDING /* Can't unwind into the bootloader! */ /* Make sure interrupts are disabled. */ cpsid ifa mov r8, r0 /* 0 or boot mode from boot2 */ mov r9, r1 /* Save Machine type */ mov r10, r2 /* Save meta data */ mov r11, r3 /* Future expansion */ # If HYP-MODE is active, install an exception vector stub HANDLE_HYP /* * Check whether data cache is enabled. If it is, then we know * current tags are valid (not power-on garbage values) and there * might be dirty lines that need cleaning. Disable cache to prevent * new lines being allocated, then call wbinv_poc_all to clean it. */ mrc CP15_SCTLR(r7) tst r7, #CPU_CONTROL_DC_ENABLE blne dcache_wbinv_poc_all /* ! Do not write to memory between wbinv and disabling cache ! */ /* * Now there are no dirty lines, but there may still be lines marked * valid. Disable all caches and the MMU, and invalidate everything * before setting up new page tables and re-enabling the mmu. */ 1: bic r7, #CPU_CONTROL_DC_ENABLE bic r7, #CPU_CONTROL_AFLT_ENABLE bic r7, #CPU_CONTROL_MMU_ENABLE bic r7, #CPU_CONTROL_IC_ENABLE bic r7, #CPU_CONTROL_BPRD_ENABLE bic r7, #CPU_CONTROL_SW_ENABLE orr r7, #CPU_CONTROL_UNAL_ENABLE orr r7, #CPU_CONTROL_VECRELOC mcr CP15_SCTLR(r7) DSB ISB bl dcache_inv_poc_all mcr CP15_ICIALLU DSB ISB /* * Build page table from scratch. */ /* * Figure out the physical address we're loaded at by assuming this * entry point code is in the first L1 section and so if we clear the * offset bits of the pc that will give us the section-aligned load * address, which remains in r5 throughout all the following code. */ ldr r2, =(L1_S_OFFSET) bic r5, pc, r2 /* Find the delta between VA and PA, result stays in r0 throughout. */ adr r0, Lpagetable bl translate_va_to_pa /* * First map the entire 4GB address space as VA=PA. It's mapped as * normal (cached) memory because it's for things like accessing the * parameters passed in from the bootloader, which might be at any * physical address, different for every platform. */ mov r1, #0 mov r2, #0 mov r3, #4096 bl build_pagetables /* * Next we map the kernel starting at the physical load address, mapped * to the VA the kernel is linked for. The default size we map is 64MiB * but it can be overridden with a kernel option. */ mov r1, r5 ldr r2, =(KERNVIRTADDR) ldr r3, =(LOCORE_MAP_MB) bl build_pagetables /* Create a device mapping for early_printf if specified. */ #if defined(SOCDEV_PA) && defined(SOCDEV_VA) ldr r1, =SOCDEV_PA ldr r2, =SOCDEV_VA mov r3, #1 bl build_device_pagetables #endif bl init_mmu /* Transition the PC from physical to virtual addressing. */ ldr pc, =1f 1: /* Setup stack, clear BSS */ ldr r1, =.Lstart ldmia r1, {r1, r2, sp} /* Set initial stack and */ add sp, sp, #INIT_ARM_STACK_SIZE sub r2, r2, r1 /* get zero init data */ mov r3, #0 2: str r3, [r1], #0x0004 /* get zero init data */ subs r2, r2, #4 bgt 2b mov r1, #28 /* loader info size is 28 bytes also second arg */ subs sp, sp, r1 /* allocate arm_boot_params struct on stack */ mov r0, sp /* loader info pointer is first arg */ bic sp, sp, #7 /* align stack to 8 bytes */ str r1, [r0] /* Store length of loader info */ str r8, [r0, #4] /* Store r0 from boot loader */ str r9, [r0, #8] /* Store r1 from boot loader */ str r10, [r0, #12] /* store r2 from boot loader */ str r11, [r0, #16] /* store r3 from boot loader */ str r5, [r0, #20] /* store the physical address */ adr r4, Lpagetable /* load the pagetable address */ ldr r5, [r4, #4] str r5, [r0, #24] /* store the pagetable address */ mov fp, #0 /* trace back starts here */ bl _C_LABEL(initarm) /* Off we go */ /* init arm will return the new stack pointer. */ mov sp, r0 bl _C_LABEL(mi_startup) /* call mi_startup()! */ ldr r0, =.Lmainreturned b _C_LABEL(panic) /* NOTREACHED */ END(_start) #define VA_TO_PA_POINTER(name, table) \ name: ;\ .word . ;\ .word table /* * Returns the physical address of a magic va to pa pointer. * r0 - The pagetable data pointer. This must be built using the * VA_TO_PA_POINTER macro. * e.g. * VA_TO_PA_POINTER(Lpagetable, pagetable) * ... * adr r0, Lpagetable * bl translate_va_to_pa * r0 will now contain the physical address of pagetable * r1, r2 - Trashed */ translate_va_to_pa: ldr r1, [r0] sub r2, r1, r0 /* At this point: r2 = VA - PA */ /* * Find the physical address of the table. After these two * instructions: * r1 = va(pagetable) * * r0 = va(pagetable) - (VA - PA) * = va(pagetable) - VA + PA * = pa(pagetable) */ ldr r1, [r0, #4] sub r0, r1, r2 mov pc, lr /* * Init MMU * r0 - the table base address */ ASENTRY_NP(init_mmu) /* Setup TLB and MMU registers */ mcr CP15_TTBR0(r0) /* Set TTB */ mov r0, #0 mcr CP15_CONTEXTIDR(r0) /* Set ASID to 0 */ /* Set the Domain Access register */ mov r0, #DOMAIN_CLIENT /* Only domain #0 is used */ mcr CP15_DACR(r0) /* * Set TEX remap registers * - All is set to uncacheable memory */ ldr r0, =0xAAAAA mcr CP15_PRRR(r0) mov r0, #0 mcr CP15_NMRR(r0) mcr CP15_TLBIALL /* Flush TLB */ DSB ISB /* Enable MMU */ mrc CP15_SCTLR(r0) orr r0, r0, #CPU_CONTROL_MMU_ENABLE orr r0, r0, #CPU_CONTROL_V6_EXTPAGE orr r0, r0, #CPU_CONTROL_TR_ENABLE orr r0, r0, #CPU_CONTROL_AF_ENABLE mcr CP15_SCTLR(r0) DSB ISB mcr CP15_TLBIALL /* Flush TLB */ mcr CP15_BPIALL /* Flush Branch predictor */ DSB ISB mov pc, lr END(init_mmu) /* * Init SMP coherent mode, enable caching and switch to final MMU table. * Called with disabled caches * r0 - The table base address * r1 - clear bits for aux register * r2 - set bits for aux register */ ASENTRY_NP(reinit_mmu) push {r4-r11, lr} mov r4, r0 mov r5, r1 mov r6, r2 /* !! Be very paranoid here !! */ /* !! We cannot write single bit here !! */ #if 0 /* XXX writeback shouldn't be necessary */ /* Write back and invalidate all integrated caches */ bl dcache_wbinv_poc_all #else bl dcache_inv_pou_all #endif mcr CP15_ICIALLU DSB ISB /* Set auxiliary register */ mrc CP15_ACTLR(r7) bic r8, r7, r5 /* Mask bits */ eor r8, r8, r6 /* Set bits */ teq r7, r8 mcrne CP15_ACTLR(r8) DSB ISB /* Enable caches. */ mrc CP15_SCTLR(r7) orr r7, #CPU_CONTROL_DC_ENABLE orr r7, #CPU_CONTROL_IC_ENABLE orr r7, #CPU_CONTROL_BPRD_ENABLE mcr CP15_SCTLR(r7) DSB mcr CP15_TTBR0(r4) /* Set new TTB */ DSB ISB mcr CP15_TLBIALL /* Flush TLB */ mcr CP15_BPIALL /* Flush Branch predictor */ DSB ISB #if 0 /* XXX writeback shouldn't be necessary */ /* Write back and invalidate all integrated caches */ bl dcache_wbinv_poc_all #else bl dcache_inv_pou_all #endif mcr CP15_ICIALLU DSB ISB pop {r4-r11, pc} END(reinit_mmu) /* * Builds the page table * r0 - The table base address * r1 - The physical address (trashed) * r2 - The virtual address (trashed) * r3 - The number of 1MiB sections * r4 - Trashed * * Addresses must be 1MiB aligned */ build_device_pagetables: ldr r4, =PTE1_V|PTE1_A|PTE1_AP_KRW|TEX1_CLASS_0 b 1f build_pagetables: /* Set the required page attributed */ ldr r4, =PTE1_V|PTE1_A|PTE1_AP_KRW|TEX1_CLASS_0 1: orr r1, r4 /* Move the virtual address to the correct bit location */ lsr r2, #(PTE1_SHIFT - 2) mov r4, r3 2: str r1, [r0, r2] add r2, r2, #4 add r1, r1, #(PTE1_SIZE) adds r4, r4, #-1 bhi 2b mov pc, lr VA_TO_PA_POINTER(Lpagetable, boot_pt1) .global _C_LABEL(hypmode_enabled) _C_LABEL(hypmode_enabled): .word 0 .Lstart: .word _edata /* Note that these three items are */ .word _ebss /* loaded with a single ldmia and */ .word svcstk /* must remain in order together. */ .Lmainreturned: .asciz "main() returned" .align 2 .bss svcstk: .space INIT_ARM_STACK_SIZE * MAXCPU /* * Memory for the initial pagetable. We are unable to place this in * the bss as this will be cleared after the table is loaded. */ .section ".init_pagetable", "aw", %nobits .align 14 /* 16KiB aligned */ .globl boot_pt1 boot_pt1: .space L1_TABLE_SIZE .text .align 2 #if defined(SMP) ASENTRY_NP(mpentry) /* Make sure interrupts are disabled. */ cpsid ifa HANDLE_HYP /* Setup core, disable all caches. */ mrc CP15_SCTLR(r0) bic r0, #CPU_CONTROL_MMU_ENABLE bic r0, #CPU_CONTROL_AFLT_ENABLE bic r0, #CPU_CONTROL_DC_ENABLE bic r0, #CPU_CONTROL_IC_ENABLE bic r0, #CPU_CONTROL_BPRD_ENABLE bic r0, #CPU_CONTROL_SW_ENABLE orr r0, #CPU_CONTROL_UNAL_ENABLE orr r0, #CPU_CONTROL_VECRELOC mcr CP15_SCTLR(r0) DSB ISB /* Invalidate L1 cache I+D cache */ bl dcache_inv_pou_all mcr CP15_ICIALLU DSB ISB /* Find the delta between VA and PA */ adr r0, Lpagetable bl translate_va_to_pa bl init_mmu adr r1, .Lstart+8 /* Get initstack pointer from */ ldr sp, [r1] /* startup data. */ mrc CP15_MPIDR(r0) /* Get processor id number. */ and r0, r0, #0x0f mov r1, #INIT_ARM_STACK_SIZE mul r2, r1, r0 /* Point sp to initstack */ add sp, sp, r2 /* area for this processor. */ /* Switch to virtual addresses. */ ldr pc, =1f 1: mov fp, #0 /* trace back starts here */ bl _C_LABEL(init_secondary)/* Off we go, cpu id in r0. */ adr r0, .Lmpreturned b _C_LABEL(panic) /* NOTREACHED */ END(mpentry) .Lmpreturned: .asciz "init_secondary() returned" .align 2 #endif ENTRY_NP(cpu_halt) /* XXX re-implement !!! */ cpsid ifa bl dcache_wbinv_poc_all ldr r4, .Lcpu_reset_address ldr r4, [r4] teq r4, #0 movne pc, r4 1: WFI b 1b /* * _cpu_reset_address contains the address to branch to, to complete * the cpu reset after turning the MMU off * This variable is provided by the hardware specific code */ .Lcpu_reset_address: .word _C_LABEL(cpu_reset_address) END(cpu_halt) /* * setjump + longjmp */ ENTRY(setjmp) stmia r0, {r4-r14} mov r0, #0x00000000 RET END(setjmp) ENTRY(longjmp) ldmia r0, {r4-r14} mov r0, #0x00000001 RET END(longjmp) .data .global _C_LABEL(esym) _C_LABEL(esym): .word _C_LABEL(end) ENTRY_NP(abort) b _C_LABEL(abort) END(abort) ENTRY_NP(sigcode) mov r0, sp add r0, r0, #SIGF_UC /* * Call the sigreturn system call. * * We have to load r7 manually rather than using * "ldr r7, =SYS_sigreturn" to ensure the value of szsigcode is * correct. Using the alternative places esigcode at the address * of the data rather than the address one past the data. */ ldr r7, [pc, #12] /* Load SYS_sigreturn */ swi SYS_sigreturn /* Well if that failed we better exit quick ! */ ldr r7, [pc, #8] /* Load SYS_exit */ swi SYS_exit /* Branch back to retry SYS_sigreturn */ b . - 16 END(sigcode) .word SYS_sigreturn .word SYS_exit .align 2 .global _C_LABEL(esigcode) _C_LABEL(esigcode): .data .global szsigcode szsigcode: .long esigcode-sigcode /* End of locore.S */ Index: head/sys/arm/arm/support.S =================================================================== --- head/sys/arm/arm/support.S (revision 331253) +++ head/sys/arm/arm/support.S (revision 331254) @@ -1,2965 +1,2965 @@ /*- * Copyright (c) 2004 Olivier Houchard * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c) 1997 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Neil A. Carson and Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); -#include "assym.s" +#include "assym.inc" .syntax unified .L_arm_memcpy: .word _C_LABEL(_arm_memcpy) .L_arm_bzero: .word _C_LABEL(_arm_bzero) .L_min_memcpy_size: .word _C_LABEL(_min_memcpy_size) .L_min_bzero_size: .word _C_LABEL(_min_bzero_size) /* * memset: Sets a block of memory to the specified value * * On entry: * r0 - dest address * r1 - byte to write * r2 - number of bytes to write * * On exit: * r0 - dest address */ /* LINTSTUB: Func: void bzero(void *, size_t) */ ENTRY(bzero) ldr r3, .L_arm_bzero ldr r3, [r3] cmp r3, #0 beq .Lnormal0 ldr r2, .L_min_bzero_size ldr r2, [r2] cmp r1, r2 blt .Lnormal0 stmfd sp!, {r0, r1, lr} mov r2, #0 mov lr, pc mov pc, r3 cmp r0, #0 ldmfd sp!, {r0, r1, lr} RETeq .Lnormal0: mov r3, #0x00 b do_memset END(bzero) /* LINTSTUB: Func: void *memset(void *, int, size_t) */ ENTRY(memset) and r3, r1, #0xff /* We deal with bytes */ mov r1, r2 do_memset: cmp r1, #0x04 /* Do we have less than 4 bytes */ mov ip, r0 blt .Lmemset_lessthanfour /* Ok first we will word align the address */ ands r2, ip, #0x03 /* Get the bottom two bits */ bne .Lmemset_wordunaligned /* The address is not word aligned */ /* We are now word aligned */ .Lmemset_wordaligned: orr r3, r3, r3, lsl #8 /* Extend value to 16-bits */ #ifdef _ARM_ARCH_5E tst ip, #0x04 /* Quad-align for armv5e */ #else cmp r1, #0x10 #endif orr r3, r3, r3, lsl #16 /* Extend value to 32-bits */ #ifdef _ARM_ARCH_5E subne r1, r1, #0x04 /* Quad-align if necessary */ strne r3, [ip], #0x04 cmp r1, #0x10 #endif blt .Lmemset_loop4 /* If less than 16 then use words */ mov r2, r3 /* Duplicate data */ cmp r1, #0x80 /* If < 128 then skip the big loop */ blt .Lmemset_loop32 /* Do 128 bytes at a time */ .Lmemset_loop128: subs r1, r1, #0x80 #ifdef _ARM_ARCH_5E strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 #else stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} #endif bgt .Lmemset_loop128 RETeq /* Zero length so just exit */ add r1, r1, #0x80 /* Adjust for extra sub */ /* Do 32 bytes at a time */ .Lmemset_loop32: subs r1, r1, #0x20 #ifdef _ARM_ARCH_5E strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 #else stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} #endif bgt .Lmemset_loop32 RETeq /* Zero length so just exit */ adds r1, r1, #0x10 /* Partially adjust for extra sub */ /* Deal with 16 bytes or more */ #ifdef _ARM_ARCH_5E strdge r2, [ip], #0x08 strdge r2, [ip], #0x08 #else stmiage ip!, {r2-r3} stmiage ip!, {r2-r3} #endif RETeq /* Zero length so just exit */ addlt r1, r1, #0x10 /* Possibly adjust for extra sub */ /* We have at least 4 bytes so copy as words */ .Lmemset_loop4: subs r1, r1, #0x04 strge r3, [ip], #0x04 bgt .Lmemset_loop4 RETeq /* Zero length so just exit */ #ifdef _ARM_ARCH_5E /* Compensate for 64-bit alignment check */ adds r1, r1, #0x04 RETeq cmp r1, #2 #else cmp r1, #-2 #endif strb r3, [ip], #0x01 /* Set 1 byte */ strbge r3, [ip], #0x01 /* Set another byte */ strbgt r3, [ip] /* and a third */ RET /* Exit */ .Lmemset_wordunaligned: rsb r2, r2, #0x004 strb r3, [ip], #0x01 /* Set 1 byte */ cmp r2, #0x02 strbge r3, [ip], #0x01 /* Set another byte */ sub r1, r1, r2 strbgt r3, [ip], #0x01 /* and a third */ cmp r1, #0x04 /* More than 4 bytes left? */ bge .Lmemset_wordaligned /* Yup */ .Lmemset_lessthanfour: cmp r1, #0x00 RETeq /* Zero length so exit */ strb r3, [ip], #0x01 /* Set 1 byte */ cmp r1, #0x02 strbge r3, [ip], #0x01 /* Set another byte */ strbgt r3, [ip] /* and a third */ RET /* Exit */ EEND(memset) END(bzero) ENTRY(bcmp) mov ip, r0 cmp r2, #0x06 beq .Lmemcmp_6bytes mov r0, #0x00 /* Are both addresses aligned the same way? */ cmp r2, #0x00 eorsne r3, ip, r1 RETeq /* len == 0, or same addresses! */ tst r3, #0x03 subne r2, r2, #0x01 bne .Lmemcmp_bytewise2 /* Badly aligned. Do it the slow way */ /* Word-align the addresses, if necessary */ sub r3, r1, #0x05 ands r3, r3, #0x03 add r3, r3, r3, lsl #1 addne pc, pc, r3, lsl #3 nop /* Compare up to 3 bytes */ ldrb r0, [ip], #0x01 ldrb r3, [r1], #0x01 subs r0, r0, r3 RETne subs r2, r2, #0x01 RETeq /* Compare up to 2 bytes */ ldrb r0, [ip], #0x01 ldrb r3, [r1], #0x01 subs r0, r0, r3 RETne subs r2, r2, #0x01 RETeq /* Compare 1 byte */ ldrb r0, [ip], #0x01 ldrb r3, [r1], #0x01 subs r0, r0, r3 RETne subs r2, r2, #0x01 RETeq /* Compare 4 bytes at a time, if possible */ subs r2, r2, #0x04 bcc .Lmemcmp_bytewise .Lmemcmp_word_aligned: ldr r0, [ip], #0x04 ldr r3, [r1], #0x04 subs r2, r2, #0x04 cmpcs r0, r3 beq .Lmemcmp_word_aligned sub r0, r0, r3 /* Correct for extra subtraction, and check if done */ adds r2, r2, #0x04 cmpeq r0, #0x00 /* If done, did all bytes match? */ RETeq /* Yup. Just return */ /* Re-do the final word byte-wise */ sub ip, ip, #0x04 sub r1, r1, #0x04 .Lmemcmp_bytewise: add r2, r2, #0x03 .Lmemcmp_bytewise2: ldrb r0, [ip], #0x01 ldrb r3, [r1], #0x01 subs r2, r2, #0x01 cmpcs r0, r3 beq .Lmemcmp_bytewise2 sub r0, r0, r3 RET /* * 6 byte compares are very common, thanks to the network stack. * This code is hand-scheduled to reduce the number of stalls for * load results. Everything else being equal, this will be ~32% * faster than a byte-wise memcmp. */ .align 5 .Lmemcmp_6bytes: ldrb r3, [r1, #0x00] /* r3 = b2#0 */ ldrb r0, [ip, #0x00] /* r0 = b1#0 */ ldrb r2, [r1, #0x01] /* r2 = b2#1 */ subs r0, r0, r3 /* r0 = b1#0 - b2#0 */ ldrbeq r3, [ip, #0x01] /* r3 = b1#1 */ RETne /* Return if mismatch on #0 */ subs r0, r3, r2 /* r0 = b1#1 - b2#1 */ ldrbeq r3, [r1, #0x02] /* r3 = b2#2 */ ldrbeq r0, [ip, #0x02] /* r0 = b1#2 */ RETne /* Return if mismatch on #1 */ ldrb r2, [r1, #0x03] /* r2 = b2#3 */ subs r0, r0, r3 /* r0 = b1#2 - b2#2 */ ldrbeq r3, [ip, #0x03] /* r3 = b1#3 */ RETne /* Return if mismatch on #2 */ subs r0, r3, r2 /* r0 = b1#3 - b2#3 */ ldrbeq r3, [r1, #0x04] /* r3 = b2#4 */ ldrbeq r0, [ip, #0x04] /* r0 = b1#4 */ RETne /* Return if mismatch on #3 */ ldrb r2, [r1, #0x05] /* r2 = b2#5 */ subs r0, r0, r3 /* r0 = b1#4 - b2#4 */ ldrbeq r3, [ip, #0x05] /* r3 = b1#5 */ RETne /* Return if mismatch on #4 */ sub r0, r3, r2 /* r0 = b1#5 - b2#5 */ RET END(bcmp) ENTRY(bcopy) /* switch the source and destination registers */ eor r0, r1, r0 eor r1, r0, r1 eor r0, r1, r0 EENTRY(memmove) /* Do the buffers overlap? */ cmp r0, r1 RETeq /* Bail now if src/dst are the same */ subcc r3, r0, r1 /* if (dst > src) r3 = dst - src */ subcs r3, r1, r0 /* if (src > dsr) r3 = src - dst */ cmp r3, r2 /* if (r3 < len) we have an overlap */ bcc PIC_SYM(_C_LABEL(memcpy), PLT) /* Determine copy direction */ cmp r1, r0 bcc .Lmemmove_backwards moveq r0, #0 /* Quick abort for len=0 */ RETeq stmdb sp!, {r0, lr} /* memmove() returns dest addr */ subs r2, r2, #4 blt .Lmemmove_fl4 /* less than 4 bytes */ ands r12, r0, #3 bne .Lmemmove_fdestul /* oh unaligned destination addr */ ands r12, r1, #3 bne .Lmemmove_fsrcul /* oh unaligned source addr */ .Lmemmove_ft8: /* We have aligned source and destination */ subs r2, r2, #8 blt .Lmemmove_fl12 /* less than 12 bytes (4 from above) */ subs r2, r2, #0x14 blt .Lmemmove_fl32 /* less than 32 bytes (12 from above) */ stmdb sp!, {r4} /* borrow r4 */ /* blat 32 bytes at a time */ /* XXX for really big copies perhaps we should use more registers */ .Lmemmove_floop32: ldmia r1!, {r3, r4, r12, lr} stmia r0!, {r3, r4, r12, lr} ldmia r1!, {r3, r4, r12, lr} stmia r0!, {r3, r4, r12, lr} subs r2, r2, #0x20 bge .Lmemmove_floop32 cmn r2, #0x10 ldmiage r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */ stmiage r0!, {r3, r4, r12, lr} subge r2, r2, #0x10 ldmia sp!, {r4} /* return r4 */ .Lmemmove_fl32: adds r2, r2, #0x14 /* blat 12 bytes at a time */ .Lmemmove_floop12: ldmiage r1!, {r3, r12, lr} stmiage r0!, {r3, r12, lr} subsge r2, r2, #0x0c bge .Lmemmove_floop12 .Lmemmove_fl12: adds r2, r2, #8 blt .Lmemmove_fl4 subs r2, r2, #4 ldrlt r3, [r1], #4 strlt r3, [r0], #4 ldmiage r1!, {r3, r12} stmiage r0!, {r3, r12} subge r2, r2, #4 .Lmemmove_fl4: /* less than 4 bytes to go */ adds r2, r2, #4 ldmiaeq sp!, {r0, pc} /* done */ /* copy the crud byte at a time */ cmp r2, #2 ldrb r3, [r1], #1 strb r3, [r0], #1 ldrbge r3, [r1], #1 strbge r3, [r0], #1 ldrbgt r3, [r1], #1 strbgt r3, [r0], #1 ldmia sp!, {r0, pc} /* erg - unaligned destination */ .Lmemmove_fdestul: rsb r12, r12, #4 cmp r12, #2 /* align destination with byte copies */ ldrb r3, [r1], #1 strb r3, [r0], #1 ldrbge r3, [r1], #1 strbge r3, [r0], #1 ldrbgt r3, [r1], #1 strbgt r3, [r0], #1 subs r2, r2, r12 blt .Lmemmove_fl4 /* less the 4 bytes */ ands r12, r1, #3 beq .Lmemmove_ft8 /* we have an aligned source */ /* erg - unaligned source */ /* This is where it gets nasty ... */ .Lmemmove_fsrcul: bic r1, r1, #3 ldr lr, [r1], #4 cmp r12, #2 bgt .Lmemmove_fsrcul3 beq .Lmemmove_fsrcul2 cmp r2, #0x0c blt .Lmemmove_fsrcul1loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5} .Lmemmove_fsrcul1loop16: #ifdef __ARMEB__ mov r3, lr, lsl #8 #else mov r3, lr, lsr #8 #endif ldmia r1!, {r4, r5, r12, lr} #ifdef __ARMEB__ orr r3, r3, r4, lsr #24 mov r4, r4, lsl #8 orr r4, r4, r5, lsr #24 mov r5, r5, lsl #8 orr r5, r5, r12, lsr #24 mov r12, r12, lsl #8 orr r12, r12, lr, lsr #24 #else orr r3, r3, r4, lsl #24 mov r4, r4, lsr #8 orr r4, r4, r5, lsl #24 mov r5, r5, lsr #8 orr r5, r5, r12, lsl #24 mov r12, r12, lsr #8 orr r12, r12, lr, lsl #24 #endif stmia r0!, {r3-r5, r12} subs r2, r2, #0x10 bge .Lmemmove_fsrcul1loop16 ldmia sp!, {r4, r5} adds r2, r2, #0x0c blt .Lmemmove_fsrcul1l4 .Lmemmove_fsrcul1loop4: #ifdef __ARMEB__ mov r12, lr, lsl #8 #else mov r12, lr, lsr #8 #endif ldr lr, [r1], #4 #ifdef __ARMEB__ orr r12, r12, lr, lsr #24 #else orr r12, r12, lr, lsl #24 #endif str r12, [r0], #4 subs r2, r2, #4 bge .Lmemmove_fsrcul1loop4 .Lmemmove_fsrcul1l4: sub r1, r1, #3 b .Lmemmove_fl4 .Lmemmove_fsrcul2: cmp r2, #0x0c blt .Lmemmove_fsrcul2loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5} .Lmemmove_fsrcul2loop16: #ifdef __ARMEB__ mov r3, lr, lsl #16 #else mov r3, lr, lsr #16 #endif ldmia r1!, {r4, r5, r12, lr} #ifdef __ARMEB__ orr r3, r3, r4, lsr #16 mov r4, r4, lsl #16 orr r4, r4, r5, lsr #16 mov r5, r5, lsl #16 orr r5, r5, r12, lsr #16 mov r12, r12, lsl #16 orr r12, r12, lr, lsr #16 #else orr r3, r3, r4, lsl #16 mov r4, r4, lsr #16 orr r4, r4, r5, lsl #16 mov r5, r5, lsr #16 orr r5, r5, r12, lsl #16 mov r12, r12, lsr #16 orr r12, r12, lr, lsl #16 #endif stmia r0!, {r3-r5, r12} subs r2, r2, #0x10 bge .Lmemmove_fsrcul2loop16 ldmia sp!, {r4, r5} adds r2, r2, #0x0c blt .Lmemmove_fsrcul2l4 .Lmemmove_fsrcul2loop4: #ifdef __ARMEB__ mov r12, lr, lsl #16 #else mov r12, lr, lsr #16 #endif ldr lr, [r1], #4 #ifdef __ARMEB__ orr r12, r12, lr, lsr #16 #else orr r12, r12, lr, lsl #16 #endif str r12, [r0], #4 subs r2, r2, #4 bge .Lmemmove_fsrcul2loop4 .Lmemmove_fsrcul2l4: sub r1, r1, #2 b .Lmemmove_fl4 .Lmemmove_fsrcul3: cmp r2, #0x0c blt .Lmemmove_fsrcul3loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5} .Lmemmove_fsrcul3loop16: #ifdef __ARMEB__ mov r3, lr, lsl #24 #else mov r3, lr, lsr #24 #endif ldmia r1!, {r4, r5, r12, lr} #ifdef __ARMEB__ orr r3, r3, r4, lsr #8 mov r4, r4, lsl #24 orr r4, r4, r5, lsr #8 mov r5, r5, lsl #24 orr r5, r5, r12, lsr #8 mov r12, r12, lsl #24 orr r12, r12, lr, lsr #8 #else orr r3, r3, r4, lsl #8 mov r4, r4, lsr #24 orr r4, r4, r5, lsl #8 mov r5, r5, lsr #24 orr r5, r5, r12, lsl #8 mov r12, r12, lsr #24 orr r12, r12, lr, lsl #8 #endif stmia r0!, {r3-r5, r12} subs r2, r2, #0x10 bge .Lmemmove_fsrcul3loop16 ldmia sp!, {r4, r5} adds r2, r2, #0x0c blt .Lmemmove_fsrcul3l4 .Lmemmove_fsrcul3loop4: #ifdef __ARMEB__ mov r12, lr, lsl #24 #else mov r12, lr, lsr #24 #endif ldr lr, [r1], #4 #ifdef __ARMEB__ orr r12, r12, lr, lsr #8 #else orr r12, r12, lr, lsl #8 #endif str r12, [r0], #4 subs r2, r2, #4 bge .Lmemmove_fsrcul3loop4 .Lmemmove_fsrcul3l4: sub r1, r1, #1 b .Lmemmove_fl4 .Lmemmove_backwards: add r1, r1, r2 add r0, r0, r2 subs r2, r2, #4 blt .Lmemmove_bl4 /* less than 4 bytes */ ands r12, r0, #3 bne .Lmemmove_bdestul /* oh unaligned destination addr */ ands r12, r1, #3 bne .Lmemmove_bsrcul /* oh unaligned source addr */ .Lmemmove_bt8: /* We have aligned source and destination */ subs r2, r2, #8 blt .Lmemmove_bl12 /* less than 12 bytes (4 from above) */ stmdb sp!, {r4, lr} subs r2, r2, #0x14 /* less than 32 bytes (12 from above) */ blt .Lmemmove_bl32 /* blat 32 bytes at a time */ /* XXX for really big copies perhaps we should use more registers */ .Lmemmove_bloop32: ldmdb r1!, {r3, r4, r12, lr} stmdb r0!, {r3, r4, r12, lr} ldmdb r1!, {r3, r4, r12, lr} stmdb r0!, {r3, r4, r12, lr} subs r2, r2, #0x20 bge .Lmemmove_bloop32 .Lmemmove_bl32: cmn r2, #0x10 ldmdbge r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */ stmdbge r0!, {r3, r4, r12, lr} subge r2, r2, #0x10 adds r2, r2, #0x14 ldmdbge r1!, {r3, r12, lr} /* blat a remaining 12 bytes */ stmdbge r0!, {r3, r12, lr} subge r2, r2, #0x0c ldmia sp!, {r4, lr} .Lmemmove_bl12: adds r2, r2, #8 blt .Lmemmove_bl4 subs r2, r2, #4 ldrlt r3, [r1, #-4]! strlt r3, [r0, #-4]! ldmdbge r1!, {r3, r12} stmdbge r0!, {r3, r12} subge r2, r2, #4 .Lmemmove_bl4: /* less than 4 bytes to go */ adds r2, r2, #4 RETeq /* done */ /* copy the crud byte at a time */ cmp r2, #2 ldrb r3, [r1, #-1]! strb r3, [r0, #-1]! ldrbge r3, [r1, #-1]! strbge r3, [r0, #-1]! ldrbgt r3, [r1, #-1]! strbgt r3, [r0, #-1]! RET /* erg - unaligned destination */ .Lmemmove_bdestul: cmp r12, #2 /* align destination with byte copies */ ldrb r3, [r1, #-1]! strb r3, [r0, #-1]! ldrbge r3, [r1, #-1]! strbge r3, [r0, #-1]! ldrbgt r3, [r1, #-1]! strbgt r3, [r0, #-1]! subs r2, r2, r12 blt .Lmemmove_bl4 /* less than 4 bytes to go */ ands r12, r1, #3 beq .Lmemmove_bt8 /* we have an aligned source */ /* erg - unaligned source */ /* This is where it gets nasty ... */ .Lmemmove_bsrcul: bic r1, r1, #3 ldr r3, [r1, #0] cmp r12, #2 blt .Lmemmove_bsrcul1 beq .Lmemmove_bsrcul2 cmp r2, #0x0c blt .Lmemmove_bsrcul3loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5, lr} .Lmemmove_bsrcul3loop16: #ifdef __ARMEB__ mov lr, r3, lsr #8 #else mov lr, r3, lsl #8 #endif ldmdb r1!, {r3-r5, r12} #ifdef __ARMEB__ orr lr, lr, r12, lsl #24 mov r12, r12, lsr #8 orr r12, r12, r5, lsl #24 mov r5, r5, lsr #8 orr r5, r5, r4, lsl #24 mov r4, r4, lsr #8 orr r4, r4, r3, lsl #24 #else orr lr, lr, r12, lsr #24 mov r12, r12, lsl #8 orr r12, r12, r5, lsr #24 mov r5, r5, lsl #8 orr r5, r5, r4, lsr #24 mov r4, r4, lsl #8 orr r4, r4, r3, lsr #24 #endif stmdb r0!, {r4, r5, r12, lr} subs r2, r2, #0x10 bge .Lmemmove_bsrcul3loop16 ldmia sp!, {r4, r5, lr} adds r2, r2, #0x0c blt .Lmemmove_bsrcul3l4 .Lmemmove_bsrcul3loop4: #ifdef __ARMEB__ mov r12, r3, lsr #8 #else mov r12, r3, lsl #8 #endif ldr r3, [r1, #-4]! #ifdef __ARMEB__ orr r12, r12, r3, lsl #24 #else orr r12, r12, r3, lsr #24 #endif str r12, [r0, #-4]! subs r2, r2, #4 bge .Lmemmove_bsrcul3loop4 .Lmemmove_bsrcul3l4: add r1, r1, #3 b .Lmemmove_bl4 .Lmemmove_bsrcul2: cmp r2, #0x0c blt .Lmemmove_bsrcul2loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5, lr} .Lmemmove_bsrcul2loop16: #ifdef __ARMEB__ mov lr, r3, lsr #16 #else mov lr, r3, lsl #16 #endif ldmdb r1!, {r3-r5, r12} #ifdef __ARMEB__ orr lr, lr, r12, lsl #16 mov r12, r12, lsr #16 orr r12, r12, r5, lsl #16 mov r5, r5, lsr #16 orr r5, r5, r4, lsl #16 mov r4, r4, lsr #16 orr r4, r4, r3, lsl #16 #else orr lr, lr, r12, lsr #16 mov r12, r12, lsl #16 orr r12, r12, r5, lsr #16 mov r5, r5, lsl #16 orr r5, r5, r4, lsr #16 mov r4, r4, lsl #16 orr r4, r4, r3, lsr #16 #endif stmdb r0!, {r4, r5, r12, lr} subs r2, r2, #0x10 bge .Lmemmove_bsrcul2loop16 ldmia sp!, {r4, r5, lr} adds r2, r2, #0x0c blt .Lmemmove_bsrcul2l4 .Lmemmove_bsrcul2loop4: #ifdef __ARMEB__ mov r12, r3, lsr #16 #else mov r12, r3, lsl #16 #endif ldr r3, [r1, #-4]! #ifdef __ARMEB__ orr r12, r12, r3, lsl #16 #else orr r12, r12, r3, lsr #16 #endif str r12, [r0, #-4]! subs r2, r2, #4 bge .Lmemmove_bsrcul2loop4 .Lmemmove_bsrcul2l4: add r1, r1, #2 b .Lmemmove_bl4 .Lmemmove_bsrcul1: cmp r2, #0x0c blt .Lmemmove_bsrcul1loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5, lr} .Lmemmove_bsrcul1loop32: #ifdef __ARMEB__ mov lr, r3, lsr #24 #else mov lr, r3, lsl #24 #endif ldmdb r1!, {r3-r5, r12} #ifdef __ARMEB__ orr lr, lr, r12, lsl #8 mov r12, r12, lsr #24 orr r12, r12, r5, lsl #8 mov r5, r5, lsr #24 orr r5, r5, r4, lsl #8 mov r4, r4, lsr #24 orr r4, r4, r3, lsl #8 #else orr lr, lr, r12, lsr #8 mov r12, r12, lsl #24 orr r12, r12, r5, lsr #8 mov r5, r5, lsl #24 orr r5, r5, r4, lsr #8 mov r4, r4, lsl #24 orr r4, r4, r3, lsr #8 #endif stmdb r0!, {r4, r5, r12, lr} subs r2, r2, #0x10 bge .Lmemmove_bsrcul1loop32 ldmia sp!, {r4, r5, lr} adds r2, r2, #0x0c blt .Lmemmove_bsrcul1l4 .Lmemmove_bsrcul1loop4: #ifdef __ARMEB__ mov r12, r3, lsr #24 #else mov r12, r3, lsl #24 #endif ldr r3, [r1, #-4]! #ifdef __ARMEB__ orr r12, r12, r3, lsl #8 #else orr r12, r12, r3, lsr #8 #endif str r12, [r0, #-4]! subs r2, r2, #4 bge .Lmemmove_bsrcul1loop4 .Lmemmove_bsrcul1l4: add r1, r1, #1 b .Lmemmove_bl4 EEND(memmove) END(bcopy) #if !defined(_ARM_ARCH_5E) ENTRY(memcpy) /* save leaf functions having to store this away */ /* Do not check arm_memcpy if we're running from flash */ #if defined(FLASHADDR) && defined(PHYSADDR) #if FLASHADDR > PHYSADDR ldr r3, =FLASHADDR cmp r3, pc bls .Lnormal #else ldr r3, =FLASHADDR cmp r3, pc bhi .Lnormal #endif #endif ldr r3, .L_arm_memcpy ldr r3, [r3] cmp r3, #0 beq .Lnormal ldr r3, .L_min_memcpy_size ldr r3, [r3] cmp r2, r3 blt .Lnormal stmfd sp!, {r0-r2, r4, lr} mov r3, #0 ldr r4, .L_arm_memcpy mov lr, pc ldr pc, [r4] cmp r0, #0 ldmfd sp!, {r0-r2, r4, lr} RETeq .Lnormal: stmdb sp!, {r0, lr} /* memcpy() returns dest addr */ subs r2, r2, #4 blt .Lmemcpy_l4 /* less than 4 bytes */ ands r12, r0, #3 bne .Lmemcpy_destul /* oh unaligned destination addr */ ands r12, r1, #3 bne .Lmemcpy_srcul /* oh unaligned source addr */ .Lmemcpy_t8: /* We have aligned source and destination */ subs r2, r2, #8 blt .Lmemcpy_l12 /* less than 12 bytes (4 from above) */ subs r2, r2, #0x14 blt .Lmemcpy_l32 /* less than 32 bytes (12 from above) */ stmdb sp!, {r4} /* borrow r4 */ /* blat 32 bytes at a time */ /* XXX for really big copies perhaps we should use more registers */ .Lmemcpy_loop32: ldmia r1!, {r3, r4, r12, lr} stmia r0!, {r3, r4, r12, lr} ldmia r1!, {r3, r4, r12, lr} stmia r0!, {r3, r4, r12, lr} subs r2, r2, #0x20 bge .Lmemcpy_loop32 cmn r2, #0x10 ldmiage r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */ stmiage r0!, {r3, r4, r12, lr} subge r2, r2, #0x10 ldmia sp!, {r4} /* return r4 */ .Lmemcpy_l32: adds r2, r2, #0x14 /* blat 12 bytes at a time */ .Lmemcpy_loop12: ldmiage r1!, {r3, r12, lr} stmiage r0!, {r3, r12, lr} subsge r2, r2, #0x0c bge .Lmemcpy_loop12 .Lmemcpy_l12: adds r2, r2, #8 blt .Lmemcpy_l4 subs r2, r2, #4 ldrlt r3, [r1], #4 strlt r3, [r0], #4 ldmiage r1!, {r3, r12} stmiage r0!, {r3, r12} subge r2, r2, #4 .Lmemcpy_l4: /* less than 4 bytes to go */ adds r2, r2, #4 #ifdef __APCS_26_ ldmiaeq sp!, {r0, pc}^ /* done */ #else ldmiaeq sp!, {r0, pc} /* done */ #endif /* copy the crud byte at a time */ cmp r2, #2 ldrb r3, [r1], #1 strb r3, [r0], #1 ldrbge r3, [r1], #1 strbge r3, [r0], #1 ldrbgt r3, [r1], #1 strbgt r3, [r0], #1 ldmia sp!, {r0, pc} /* erg - unaligned destination */ .Lmemcpy_destul: rsb r12, r12, #4 cmp r12, #2 /* align destination with byte copies */ ldrb r3, [r1], #1 strb r3, [r0], #1 ldrbge r3, [r1], #1 strbge r3, [r0], #1 ldrbgt r3, [r1], #1 strbgt r3, [r0], #1 subs r2, r2, r12 blt .Lmemcpy_l4 /* less the 4 bytes */ ands r12, r1, #3 beq .Lmemcpy_t8 /* we have an aligned source */ /* erg - unaligned source */ /* This is where it gets nasty ... */ .Lmemcpy_srcul: bic r1, r1, #3 ldr lr, [r1], #4 cmp r12, #2 bgt .Lmemcpy_srcul3 beq .Lmemcpy_srcul2 cmp r2, #0x0c blt .Lmemcpy_srcul1loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5} .Lmemcpy_srcul1loop16: mov r3, lr, lsr #8 ldmia r1!, {r4, r5, r12, lr} orr r3, r3, r4, lsl #24 mov r4, r4, lsr #8 orr r4, r4, r5, lsl #24 mov r5, r5, lsr #8 orr r5, r5, r12, lsl #24 mov r12, r12, lsr #8 orr r12, r12, lr, lsl #24 stmia r0!, {r3-r5, r12} subs r2, r2, #0x10 bge .Lmemcpy_srcul1loop16 ldmia sp!, {r4, r5} adds r2, r2, #0x0c blt .Lmemcpy_srcul1l4 .Lmemcpy_srcul1loop4: mov r12, lr, lsr #8 ldr lr, [r1], #4 orr r12, r12, lr, lsl #24 str r12, [r0], #4 subs r2, r2, #4 bge .Lmemcpy_srcul1loop4 .Lmemcpy_srcul1l4: sub r1, r1, #3 b .Lmemcpy_l4 .Lmemcpy_srcul2: cmp r2, #0x0c blt .Lmemcpy_srcul2loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5} .Lmemcpy_srcul2loop16: mov r3, lr, lsr #16 ldmia r1!, {r4, r5, r12, lr} orr r3, r3, r4, lsl #16 mov r4, r4, lsr #16 orr r4, r4, r5, lsl #16 mov r5, r5, lsr #16 orr r5, r5, r12, lsl #16 mov r12, r12, lsr #16 orr r12, r12, lr, lsl #16 stmia r0!, {r3-r5, r12} subs r2, r2, #0x10 bge .Lmemcpy_srcul2loop16 ldmia sp!, {r4, r5} adds r2, r2, #0x0c blt .Lmemcpy_srcul2l4 .Lmemcpy_srcul2loop4: mov r12, lr, lsr #16 ldr lr, [r1], #4 orr r12, r12, lr, lsl #16 str r12, [r0], #4 subs r2, r2, #4 bge .Lmemcpy_srcul2loop4 .Lmemcpy_srcul2l4: sub r1, r1, #2 b .Lmemcpy_l4 .Lmemcpy_srcul3: cmp r2, #0x0c blt .Lmemcpy_srcul3loop4 sub r2, r2, #0x0c stmdb sp!, {r4, r5} .Lmemcpy_srcul3loop16: mov r3, lr, lsr #24 ldmia r1!, {r4, r5, r12, lr} orr r3, r3, r4, lsl #8 mov r4, r4, lsr #24 orr r4, r4, r5, lsl #8 mov r5, r5, lsr #24 orr r5, r5, r12, lsl #8 mov r12, r12, lsr #24 orr r12, r12, lr, lsl #8 stmia r0!, {r3-r5, r12} subs r2, r2, #0x10 bge .Lmemcpy_srcul3loop16 ldmia sp!, {r4, r5} adds r2, r2, #0x0c blt .Lmemcpy_srcul3l4 .Lmemcpy_srcul3loop4: mov r12, lr, lsr #24 ldr lr, [r1], #4 orr r12, r12, lr, lsl #8 str r12, [r0], #4 subs r2, r2, #4 bge .Lmemcpy_srcul3loop4 .Lmemcpy_srcul3l4: sub r1, r1, #1 b .Lmemcpy_l4 END(memcpy) #else /* LINTSTUB: Func: void *memcpy(void *dst, const void *src, size_t len) */ ENTRY(memcpy) pld [r1] cmp r2, #0x0c ble .Lmemcpy_short /* <= 12 bytes */ #ifdef FLASHADDR #if FLASHADDR > PHYSADDR ldr r3, =FLASHADDR cmp r3, pc bls .Lnormal #else ldr r3, =FLASHADDR cmp r3, pc bhi .Lnormal #endif #endif ldr r3, .L_arm_memcpy ldr r3, [r3] cmp r3, #0 beq .Lnormal ldr r3, .L_min_memcpy_size ldr r3, [r3] cmp r2, r3 blt .Lnormal stmfd sp!, {r0-r2, r4, lr} mov r3, #0 ldr r4, .L_arm_memcpy mov lr, pc ldr pc, [r4] cmp r0, #0 ldmfd sp!, {r0-r2, r4, lr} RETeq .Lnormal: mov r3, r0 /* We must not clobber r0 */ /* Word-align the destination buffer */ ands ip, r3, #0x03 /* Already word aligned? */ beq .Lmemcpy_wordaligned /* Yup */ cmp ip, #0x02 ldrb ip, [r1], #0x01 sub r2, r2, #0x01 strb ip, [r3], #0x01 ldrble ip, [r1], #0x01 suble r2, r2, #0x01 strble ip, [r3], #0x01 ldrblt ip, [r1], #0x01 sublt r2, r2, #0x01 strblt ip, [r3], #0x01 /* Destination buffer is now word aligned */ .Lmemcpy_wordaligned: ands ip, r1, #0x03 /* Is src also word-aligned? */ bne .Lmemcpy_bad_align /* Nope. Things just got bad */ /* Quad-align the destination buffer */ tst r3, #0x07 /* Already quad aligned? */ ldrne ip, [r1], #0x04 stmfd sp!, {r4-r9} /* Free up some registers */ subne r2, r2, #0x04 strne ip, [r3], #0x04 /* Destination buffer quad aligned, source is at least word aligned */ subs r2, r2, #0x80 blt .Lmemcpy_w_lessthan128 /* Copy 128 bytes at a time */ .Lmemcpy_w_loop128: ldr r4, [r1], #0x04 /* LD:00-03 */ ldr r5, [r1], #0x04 /* LD:04-07 */ pld [r1, #0x18] /* Prefetch 0x20 */ ldr r6, [r1], #0x04 /* LD:08-0b */ ldr r7, [r1], #0x04 /* LD:0c-0f */ ldr r8, [r1], #0x04 /* LD:10-13 */ ldr r9, [r1], #0x04 /* LD:14-17 */ strd r4, [r3], #0x08 /* ST:00-07 */ ldr r4, [r1], #0x04 /* LD:18-1b */ ldr r5, [r1], #0x04 /* LD:1c-1f */ strd r6, [r3], #0x08 /* ST:08-0f */ ldr r6, [r1], #0x04 /* LD:20-23 */ ldr r7, [r1], #0x04 /* LD:24-27 */ pld [r1, #0x18] /* Prefetch 0x40 */ strd r8, [r3], #0x08 /* ST:10-17 */ ldr r8, [r1], #0x04 /* LD:28-2b */ ldr r9, [r1], #0x04 /* LD:2c-2f */ strd r4, [r3], #0x08 /* ST:18-1f */ ldr r4, [r1], #0x04 /* LD:30-33 */ ldr r5, [r1], #0x04 /* LD:34-37 */ strd r6, [r3], #0x08 /* ST:20-27 */ ldr r6, [r1], #0x04 /* LD:38-3b */ ldr r7, [r1], #0x04 /* LD:3c-3f */ strd r8, [r3], #0x08 /* ST:28-2f */ ldr r8, [r1], #0x04 /* LD:40-43 */ ldr r9, [r1], #0x04 /* LD:44-47 */ pld [r1, #0x18] /* Prefetch 0x60 */ strd r4, [r3], #0x08 /* ST:30-37 */ ldr r4, [r1], #0x04 /* LD:48-4b */ ldr r5, [r1], #0x04 /* LD:4c-4f */ strd r6, [r3], #0x08 /* ST:38-3f */ ldr r6, [r1], #0x04 /* LD:50-53 */ ldr r7, [r1], #0x04 /* LD:54-57 */ strd r8, [r3], #0x08 /* ST:40-47 */ ldr r8, [r1], #0x04 /* LD:58-5b */ ldr r9, [r1], #0x04 /* LD:5c-5f */ strd r4, [r3], #0x08 /* ST:48-4f */ ldr r4, [r1], #0x04 /* LD:60-63 */ ldr r5, [r1], #0x04 /* LD:64-67 */ pld [r1, #0x18] /* Prefetch 0x80 */ strd r6, [r3], #0x08 /* ST:50-57 */ ldr r6, [r1], #0x04 /* LD:68-6b */ ldr r7, [r1], #0x04 /* LD:6c-6f */ strd r8, [r3], #0x08 /* ST:58-5f */ ldr r8, [r1], #0x04 /* LD:70-73 */ ldr r9, [r1], #0x04 /* LD:74-77 */ strd r4, [r3], #0x08 /* ST:60-67 */ ldr r4, [r1], #0x04 /* LD:78-7b */ ldr r5, [r1], #0x04 /* LD:7c-7f */ strd r6, [r3], #0x08 /* ST:68-6f */ strd r8, [r3], #0x08 /* ST:70-77 */ subs r2, r2, #0x80 strd r4, [r3], #0x08 /* ST:78-7f */ bge .Lmemcpy_w_loop128 .Lmemcpy_w_lessthan128: adds r2, r2, #0x80 /* Adjust for extra sub */ ldmfdeq sp!, {r4-r9} RETeq /* Return now if done */ subs r2, r2, #0x20 blt .Lmemcpy_w_lessthan32 /* Copy 32 bytes at a time */ .Lmemcpy_w_loop32: ldr r4, [r1], #0x04 ldr r5, [r1], #0x04 pld [r1, #0x18] ldr r6, [r1], #0x04 ldr r7, [r1], #0x04 ldr r8, [r1], #0x04 ldr r9, [r1], #0x04 strd r4, [r3], #0x08 ldr r4, [r1], #0x04 ldr r5, [r1], #0x04 strd r6, [r3], #0x08 strd r8, [r3], #0x08 subs r2, r2, #0x20 strd r4, [r3], #0x08 bge .Lmemcpy_w_loop32 .Lmemcpy_w_lessthan32: adds r2, r2, #0x20 /* Adjust for extra sub */ ldmfdeq sp!, {r4-r9} RETeq /* Return now if done */ and r4, r2, #0x18 rsbs r4, r4, #0x18 addne pc, pc, r4, lsl #1 nop /* At least 24 bytes remaining */ ldr r4, [r1], #0x04 ldr r5, [r1], #0x04 sub r2, r2, #0x08 strd r4, [r3], #0x08 /* At least 16 bytes remaining */ ldr r4, [r1], #0x04 ldr r5, [r1], #0x04 sub r2, r2, #0x08 strd r4, [r3], #0x08 /* At least 8 bytes remaining */ ldr r4, [r1], #0x04 ldr r5, [r1], #0x04 subs r2, r2, #0x08 strd r4, [r3], #0x08 /* Less than 8 bytes remaining */ ldmfd sp!, {r4-r9} RETeq /* Return now if done */ subs r2, r2, #0x04 ldrge ip, [r1], #0x04 strge ip, [r3], #0x04 RETeq /* Return now if done */ addlt r2, r2, #0x04 ldrb ip, [r1], #0x01 cmp r2, #0x02 ldrbge r2, [r1], #0x01 strb ip, [r3], #0x01 ldrbgt ip, [r1] strbge r2, [r3], #0x01 strbgt ip, [r3] RET /* Place a literal pool here for the above ldr instructions to use */ .ltorg /* * At this point, it has not been possible to word align both buffers. * The destination buffer is word aligned, but the source buffer is not. */ .Lmemcpy_bad_align: stmfd sp!, {r4-r7} bic r1, r1, #0x03 cmp ip, #2 ldr ip, [r1], #0x04 bgt .Lmemcpy_bad3 beq .Lmemcpy_bad2 b .Lmemcpy_bad1 .Lmemcpy_bad1_loop16: #ifdef __ARMEB__ mov r4, ip, lsl #8 #else mov r4, ip, lsr #8 #endif ldr r5, [r1], #0x04 pld [r1, #0x018] ldr r6, [r1], #0x04 ldr r7, [r1], #0x04 ldr ip, [r1], #0x04 #ifdef __ARMEB__ orr r4, r4, r5, lsr #24 mov r5, r5, lsl #8 orr r5, r5, r6, lsr #24 mov r6, r6, lsl #8 orr r6, r6, r7, lsr #24 mov r7, r7, lsl #8 orr r7, r7, ip, lsr #24 #else orr r4, r4, r5, lsl #24 mov r5, r5, lsr #8 orr r5, r5, r6, lsl #24 mov r6, r6, lsr #8 orr r6, r6, r7, lsl #24 mov r7, r7, lsr #8 orr r7, r7, ip, lsl #24 #endif str r4, [r3], #0x04 str r5, [r3], #0x04 str r6, [r3], #0x04 str r7, [r3], #0x04 .Lmemcpy_bad1: subs r2, r2, #0x10 bge .Lmemcpy_bad1_loop16 adds r2, r2, #0x10 ldmfdeq sp!, {r4-r7} RETeq /* Return now if done */ subs r2, r2, #0x04 sublt r1, r1, #0x03 blt .Lmemcpy_bad_done .Lmemcpy_bad1_loop4: #ifdef __ARMEB__ mov r4, ip, lsl #8 #else mov r4, ip, lsr #8 #endif ldr ip, [r1], #0x04 subs r2, r2, #0x04 #ifdef __ARMEB__ orr r4, r4, ip, lsr #24 #else orr r4, r4, ip, lsl #24 #endif str r4, [r3], #0x04 bge .Lmemcpy_bad1_loop4 sub r1, r1, #0x03 b .Lmemcpy_bad_done .Lmemcpy_bad2_loop16: #ifdef __ARMEB__ mov r4, ip, lsl #16 #else mov r4, ip, lsr #16 #endif ldr r5, [r1], #0x04 pld [r1, #0x018] ldr r6, [r1], #0x04 ldr r7, [r1], #0x04 ldr ip, [r1], #0x04 #ifdef __ARMEB__ orr r4, r4, r5, lsr #16 mov r5, r5, lsl #16 orr r5, r5, r6, lsr #16 mov r6, r6, lsl #16 orr r6, r6, r7, lsr #16 mov r7, r7, lsl #16 orr r7, r7, ip, lsr #16 #else orr r4, r4, r5, lsl #16 mov r5, r5, lsr #16 orr r5, r5, r6, lsl #16 mov r6, r6, lsr #16 orr r6, r6, r7, lsl #16 mov r7, r7, lsr #16 orr r7, r7, ip, lsl #16 #endif str r4, [r3], #0x04 str r5, [r3], #0x04 str r6, [r3], #0x04 str r7, [r3], #0x04 .Lmemcpy_bad2: subs r2, r2, #0x10 bge .Lmemcpy_bad2_loop16 adds r2, r2, #0x10 ldmfdeq sp!, {r4-r7} RETeq /* Return now if done */ subs r2, r2, #0x04 sublt r1, r1, #0x02 blt .Lmemcpy_bad_done .Lmemcpy_bad2_loop4: #ifdef __ARMEB__ mov r4, ip, lsl #16 #else mov r4, ip, lsr #16 #endif ldr ip, [r1], #0x04 subs r2, r2, #0x04 #ifdef __ARMEB__ orr r4, r4, ip, lsr #16 #else orr r4, r4, ip, lsl #16 #endif str r4, [r3], #0x04 bge .Lmemcpy_bad2_loop4 sub r1, r1, #0x02 b .Lmemcpy_bad_done .Lmemcpy_bad3_loop16: #ifdef __ARMEB__ mov r4, ip, lsl #24 #else mov r4, ip, lsr #24 #endif ldr r5, [r1], #0x04 pld [r1, #0x018] ldr r6, [r1], #0x04 ldr r7, [r1], #0x04 ldr ip, [r1], #0x04 #ifdef __ARMEB__ orr r4, r4, r5, lsr #8 mov r5, r5, lsl #24 orr r5, r5, r6, lsr #8 mov r6, r6, lsl #24 orr r6, r6, r7, lsr #8 mov r7, r7, lsl #24 orr r7, r7, ip, lsr #8 #else orr r4, r4, r5, lsl #8 mov r5, r5, lsr #24 orr r5, r5, r6, lsl #8 mov r6, r6, lsr #24 orr r6, r6, r7, lsl #8 mov r7, r7, lsr #24 orr r7, r7, ip, lsl #8 #endif str r4, [r3], #0x04 str r5, [r3], #0x04 str r6, [r3], #0x04 str r7, [r3], #0x04 .Lmemcpy_bad3: subs r2, r2, #0x10 bge .Lmemcpy_bad3_loop16 adds r2, r2, #0x10 ldmfdeq sp!, {r4-r7} RETeq /* Return now if done */ subs r2, r2, #0x04 sublt r1, r1, #0x01 blt .Lmemcpy_bad_done .Lmemcpy_bad3_loop4: #ifdef __ARMEB__ mov r4, ip, lsl #24 #else mov r4, ip, lsr #24 #endif ldr ip, [r1], #0x04 subs r2, r2, #0x04 #ifdef __ARMEB__ orr r4, r4, ip, lsr #8 #else orr r4, r4, ip, lsl #8 #endif str r4, [r3], #0x04 bge .Lmemcpy_bad3_loop4 sub r1, r1, #0x01 .Lmemcpy_bad_done: ldmfd sp!, {r4-r7} adds r2, r2, #0x04 RETeq ldrb ip, [r1], #0x01 cmp r2, #0x02 ldrbge r2, [r1], #0x01 strb ip, [r3], #0x01 ldrbgt ip, [r1] strbge r2, [r3], #0x01 strbgt ip, [r3] RET /* * Handle short copies (less than 16 bytes), possibly misaligned. * Some of these are *very* common, thanks to the network stack, * and so are handled specially. */ .Lmemcpy_short: add pc, pc, r2, lsl #2 nop RET /* 0x00 */ b .Lmemcpy_bytewise /* 0x01 */ b .Lmemcpy_bytewise /* 0x02 */ b .Lmemcpy_bytewise /* 0x03 */ b .Lmemcpy_4 /* 0x04 */ b .Lmemcpy_bytewise /* 0x05 */ b .Lmemcpy_6 /* 0x06 */ b .Lmemcpy_bytewise /* 0x07 */ b .Lmemcpy_8 /* 0x08 */ b .Lmemcpy_bytewise /* 0x09 */ b .Lmemcpy_bytewise /* 0x0a */ b .Lmemcpy_bytewise /* 0x0b */ b .Lmemcpy_c /* 0x0c */ .Lmemcpy_bytewise: mov r3, r0 /* We must not clobber r0 */ ldrb ip, [r1], #0x01 1: subs r2, r2, #0x01 strb ip, [r3], #0x01 ldrbne ip, [r1], #0x01 bne 1b RET /****************************************************************************** * Special case for 4 byte copies */ #define LMEMCPY_4_LOG2 6 /* 64 bytes */ #define LMEMCPY_4_PAD .align LMEMCPY_4_LOG2 LMEMCPY_4_PAD .Lmemcpy_4: and r2, r1, #0x03 orr r2, r2, r0, lsl #2 ands r2, r2, #0x0f sub r3, pc, #0x14 addne pc, r3, r2, lsl #LMEMCPY_4_LOG2 /* * 0000: dst is 32-bit aligned, src is 32-bit aligned */ ldr r2, [r1] str r2, [r0] RET LMEMCPY_4_PAD /* * 0001: dst is 32-bit aligned, src is 8-bit aligned */ ldr r3, [r1, #-1] /* BE:r3 = x012 LE:r3 = 210x */ ldr r2, [r1, #3] /* BE:r2 = 3xxx LE:r2 = xxx3 */ #ifdef __ARMEB__ mov r3, r3, lsl #8 /* r3 = 012. */ orr r3, r3, r2, lsr #24 /* r3 = 0123 */ #else mov r3, r3, lsr #8 /* r3 = .210 */ orr r3, r3, r2, lsl #24 /* r3 = 3210 */ #endif str r3, [r0] RET LMEMCPY_4_PAD /* * 0010: dst is 32-bit aligned, src is 16-bit aligned */ #ifdef __ARMEB__ ldrh r3, [r1] ldrh r2, [r1, #0x02] #else ldrh r3, [r1, #0x02] ldrh r2, [r1] #endif orr r3, r2, r3, lsl #16 str r3, [r0] RET LMEMCPY_4_PAD /* * 0011: dst is 32-bit aligned, src is 8-bit aligned */ ldr r3, [r1, #-3] /* BE:r3 = xxx0 LE:r3 = 0xxx */ ldr r2, [r1, #1] /* BE:r2 = 123x LE:r2 = x321 */ #ifdef __ARMEB__ mov r3, r3, lsl #24 /* r3 = 0... */ orr r3, r3, r2, lsr #8 /* r3 = 0123 */ #else mov r3, r3, lsr #24 /* r3 = ...0 */ orr r3, r3, r2, lsl #8 /* r3 = 3210 */ #endif str r3, [r0] RET LMEMCPY_4_PAD /* * 0100: dst is 8-bit aligned, src is 32-bit aligned */ ldr r2, [r1] #ifdef __ARMEB__ strb r2, [r0, #0x03] mov r3, r2, lsr #8 mov r1, r2, lsr #24 strb r1, [r0] #else strb r2, [r0] mov r3, r2, lsr #8 mov r1, r2, lsr #24 strb r1, [r0, #0x03] #endif strh r3, [r0, #0x01] RET LMEMCPY_4_PAD /* * 0101: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldrb r1, [r1, #0x03] strb r2, [r0] strh r3, [r0, #0x01] strb r1, [r0, #0x03] RET LMEMCPY_4_PAD /* * 0110: dst is 8-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldrh r3, [r1, #0x02] /* LE:r3 = ..23 LE:r3 = ..32 */ #ifdef __ARMEB__ mov r1, r2, lsr #8 /* r1 = ...0 */ strb r1, [r0] mov r2, r2, lsl #8 /* r2 = .01. */ orr r2, r2, r3, lsr #8 /* r2 = .012 */ #else strb r2, [r0] mov r2, r2, lsr #8 /* r2 = ...1 */ orr r2, r2, r3, lsl #8 /* r2 = .321 */ mov r3, r3, lsr #8 /* r3 = ...3 */ #endif strh r2, [r0, #0x01] strb r3, [r0, #0x03] RET LMEMCPY_4_PAD /* * 0111: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldrb r1, [r1, #0x03] strb r2, [r0] strh r3, [r0, #0x01] strb r1, [r0, #0x03] RET LMEMCPY_4_PAD /* * 1000: dst is 16-bit aligned, src is 32-bit aligned */ ldr r2, [r1] #ifdef __ARMEB__ strh r2, [r0, #0x02] mov r3, r2, lsr #16 strh r3, [r0] #else strh r2, [r0] mov r3, r2, lsr #16 strh r3, [r0, #0x02] #endif RET LMEMCPY_4_PAD /* * 1001: dst is 16-bit aligned, src is 8-bit aligned */ ldr r2, [r1, #-1] /* BE:r2 = x012 LE:r2 = 210x */ ldr r3, [r1, #3] /* BE:r3 = 3xxx LE:r3 = xxx3 */ mov r1, r2, lsr #8 /* BE:r1 = .x01 LE:r1 = .210 */ strh r1, [r0] #ifdef __ARMEB__ mov r2, r2, lsl #8 /* r2 = 012. */ orr r2, r2, r3, lsr #24 /* r2 = 0123 */ #else mov r2, r2, lsr #24 /* r2 = ...2 */ orr r2, r2, r3, lsl #8 /* r2 = xx32 */ #endif strh r2, [r0, #0x02] RET LMEMCPY_4_PAD /* * 1010: dst is 16-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] ldrh r3, [r1, #0x02] strh r2, [r0] strh r3, [r0, #0x02] RET LMEMCPY_4_PAD /* * 1011: dst is 16-bit aligned, src is 8-bit aligned */ ldr r3, [r1, #1] /* BE:r3 = 123x LE:r3 = x321 */ ldr r2, [r1, #-3] /* BE:r2 = xxx0 LE:r2 = 0xxx */ mov r1, r3, lsr #8 /* BE:r1 = .123 LE:r1 = .x32 */ strh r1, [r0, #0x02] #ifdef __ARMEB__ mov r3, r3, lsr #24 /* r3 = ...1 */ orr r3, r3, r2, lsl #8 /* r3 = xx01 */ #else mov r3, r3, lsl #8 /* r3 = 321. */ orr r3, r3, r2, lsr #24 /* r3 = 3210 */ #endif strh r3, [r0] RET LMEMCPY_4_PAD /* * 1100: dst is 8-bit aligned, src is 32-bit aligned */ ldr r2, [r1] /* BE:r2 = 0123 LE:r2 = 3210 */ #ifdef __ARMEB__ strb r2, [r0, #0x03] mov r3, r2, lsr #8 mov r1, r2, lsr #24 strh r3, [r0, #0x01] strb r1, [r0] #else strb r2, [r0] mov r3, r2, lsr #8 mov r1, r2, lsr #24 strh r3, [r0, #0x01] strb r1, [r0, #0x03] #endif RET LMEMCPY_4_PAD /* * 1101: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldrb r1, [r1, #0x03] strb r2, [r0] strh r3, [r0, #0x01] strb r1, [r0, #0x03] RET LMEMCPY_4_PAD /* * 1110: dst is 8-bit aligned, src is 16-bit aligned */ #ifdef __ARMEB__ ldrh r3, [r1, #0x02] /* BE:r3 = ..23 LE:r3 = ..32 */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ strb r3, [r0, #0x03] mov r3, r3, lsr #8 /* r3 = ...2 */ orr r3, r3, r2, lsl #8 /* r3 = ..12 */ strh r3, [r0, #0x01] mov r2, r2, lsr #8 /* r2 = ...0 */ strb r2, [r0] #else ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldrh r3, [r1, #0x02] /* BE:r3 = ..23 LE:r3 = ..32 */ strb r2, [r0] mov r2, r2, lsr #8 /* r2 = ...1 */ orr r2, r2, r3, lsl #8 /* r2 = .321 */ strh r2, [r0, #0x01] mov r3, r3, lsr #8 /* r3 = ...3 */ strb r3, [r0, #0x03] #endif RET LMEMCPY_4_PAD /* * 1111: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldrb r1, [r1, #0x03] strb r2, [r0] strh r3, [r0, #0x01] strb r1, [r0, #0x03] RET LMEMCPY_4_PAD /****************************************************************************** * Special case for 6 byte copies */ #define LMEMCPY_6_LOG2 6 /* 64 bytes */ #define LMEMCPY_6_PAD .align LMEMCPY_6_LOG2 LMEMCPY_6_PAD .Lmemcpy_6: and r2, r1, #0x03 orr r2, r2, r0, lsl #2 ands r2, r2, #0x0f sub r3, pc, #0x14 addne pc, r3, r2, lsl #LMEMCPY_6_LOG2 /* * 0000: dst is 32-bit aligned, src is 32-bit aligned */ ldr r2, [r1] ldrh r3, [r1, #0x04] str r2, [r0] strh r3, [r0, #0x04] RET LMEMCPY_6_PAD /* * 0001: dst is 32-bit aligned, src is 8-bit aligned */ ldr r2, [r1, #-1] /* BE:r2 = x012 LE:r2 = 210x */ ldr r3, [r1, #0x03] /* BE:r3 = 345x LE:r3 = x543 */ #ifdef __ARMEB__ mov r2, r2, lsl #8 /* r2 = 012. */ orr r2, r2, r3, lsr #24 /* r2 = 0123 */ #else mov r2, r2, lsr #8 /* r2 = .210 */ orr r2, r2, r3, lsl #24 /* r2 = 3210 */ #endif mov r3, r3, lsr #8 /* BE:r3 = .345 LE:r3 = .x54 */ str r2, [r0] strh r3, [r0, #0x04] RET LMEMCPY_6_PAD /* * 0010: dst is 32-bit aligned, src is 16-bit aligned */ ldr r3, [r1, #0x02] /* BE:r3 = 2345 LE:r3 = 5432 */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ #ifdef __ARMEB__ mov r1, r3, lsr #16 /* r1 = ..23 */ orr r1, r1, r2, lsl #16 /* r1 = 0123 */ str r1, [r0] strh r3, [r0, #0x04] #else mov r1, r3, lsr #16 /* r1 = ..54 */ orr r2, r2, r3, lsl #16 /* r2 = 3210 */ str r2, [r0] strh r1, [r0, #0x04] #endif RET LMEMCPY_6_PAD /* * 0011: dst is 32-bit aligned, src is 8-bit aligned */ ldr r2, [r1, #-3] /* BE:r2 = xxx0 LE:r2 = 0xxx */ ldr r3, [r1, #1] /* BE:r3 = 1234 LE:r3 = 4321 */ ldr r1, [r1, #5] /* BE:r1 = 5xxx LE:r3 = xxx5 */ #ifdef __ARMEB__ mov r2, r2, lsl #24 /* r2 = 0... */ orr r2, r2, r3, lsr #8 /* r2 = 0123 */ mov r3, r3, lsl #8 /* r3 = 234. */ orr r1, r3, r1, lsr #24 /* r1 = 2345 */ #else mov r2, r2, lsr #24 /* r2 = ...0 */ orr r2, r2, r3, lsl #8 /* r2 = 3210 */ mov r1, r1, lsl #8 /* r1 = xx5. */ orr r1, r1, r3, lsr #24 /* r1 = xx54 */ #endif str r2, [r0] strh r1, [r0, #0x04] RET LMEMCPY_6_PAD /* * 0100: dst is 8-bit aligned, src is 32-bit aligned */ ldr r3, [r1] /* BE:r3 = 0123 LE:r3 = 3210 */ ldrh r2, [r1, #0x04] /* BE:r2 = ..45 LE:r2 = ..54 */ mov r1, r3, lsr #8 /* BE:r1 = .012 LE:r1 = .321 */ strh r1, [r0, #0x01] #ifdef __ARMEB__ mov r1, r3, lsr #24 /* r1 = ...0 */ strb r1, [r0] mov r3, r3, lsl #8 /* r3 = 123. */ orr r3, r3, r2, lsr #8 /* r3 = 1234 */ #else strb r3, [r0] mov r3, r3, lsr #24 /* r3 = ...3 */ orr r3, r3, r2, lsl #8 /* r3 = .543 */ mov r2, r2, lsr #8 /* r2 = ...5 */ #endif strh r3, [r0, #0x03] strb r2, [r0, #0x05] RET LMEMCPY_6_PAD /* * 0101: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldrh ip, [r1, #0x03] ldrb r1, [r1, #0x05] strb r2, [r0] strh r3, [r0, #0x01] strh ip, [r0, #0x03] strb r1, [r0, #0x05] RET LMEMCPY_6_PAD /* * 0110: dst is 8-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldr r1, [r1, #0x02] /* BE:r1 = 2345 LE:r1 = 5432 */ #ifdef __ARMEB__ mov r3, r2, lsr #8 /* r3 = ...0 */ strb r3, [r0] strb r1, [r0, #0x05] mov r3, r1, lsr #8 /* r3 = .234 */ strh r3, [r0, #0x03] mov r3, r2, lsl #8 /* r3 = .01. */ orr r3, r3, r1, lsr #24 /* r3 = .012 */ strh r3, [r0, #0x01] #else strb r2, [r0] mov r3, r1, lsr #24 strb r3, [r0, #0x05] mov r3, r1, lsr #8 /* r3 = .543 */ strh r3, [r0, #0x03] mov r3, r2, lsr #8 /* r3 = ...1 */ orr r3, r3, r1, lsl #8 /* r3 = 4321 */ strh r3, [r0, #0x01] #endif RET LMEMCPY_6_PAD /* * 0111: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldrh ip, [r1, #0x03] ldrb r1, [r1, #0x05] strb r2, [r0] strh r3, [r0, #0x01] strh ip, [r0, #0x03] strb r1, [r0, #0x05] RET LMEMCPY_6_PAD /* * 1000: dst is 16-bit aligned, src is 32-bit aligned */ #ifdef __ARMEB__ ldr r2, [r1] /* r2 = 0123 */ ldrh r3, [r1, #0x04] /* r3 = ..45 */ mov r1, r2, lsr #16 /* r1 = ..01 */ orr r3, r3, r2, lsl#16 /* r3 = 2345 */ strh r1, [r0] str r3, [r0, #0x02] #else ldrh r2, [r1, #0x04] /* r2 = ..54 */ ldr r3, [r1] /* r3 = 3210 */ mov r2, r2, lsl #16 /* r2 = 54.. */ orr r2, r2, r3, lsr #16 /* r2 = 5432 */ strh r3, [r0] str r2, [r0, #0x02] #endif RET LMEMCPY_6_PAD /* * 1001: dst is 16-bit aligned, src is 8-bit aligned */ ldr r3, [r1, #-1] /* BE:r3 = x012 LE:r3 = 210x */ ldr r2, [r1, #3] /* BE:r2 = 345x LE:r2 = x543 */ mov r1, r3, lsr #8 /* BE:r1 = .x01 LE:r1 = .210 */ #ifdef __ARMEB__ mov r2, r2, lsr #8 /* r2 = .345 */ orr r2, r2, r3, lsl #24 /* r2 = 2345 */ #else mov r2, r2, lsl #8 /* r2 = 543. */ orr r2, r2, r3, lsr #24 /* r2 = 5432 */ #endif strh r1, [r0] str r2, [r0, #0x02] RET LMEMCPY_6_PAD /* * 1010: dst is 16-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] ldr r3, [r1, #0x02] strh r2, [r0] str r3, [r0, #0x02] RET LMEMCPY_6_PAD /* * 1011: dst is 16-bit aligned, src is 8-bit aligned */ ldrb r3, [r1] /* r3 = ...0 */ ldr r2, [r1, #0x01] /* BE:r2 = 1234 LE:r2 = 4321 */ ldrb r1, [r1, #0x05] /* r1 = ...5 */ #ifdef __ARMEB__ mov r3, r3, lsl #8 /* r3 = ..0. */ orr r3, r3, r2, lsr #24 /* r3 = ..01 */ orr r1, r1, r2, lsl #8 /* r1 = 2345 */ #else orr r3, r3, r2, lsl #8 /* r3 = 3210 */ mov r1, r1, lsl #24 /* r1 = 5... */ orr r1, r1, r2, lsr #8 /* r1 = 5432 */ #endif strh r3, [r0] str r1, [r0, #0x02] RET LMEMCPY_6_PAD /* * 1100: dst is 8-bit aligned, src is 32-bit aligned */ ldr r2, [r1] /* BE:r2 = 0123 LE:r2 = 3210 */ ldrh r1, [r1, #0x04] /* BE:r1 = ..45 LE:r1 = ..54 */ #ifdef __ARMEB__ mov r3, r2, lsr #24 /* r3 = ...0 */ strb r3, [r0] mov r2, r2, lsl #8 /* r2 = 123. */ orr r2, r2, r1, lsr #8 /* r2 = 1234 */ #else strb r2, [r0] mov r2, r2, lsr #8 /* r2 = .321 */ orr r2, r2, r1, lsl #24 /* r2 = 4321 */ mov r1, r1, lsr #8 /* r1 = ...5 */ #endif str r2, [r0, #0x01] strb r1, [r0, #0x05] RET LMEMCPY_6_PAD /* * 1101: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldrh ip, [r1, #0x03] ldrb r1, [r1, #0x05] strb r2, [r0] strh r3, [r0, #0x01] strh ip, [r0, #0x03] strb r1, [r0, #0x05] RET LMEMCPY_6_PAD /* * 1110: dst is 8-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldr r1, [r1, #0x02] /* BE:r1 = 2345 LE:r1 = 5432 */ #ifdef __ARMEB__ mov r3, r2, lsr #8 /* r3 = ...0 */ strb r3, [r0] mov r2, r2, lsl #24 /* r2 = 1... */ orr r2, r2, r1, lsr #8 /* r2 = 1234 */ #else strb r2, [r0] mov r2, r2, lsr #8 /* r2 = ...1 */ orr r2, r2, r1, lsl #8 /* r2 = 4321 */ mov r1, r1, lsr #24 /* r1 = ...5 */ #endif str r2, [r0, #0x01] strb r1, [r0, #0x05] RET LMEMCPY_6_PAD /* * 1111: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldr r3, [r1, #0x01] ldrb r1, [r1, #0x05] strb r2, [r0] str r3, [r0, #0x01] strb r1, [r0, #0x05] RET LMEMCPY_6_PAD /****************************************************************************** * Special case for 8 byte copies */ #define LMEMCPY_8_LOG2 6 /* 64 bytes */ #define LMEMCPY_8_PAD .align LMEMCPY_8_LOG2 LMEMCPY_8_PAD .Lmemcpy_8: and r2, r1, #0x03 orr r2, r2, r0, lsl #2 ands r2, r2, #0x0f sub r3, pc, #0x14 addne pc, r3, r2, lsl #LMEMCPY_8_LOG2 /* * 0000: dst is 32-bit aligned, src is 32-bit aligned */ ldr r2, [r1] ldr r3, [r1, #0x04] str r2, [r0] str r3, [r0, #0x04] RET LMEMCPY_8_PAD /* * 0001: dst is 32-bit aligned, src is 8-bit aligned */ ldr r3, [r1, #-1] /* BE:r3 = x012 LE:r3 = 210x */ ldr r2, [r1, #0x03] /* BE:r2 = 3456 LE:r2 = 6543 */ ldrb r1, [r1, #0x07] /* r1 = ...7 */ #ifdef __ARMEB__ mov r3, r3, lsl #8 /* r3 = 012. */ orr r3, r3, r2, lsr #24 /* r3 = 0123 */ orr r2, r1, r2, lsl #8 /* r2 = 4567 */ #else mov r3, r3, lsr #8 /* r3 = .210 */ orr r3, r3, r2, lsl #24 /* r3 = 3210 */ mov r1, r1, lsl #24 /* r1 = 7... */ orr r2, r1, r2, lsr #8 /* r2 = 7654 */ #endif str r3, [r0] str r2, [r0, #0x04] RET LMEMCPY_8_PAD /* * 0010: dst is 32-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldr r3, [r1, #0x02] /* BE:r3 = 2345 LE:r3 = 5432 */ ldrh r1, [r1, #0x06] /* BE:r1 = ..67 LE:r1 = ..76 */ #ifdef __ARMEB__ mov r2, r2, lsl #16 /* r2 = 01.. */ orr r2, r2, r3, lsr #16 /* r2 = 0123 */ orr r3, r1, r3, lsl #16 /* r3 = 4567 */ #else orr r2, r2, r3, lsl #16 /* r2 = 3210 */ mov r3, r3, lsr #16 /* r3 = ..54 */ orr r3, r3, r1, lsl #16 /* r3 = 7654 */ #endif str r2, [r0] str r3, [r0, #0x04] RET LMEMCPY_8_PAD /* * 0011: dst is 32-bit aligned, src is 8-bit aligned */ ldrb r3, [r1] /* r3 = ...0 */ ldr r2, [r1, #0x01] /* BE:r2 = 1234 LE:r2 = 4321 */ ldr r1, [r1, #0x05] /* BE:r1 = 567x LE:r1 = x765 */ #ifdef __ARMEB__ mov r3, r3, lsl #24 /* r3 = 0... */ orr r3, r3, r2, lsr #8 /* r3 = 0123 */ mov r2, r2, lsl #24 /* r2 = 4... */ orr r2, r2, r1, lsr #8 /* r2 = 4567 */ #else orr r3, r3, r2, lsl #8 /* r3 = 3210 */ mov r2, r2, lsr #24 /* r2 = ...4 */ orr r2, r2, r1, lsl #8 /* r2 = 7654 */ #endif str r3, [r0] str r2, [r0, #0x04] RET LMEMCPY_8_PAD /* * 0100: dst is 8-bit aligned, src is 32-bit aligned */ ldr r3, [r1] /* BE:r3 = 0123 LE:r3 = 3210 */ ldr r2, [r1, #0x04] /* BE:r2 = 4567 LE:r2 = 7654 */ #ifdef __ARMEB__ mov r1, r3, lsr #24 /* r1 = ...0 */ strb r1, [r0] mov r1, r3, lsr #8 /* r1 = .012 */ strb r2, [r0, #0x07] mov r3, r3, lsl #24 /* r3 = 3... */ orr r3, r3, r2, lsr #8 /* r3 = 3456 */ #else strb r3, [r0] mov r1, r2, lsr #24 /* r1 = ...7 */ strb r1, [r0, #0x07] mov r1, r3, lsr #8 /* r1 = .321 */ mov r3, r3, lsr #24 /* r3 = ...3 */ orr r3, r3, r2, lsl #8 /* r3 = 6543 */ #endif strh r1, [r0, #0x01] str r3, [r0, #0x03] RET LMEMCPY_8_PAD /* * 0101: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldr ip, [r1, #0x03] ldrb r1, [r1, #0x07] strb r2, [r0] strh r3, [r0, #0x01] str ip, [r0, #0x03] strb r1, [r0, #0x07] RET LMEMCPY_8_PAD /* * 0110: dst is 8-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldr r3, [r1, #0x02] /* BE:r3 = 2345 LE:r3 = 5432 */ ldrh r1, [r1, #0x06] /* BE:r1 = ..67 LE:r1 = ..76 */ #ifdef __ARMEB__ mov ip, r2, lsr #8 /* ip = ...0 */ strb ip, [r0] mov ip, r2, lsl #8 /* ip = .01. */ orr ip, ip, r3, lsr #24 /* ip = .012 */ strb r1, [r0, #0x07] mov r3, r3, lsl #8 /* r3 = 345. */ orr r3, r3, r1, lsr #8 /* r3 = 3456 */ #else strb r2, [r0] /* 0 */ mov ip, r1, lsr #8 /* ip = ...7 */ strb ip, [r0, #0x07] /* 7 */ mov ip, r2, lsr #8 /* ip = ...1 */ orr ip, ip, r3, lsl #8 /* ip = 4321 */ mov r3, r3, lsr #8 /* r3 = .543 */ orr r3, r3, r1, lsl #24 /* r3 = 6543 */ #endif strh ip, [r0, #0x01] str r3, [r0, #0x03] RET LMEMCPY_8_PAD /* * 0111: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r3, [r1] /* r3 = ...0 */ ldr ip, [r1, #0x01] /* BE:ip = 1234 LE:ip = 4321 */ ldrh r2, [r1, #0x05] /* BE:r2 = ..56 LE:r2 = ..65 */ ldrb r1, [r1, #0x07] /* r1 = ...7 */ strb r3, [r0] mov r3, ip, lsr #16 /* BE:r3 = ..12 LE:r3 = ..43 */ #ifdef __ARMEB__ strh r3, [r0, #0x01] orr r2, r2, ip, lsl #16 /* r2 = 3456 */ #else strh ip, [r0, #0x01] orr r2, r3, r2, lsl #16 /* r2 = 6543 */ #endif str r2, [r0, #0x03] strb r1, [r0, #0x07] RET LMEMCPY_8_PAD /* * 1000: dst is 16-bit aligned, src is 32-bit aligned */ ldr r2, [r1] /* BE:r2 = 0123 LE:r2 = 3210 */ ldr r3, [r1, #0x04] /* BE:r3 = 4567 LE:r3 = 7654 */ mov r1, r2, lsr #16 /* BE:r1 = ..01 LE:r1 = ..32 */ #ifdef __ARMEB__ strh r1, [r0] mov r1, r3, lsr #16 /* r1 = ..45 */ orr r2, r1 ,r2, lsl #16 /* r2 = 2345 */ #else strh r2, [r0] orr r2, r1, r3, lsl #16 /* r2 = 5432 */ mov r3, r3, lsr #16 /* r3 = ..76 */ #endif str r2, [r0, #0x02] strh r3, [r0, #0x06] RET LMEMCPY_8_PAD /* * 1001: dst is 16-bit aligned, src is 8-bit aligned */ ldr r2, [r1, #-1] /* BE:r2 = x012 LE:r2 = 210x */ ldr r3, [r1, #0x03] /* BE:r3 = 3456 LE:r3 = 6543 */ ldrb ip, [r1, #0x07] /* ip = ...7 */ mov r1, r2, lsr #8 /* BE:r1 = .x01 LE:r1 = .210 */ strh r1, [r0] #ifdef __ARMEB__ mov r1, r2, lsl #24 /* r1 = 2... */ orr r1, r1, r3, lsr #8 /* r1 = 2345 */ orr r3, ip, r3, lsl #8 /* r3 = 4567 */ #else mov r1, r2, lsr #24 /* r1 = ...2 */ orr r1, r1, r3, lsl #8 /* r1 = 5432 */ mov r3, r3, lsr #24 /* r3 = ...6 */ orr r3, r3, ip, lsl #8 /* r3 = ..76 */ #endif str r1, [r0, #0x02] strh r3, [r0, #0x06] RET LMEMCPY_8_PAD /* * 1010: dst is 16-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] ldr ip, [r1, #0x02] ldrh r3, [r1, #0x06] strh r2, [r0] str ip, [r0, #0x02] strh r3, [r0, #0x06] RET LMEMCPY_8_PAD /* * 1011: dst is 16-bit aligned, src is 8-bit aligned */ ldr r3, [r1, #0x05] /* BE:r3 = 567x LE:r3 = x765 */ ldr r2, [r1, #0x01] /* BE:r2 = 1234 LE:r2 = 4321 */ ldrb ip, [r1] /* ip = ...0 */ mov r1, r3, lsr #8 /* BE:r1 = .567 LE:r1 = .x76 */ strh r1, [r0, #0x06] #ifdef __ARMEB__ mov r3, r3, lsr #24 /* r3 = ...5 */ orr r3, r3, r2, lsl #8 /* r3 = 2345 */ mov r2, r2, lsr #24 /* r2 = ...1 */ orr r2, r2, ip, lsl #8 /* r2 = ..01 */ #else mov r3, r3, lsl #24 /* r3 = 5... */ orr r3, r3, r2, lsr #8 /* r3 = 5432 */ orr r2, ip, r2, lsl #8 /* r2 = 3210 */ #endif str r3, [r0, #0x02] strh r2, [r0] RET LMEMCPY_8_PAD /* * 1100: dst is 8-bit aligned, src is 32-bit aligned */ ldr r3, [r1, #0x04] /* BE:r3 = 4567 LE:r3 = 7654 */ ldr r2, [r1] /* BE:r2 = 0123 LE:r2 = 3210 */ mov r1, r3, lsr #8 /* BE:r1 = .456 LE:r1 = .765 */ strh r1, [r0, #0x05] #ifdef __ARMEB__ strb r3, [r0, #0x07] mov r1, r2, lsr #24 /* r1 = ...0 */ strb r1, [r0] mov r2, r2, lsl #8 /* r2 = 123. */ orr r2, r2, r3, lsr #24 /* r2 = 1234 */ str r2, [r0, #0x01] #else strb r2, [r0] mov r1, r3, lsr #24 /* r1 = ...7 */ strb r1, [r0, #0x07] mov r2, r2, lsr #8 /* r2 = .321 */ orr r2, r2, r3, lsl #24 /* r2 = 4321 */ str r2, [r0, #0x01] #endif RET LMEMCPY_8_PAD /* * 1101: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r3, [r1] /* r3 = ...0 */ ldrh r2, [r1, #0x01] /* BE:r2 = ..12 LE:r2 = ..21 */ ldr ip, [r1, #0x03] /* BE:ip = 3456 LE:ip = 6543 */ ldrb r1, [r1, #0x07] /* r1 = ...7 */ strb r3, [r0] mov r3, ip, lsr #16 /* BE:r3 = ..34 LE:r3 = ..65 */ #ifdef __ARMEB__ strh ip, [r0, #0x05] orr r2, r3, r2, lsl #16 /* r2 = 1234 */ #else strh r3, [r0, #0x05] orr r2, r2, ip, lsl #16 /* r2 = 4321 */ #endif str r2, [r0, #0x01] strb r1, [r0, #0x07] RET LMEMCPY_8_PAD /* * 1110: dst is 8-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldr r3, [r1, #0x02] /* BE:r3 = 2345 LE:r3 = 5432 */ ldrh r1, [r1, #0x06] /* BE:r1 = ..67 LE:r1 = ..76 */ #ifdef __ARMEB__ mov ip, r2, lsr #8 /* ip = ...0 */ strb ip, [r0] mov ip, r2, lsl #24 /* ip = 1... */ orr ip, ip, r3, lsr #8 /* ip = 1234 */ strb r1, [r0, #0x07] mov r1, r1, lsr #8 /* r1 = ...6 */ orr r1, r1, r3, lsl #8 /* r1 = 3456 */ #else strb r2, [r0] mov ip, r2, lsr #8 /* ip = ...1 */ orr ip, ip, r3, lsl #8 /* ip = 4321 */ mov r2, r1, lsr #8 /* r2 = ...7 */ strb r2, [r0, #0x07] mov r1, r1, lsl #8 /* r1 = .76. */ orr r1, r1, r3, lsr #24 /* r1 = .765 */ #endif str ip, [r0, #0x01] strh r1, [r0, #0x05] RET LMEMCPY_8_PAD /* * 1111: dst is 8-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] ldr ip, [r1, #0x01] ldrh r3, [r1, #0x05] ldrb r1, [r1, #0x07] strb r2, [r0] str ip, [r0, #0x01] strh r3, [r0, #0x05] strb r1, [r0, #0x07] RET LMEMCPY_8_PAD /****************************************************************************** * Special case for 12 byte copies */ #define LMEMCPY_C_LOG2 7 /* 128 bytes */ #define LMEMCPY_C_PAD .align LMEMCPY_C_LOG2 LMEMCPY_C_PAD .Lmemcpy_c: and r2, r1, #0x03 orr r2, r2, r0, lsl #2 ands r2, r2, #0x0f sub r3, pc, #0x14 addne pc, r3, r2, lsl #LMEMCPY_C_LOG2 /* * 0000: dst is 32-bit aligned, src is 32-bit aligned */ ldr r2, [r1] ldr r3, [r1, #0x04] ldr r1, [r1, #0x08] str r2, [r0] str r3, [r0, #0x04] str r1, [r0, #0x08] RET LMEMCPY_C_PAD /* * 0001: dst is 32-bit aligned, src is 8-bit aligned */ ldrb r2, [r1, #0xb] /* r2 = ...B */ ldr ip, [r1, #0x07] /* BE:ip = 789A LE:ip = A987 */ ldr r3, [r1, #0x03] /* BE:r3 = 3456 LE:r3 = 6543 */ ldr r1, [r1, #-1] /* BE:r1 = x012 LE:r1 = 210x */ #ifdef __ARMEB__ orr r2, r2, ip, lsl #8 /* r2 = 89AB */ str r2, [r0, #0x08] mov r2, ip, lsr #24 /* r2 = ...7 */ orr r2, r2, r3, lsl #8 /* r2 = 4567 */ mov r1, r1, lsl #8 /* r1 = 012. */ orr r1, r1, r3, lsr #24 /* r1 = 0123 */ #else mov r2, r2, lsl #24 /* r2 = B... */ orr r2, r2, ip, lsr #8 /* r2 = BA98 */ str r2, [r0, #0x08] mov r2, ip, lsl #24 /* r2 = 7... */ orr r2, r2, r3, lsr #8 /* r2 = 7654 */ mov r1, r1, lsr #8 /* r1 = .210 */ orr r1, r1, r3, lsl #24 /* r1 = 3210 */ #endif str r2, [r0, #0x04] str r1, [r0] RET LMEMCPY_C_PAD /* * 0010: dst is 32-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldr r3, [r1, #0x02] /* BE:r3 = 2345 LE:r3 = 5432 */ ldr ip, [r1, #0x06] /* BE:ip = 6789 LE:ip = 9876 */ ldrh r1, [r1, #0x0a] /* BE:r1 = ..AB LE:r1 = ..BA */ #ifdef __ARMEB__ mov r2, r2, lsl #16 /* r2 = 01.. */ orr r2, r2, r3, lsr #16 /* r2 = 0123 */ str r2, [r0] mov r3, r3, lsl #16 /* r3 = 45.. */ orr r3, r3, ip, lsr #16 /* r3 = 4567 */ orr r1, r1, ip, lsl #16 /* r1 = 89AB */ #else orr r2, r2, r3, lsl #16 /* r2 = 3210 */ str r2, [r0] mov r3, r3, lsr #16 /* r3 = ..54 */ orr r3, r3, ip, lsl #16 /* r3 = 7654 */ mov r1, r1, lsl #16 /* r1 = BA.. */ orr r1, r1, ip, lsr #16 /* r1 = BA98 */ #endif str r3, [r0, #0x04] str r1, [r0, #0x08] RET LMEMCPY_C_PAD /* * 0011: dst is 32-bit aligned, src is 8-bit aligned */ ldrb r2, [r1] /* r2 = ...0 */ ldr r3, [r1, #0x01] /* BE:r3 = 1234 LE:r3 = 4321 */ ldr ip, [r1, #0x05] /* BE:ip = 5678 LE:ip = 8765 */ ldr r1, [r1, #0x09] /* BE:r1 = 9ABx LE:r1 = xBA9 */ #ifdef __ARMEB__ mov r2, r2, lsl #24 /* r2 = 0... */ orr r2, r2, r3, lsr #8 /* r2 = 0123 */ str r2, [r0] mov r3, r3, lsl #24 /* r3 = 4... */ orr r3, r3, ip, lsr #8 /* r3 = 4567 */ mov r1, r1, lsr #8 /* r1 = .9AB */ orr r1, r1, ip, lsl #24 /* r1 = 89AB */ #else orr r2, r2, r3, lsl #8 /* r2 = 3210 */ str r2, [r0] mov r3, r3, lsr #24 /* r3 = ...4 */ orr r3, r3, ip, lsl #8 /* r3 = 7654 */ mov r1, r1, lsl #8 /* r1 = BA9. */ orr r1, r1, ip, lsr #24 /* r1 = BA98 */ #endif str r3, [r0, #0x04] str r1, [r0, #0x08] RET LMEMCPY_C_PAD /* * 0100: dst is 8-bit aligned (byte 1), src is 32-bit aligned */ ldr r2, [r1] /* BE:r2 = 0123 LE:r2 = 3210 */ ldr r3, [r1, #0x04] /* BE:r3 = 4567 LE:r3 = 7654 */ ldr ip, [r1, #0x08] /* BE:ip = 89AB LE:ip = BA98 */ mov r1, r2, lsr #8 /* BE:r1 = .012 LE:r1 = .321 */ strh r1, [r0, #0x01] #ifdef __ARMEB__ mov r1, r2, lsr #24 /* r1 = ...0 */ strb r1, [r0] mov r1, r2, lsl #24 /* r1 = 3... */ orr r2, r1, r3, lsr #8 /* r1 = 3456 */ mov r1, r3, lsl #24 /* r1 = 7... */ orr r1, r1, ip, lsr #8 /* r1 = 789A */ #else strb r2, [r0] mov r1, r2, lsr #24 /* r1 = ...3 */ orr r2, r1, r3, lsl #8 /* r1 = 6543 */ mov r1, r3, lsr #24 /* r1 = ...7 */ orr r1, r1, ip, lsl #8 /* r1 = A987 */ mov ip, ip, lsr #24 /* ip = ...B */ #endif str r2, [r0, #0x03] str r1, [r0, #0x07] strb ip, [r0, #0x0b] RET LMEMCPY_C_PAD /* * 0101: dst is 8-bit aligned (byte 1), src is 8-bit aligned (byte 1) */ ldrb r2, [r1] ldrh r3, [r1, #0x01] ldr ip, [r1, #0x03] strb r2, [r0] ldr r2, [r1, #0x07] ldrb r1, [r1, #0x0b] strh r3, [r0, #0x01] str ip, [r0, #0x03] str r2, [r0, #0x07] strb r1, [r0, #0x0b] RET LMEMCPY_C_PAD /* * 0110: dst is 8-bit aligned (byte 1), src is 16-bit aligned */ ldrh r2, [r1] /* BE:r2 = ..01 LE:r2 = ..10 */ ldr r3, [r1, #0x02] /* BE:r3 = 2345 LE:r3 = 5432 */ ldr ip, [r1, #0x06] /* BE:ip = 6789 LE:ip = 9876 */ ldrh r1, [r1, #0x0a] /* BE:r1 = ..AB LE:r1 = ..BA */ #ifdef __ARMEB__ mov r2, r2, ror #8 /* r2 = 1..0 */ strb r2, [r0] mov r2, r2, lsr #16 /* r2 = ..1. */ orr r2, r2, r3, lsr #24 /* r2 = ..12 */ strh r2, [r0, #0x01] mov r2, r3, lsl #8 /* r2 = 345. */ orr r3, r2, ip, lsr #24 /* r3 = 3456 */ mov r2, ip, lsl #8 /* r2 = 789. */ orr r2, r2, r1, lsr #8 /* r2 = 789A */ #else strb r2, [r0] mov r2, r2, lsr #8 /* r2 = ...1 */ orr r2, r2, r3, lsl #8 /* r2 = 4321 */ strh r2, [r0, #0x01] mov r2, r3, lsr #8 /* r2 = .543 */ orr r3, r2, ip, lsl #24 /* r3 = 6543 */ mov r2, ip, lsr #8 /* r2 = .987 */ orr r2, r2, r1, lsl #24 /* r2 = A987 */ mov r1, r1, lsr #8 /* r1 = ...B */ #endif str r3, [r0, #0x03] str r2, [r0, #0x07] strb r1, [r0, #0x0b] RET LMEMCPY_C_PAD /* * 0111: dst is 8-bit aligned (byte 1), src is 8-bit aligned (byte 3) */ ldrb r2, [r1] ldr r3, [r1, #0x01] /* BE:r3 = 1234 LE:r3 = 4321 */ ldr ip, [r1, #0x05] /* BE:ip = 5678 LE:ip = 8765 */ ldr r1, [r1, #0x09] /* BE:r1 = 9ABx LE:r1 = xBA9 */ strb r2, [r0] #ifdef __ARMEB__ mov r2, r3, lsr #16 /* r2 = ..12 */ strh r2, [r0, #0x01] mov r3, r3, lsl #16 /* r3 = 34.. */ orr r3, r3, ip, lsr #16 /* r3 = 3456 */ mov ip, ip, lsl #16 /* ip = 78.. */ orr ip, ip, r1, lsr #16 /* ip = 789A */ mov r1, r1, lsr #8 /* r1 = .9AB */ #else strh r3, [r0, #0x01] mov r3, r3, lsr #16 /* r3 = ..43 */ orr r3, r3, ip, lsl #16 /* r3 = 6543 */ mov ip, ip, lsr #16 /* ip = ..87 */ orr ip, ip, r1, lsl #16 /* ip = A987 */ mov r1, r1, lsr #16 /* r1 = ..xB */ #endif str r3, [r0, #0x03] str ip, [r0, #0x07] strb r1, [r0, #0x0b] RET LMEMCPY_C_PAD /* * 1000: dst is 16-bit aligned, src is 32-bit aligned */ ldr ip, [r1] /* BE:ip = 0123 LE:ip = 3210 */ ldr r3, [r1, #0x04] /* BE:r3 = 4567 LE:r3 = 7654 */ ldr r2, [r1, #0x08] /* BE:r2 = 89AB LE:r2 = BA98 */ mov r1, ip, lsr #16 /* BE:r1 = ..01 LE:r1 = ..32 */ #ifdef __ARMEB__ strh r1, [r0] mov r1, ip, lsl #16 /* r1 = 23.. */ orr r1, r1, r3, lsr #16 /* r1 = 2345 */ mov r3, r3, lsl #16 /* r3 = 67.. */ orr r3, r3, r2, lsr #16 /* r3 = 6789 */ #else strh ip, [r0] orr r1, r1, r3, lsl #16 /* r1 = 5432 */ mov r3, r3, lsr #16 /* r3 = ..76 */ orr r3, r3, r2, lsl #16 /* r3 = 9876 */ mov r2, r2, lsr #16 /* r2 = ..BA */ #endif str r1, [r0, #0x02] str r3, [r0, #0x06] strh r2, [r0, #0x0a] RET LMEMCPY_C_PAD /* * 1001: dst is 16-bit aligned, src is 8-bit aligned (byte 1) */ ldr r2, [r1, #-1] /* BE:r2 = x012 LE:r2 = 210x */ ldr r3, [r1, #0x03] /* BE:r3 = 3456 LE:r3 = 6543 */ mov ip, r2, lsr #8 /* BE:ip = .x01 LE:ip = .210 */ strh ip, [r0] ldr ip, [r1, #0x07] /* BE:ip = 789A LE:ip = A987 */ ldrb r1, [r1, #0x0b] /* r1 = ...B */ #ifdef __ARMEB__ mov r2, r2, lsl #24 /* r2 = 2... */ orr r2, r2, r3, lsr #8 /* r2 = 2345 */ mov r3, r3, lsl #24 /* r3 = 6... */ orr r3, r3, ip, lsr #8 /* r3 = 6789 */ orr r1, r1, ip, lsl #8 /* r1 = 89AB */ #else mov r2, r2, lsr #24 /* r2 = ...2 */ orr r2, r2, r3, lsl #8 /* r2 = 5432 */ mov r3, r3, lsr #24 /* r3 = ...6 */ orr r3, r3, ip, lsl #8 /* r3 = 9876 */ mov r1, r1, lsl #8 /* r1 = ..B. */ orr r1, r1, ip, lsr #24 /* r1 = ..BA */ #endif str r2, [r0, #0x02] str r3, [r0, #0x06] strh r1, [r0, #0x0a] RET LMEMCPY_C_PAD /* * 1010: dst is 16-bit aligned, src is 16-bit aligned */ ldrh r2, [r1] ldr r3, [r1, #0x02] ldr ip, [r1, #0x06] ldrh r1, [r1, #0x0a] strh r2, [r0] str r3, [r0, #0x02] str ip, [r0, #0x06] strh r1, [r0, #0x0a] RET LMEMCPY_C_PAD /* * 1011: dst is 16-bit aligned, src is 8-bit aligned (byte 3) */ ldr r2, [r1, #0x09] /* BE:r2 = 9ABx LE:r2 = xBA9 */ ldr r3, [r1, #0x05] /* BE:r3 = 5678 LE:r3 = 8765 */ mov ip, r2, lsr #8 /* BE:ip = .9AB LE:ip = .xBA */ strh ip, [r0, #0x0a] ldr ip, [r1, #0x01] /* BE:ip = 1234 LE:ip = 4321 */ ldrb r1, [r1] /* r1 = ...0 */ #ifdef __ARMEB__ mov r2, r2, lsr #24 /* r2 = ...9 */ orr r2, r2, r3, lsl #8 /* r2 = 6789 */ mov r3, r3, lsr #24 /* r3 = ...5 */ orr r3, r3, ip, lsl #8 /* r3 = 2345 */ mov r1, r1, lsl #8 /* r1 = ..0. */ orr r1, r1, ip, lsr #24 /* r1 = ..01 */ #else mov r2, r2, lsl #24 /* r2 = 9... */ orr r2, r2, r3, lsr #8 /* r2 = 9876 */ mov r3, r3, lsl #24 /* r3 = 5... */ orr r3, r3, ip, lsr #8 /* r3 = 5432 */ orr r1, r1, ip, lsl #8 /* r1 = 3210 */ #endif str r2, [r0, #0x06] str r3, [r0, #0x02] strh r1, [r0] RET LMEMCPY_C_PAD /* * 1100: dst is 8-bit aligned (byte 3), src is 32-bit aligned */ ldr r2, [r1] /* BE:r2 = 0123 LE:r2 = 3210 */ ldr ip, [r1, #0x04] /* BE:ip = 4567 LE:ip = 7654 */ ldr r1, [r1, #0x08] /* BE:r1 = 89AB LE:r1 = BA98 */ #ifdef __ARMEB__ mov r3, r2, lsr #24 /* r3 = ...0 */ strb r3, [r0] mov r2, r2, lsl #8 /* r2 = 123. */ orr r2, r2, ip, lsr #24 /* r2 = 1234 */ str r2, [r0, #0x01] mov r2, ip, lsl #8 /* r2 = 567. */ orr r2, r2, r1, lsr #24 /* r2 = 5678 */ str r2, [r0, #0x05] mov r2, r1, lsr #8 /* r2 = ..9A */ strh r2, [r0, #0x09] strb r1, [r0, #0x0b] #else strb r2, [r0] mov r3, r2, lsr #8 /* r3 = .321 */ orr r3, r3, ip, lsl #24 /* r3 = 4321 */ str r3, [r0, #0x01] mov r3, ip, lsr #8 /* r3 = .765 */ orr r3, r3, r1, lsl #24 /* r3 = 8765 */ str r3, [r0, #0x05] mov r1, r1, lsr #8 /* r1 = .BA9 */ strh r1, [r0, #0x09] mov r1, r1, lsr #16 /* r1 = ...B */ strb r1, [r0, #0x0b] #endif RET LMEMCPY_C_PAD /* * 1101: dst is 8-bit aligned (byte 3), src is 8-bit aligned (byte 1) */ ldrb r2, [r1, #0x0b] /* r2 = ...B */ ldr r3, [r1, #0x07] /* BE:r3 = 789A LE:r3 = A987 */ ldr ip, [r1, #0x03] /* BE:ip = 3456 LE:ip = 6543 */ ldr r1, [r1, #-1] /* BE:r1 = x012 LE:r1 = 210x */ strb r2, [r0, #0x0b] #ifdef __ARMEB__ strh r3, [r0, #0x09] mov r3, r3, lsr #16 /* r3 = ..78 */ orr r3, r3, ip, lsl #16 /* r3 = 5678 */ mov ip, ip, lsr #16 /* ip = ..34 */ orr ip, ip, r1, lsl #16 /* ip = 1234 */ mov r1, r1, lsr #16 /* r1 = ..x0 */ #else mov r2, r3, lsr #16 /* r2 = ..A9 */ strh r2, [r0, #0x09] mov r3, r3, lsl #16 /* r3 = 87.. */ orr r3, r3, ip, lsr #16 /* r3 = 8765 */ mov ip, ip, lsl #16 /* ip = 43.. */ orr ip, ip, r1, lsr #16 /* ip = 4321 */ mov r1, r1, lsr #8 /* r1 = .210 */ #endif str r3, [r0, #0x05] str ip, [r0, #0x01] strb r1, [r0] RET LMEMCPY_C_PAD /* * 1110: dst is 8-bit aligned (byte 3), src is 16-bit aligned */ #ifdef __ARMEB__ ldrh r2, [r1, #0x0a] /* r2 = ..AB */ ldr ip, [r1, #0x06] /* ip = 6789 */ ldr r3, [r1, #0x02] /* r3 = 2345 */ ldrh r1, [r1] /* r1 = ..01 */ strb r2, [r0, #0x0b] mov r2, r2, lsr #8 /* r2 = ...A */ orr r2, r2, ip, lsl #8 /* r2 = 789A */ mov ip, ip, lsr #8 /* ip = .678 */ orr ip, ip, r3, lsl #24 /* ip = 5678 */ mov r3, r3, lsr #8 /* r3 = .234 */ orr r3, r3, r1, lsl #24 /* r3 = 1234 */ mov r1, r1, lsr #8 /* r1 = ...0 */ strb r1, [r0] str r3, [r0, #0x01] str ip, [r0, #0x05] strh r2, [r0, #0x09] #else ldrh r2, [r1] /* r2 = ..10 */ ldr r3, [r1, #0x02] /* r3 = 5432 */ ldr ip, [r1, #0x06] /* ip = 9876 */ ldrh r1, [r1, #0x0a] /* r1 = ..BA */ strb r2, [r0] mov r2, r2, lsr #8 /* r2 = ...1 */ orr r2, r2, r3, lsl #8 /* r2 = 4321 */ mov r3, r3, lsr #24 /* r3 = ...5 */ orr r3, r3, ip, lsl #8 /* r3 = 8765 */ mov ip, ip, lsr #24 /* ip = ...9 */ orr ip, ip, r1, lsl #8 /* ip = .BA9 */ mov r1, r1, lsr #8 /* r1 = ...B */ str r2, [r0, #0x01] str r3, [r0, #0x05] strh ip, [r0, #0x09] strb r1, [r0, #0x0b] #endif RET LMEMCPY_C_PAD /* * 1111: dst is 8-bit aligned (byte 3), src is 8-bit aligned (byte 3) */ ldrb r2, [r1] ldr r3, [r1, #0x01] ldr ip, [r1, #0x05] strb r2, [r0] ldrh r2, [r1, #0x09] ldrb r1, [r1, #0x0b] str r3, [r0, #0x01] str ip, [r0, #0x05] strh r2, [r0, #0x09] strb r1, [r0, #0x0b] RET END(memcpy) #endif /* _ARM_ARCH_5E */ #ifdef GPROF ENTRY(user) nop END(user) ENTRY(btrap) nop END(btrap) ENTRY(etrap) nop END(etrap) ENTRY(bintr) nop END(bintr) ENTRY(eintr) nop END(eintr) #endif Index: head/sys/arm/arm/swtch-v4.S =================================================================== --- head/sys/arm/arm/swtch-v4.S (revision 331253) +++ head/sys/arm/arm/swtch-v4.S (revision 331254) @@ -1,376 +1,376 @@ /* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ /*- * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpuswitch.S * * cpu switching functions * * Created : 15/10/94 * */ -#include "assym.s" +#include "assym.inc" #include "opt_sched.h" #include #include #include #include __FBSDID("$FreeBSD$"); #define GET_PCPU(tmp, tmp2) \ ldr tmp, .Lcurpcpu #ifdef VFP .fpu vfp /* allow VFP instructions */ #endif .Lcurpcpu: .word _C_LABEL(__pcpu) .Lblocked_lock: .word _C_LABEL(blocked_lock) #define DOMAIN_CLIENT 0x01 .Lcpufuncs: .word _C_LABEL(cpufuncs) /* * cpu_throw(oldtd, newtd) * * Remove current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd */ ENTRY(cpu_throw) mov r5, r1 /* * r0 = oldtd * r5 = newtd */ #ifdef VFP /* This thread is dying, disable */ bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ #endif GET_PCPU(r7, r9) ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */ /* Switch to lwp0 context */ ldr r9, .Lcpufuncs mov lr, pc ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] ldr r0, [r7, #(PCB_PL1VEC)] ldr r1, [r7, #(PCB_DACR)] /* * r0 = Pointer to L1 slot for vector_page (or NULL) * r1 = lwp0's DACR * r5 = lwp0 * r7 = lwp0's PCB * r9 = cpufuncs */ /* * Ensure the vector table is accessible by fixing up lwp0's L1 */ cmp r0, #0 /* No need to fixup vector table? */ ldrne r3, [r0] /* But if yes, fetch current value */ ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ cmpne r3, r2 /* Stuffing the same value? */ strne r2, [r0] /* Store if not. */ #ifdef PMAP_INCLUDE_PTE_SYNC /* * Need to sync the cache to make sure that last store is * visible to the MMU. */ movne r1, #4 movne lr, pc ldrne pc, [r9, #CF_DCACHE_WB_RANGE] #endif /* PMAP_INCLUDE_PTE_SYNC */ /* * Note: We don't do the same optimisation as cpu_switch() with * respect to avoiding flushing the TLB if we're switching to * the same L1 since this process' VM space may be about to go * away, so we don't want *any* turds left in the TLB. */ /* Switch the memory to the new process */ ldr r0, [r7, #(PCB_PAGEDIR)] mov lr, pc ldr pc, [r9, #CF_CONTEXT_SWITCH] GET_PCPU(r6, r4) /* Hook in a new pcb */ str r7, [r6, #PC_CURPCB] /* We have a new curthread now so make a note it */ str r5, [r6, #PC_CURTHREAD] /* Set the new tp */ ldr r6, [r5, #(TD_MD + MD_TP)] ldr r4, =ARM_TP_ADDRESS str r6, [r4] ldr r6, [r5, #(TD_MD + MD_RAS_START)] str r6, [r4, #4] /* ARM_RAS_START */ ldr r6, [r5, #(TD_MD + MD_RAS_END)] str r6, [r4, #8] /* ARM_RAS_END */ /* Restore all the saved registers and exit */ add r3, r7, #PCB_R4 ldmia r3, {r4-r12, sp, pc} END(cpu_throw) /* * cpu_switch(oldtd, newtd, lock) * * Save the current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd * r2 = lock (new lock for old thread) */ ENTRY(cpu_switch) /* Interrupts are disabled. */ /* Save all the registers in the old thread's pcb. */ ldr r3, [r0, #(TD_PCB)] /* Restore all the saved registers and exit */ add r3, #(PCB_R4) stmia r3, {r4-r12, sp, lr, pc} mov r6, r2 /* Save the mutex */ /* rem: r0 = old lwp */ /* rem: interrupts are disabled */ /* Process is now on a processor. */ /* We have a new curthread now so make a note it */ GET_PCPU(r7, r2) str r1, [r7, #PC_CURTHREAD] /* Hook in a new pcb */ ldr r2, [r1, #TD_PCB] str r2, [r7, #PC_CURPCB] /* Stage two : Save old context */ /* Get the user structure for the old thread. */ ldr r2, [r0, #(TD_PCB)] mov r4, r0 /* Save the old thread. */ /* Store the old tp; userland can change it on armv4. */ ldr r3, =ARM_TP_ADDRESS ldr r9, [r3] str r9, [r0, #(TD_MD + MD_TP)] ldr r9, [r3, #4] str r9, [r0, #(TD_MD + MD_RAS_START)] ldr r9, [r3, #8] str r9, [r0, #(TD_MD + MD_RAS_END)] /* Set the new tp */ ldr r9, [r1, #(TD_MD + MD_TP)] str r9, [r3] ldr r9, [r1, #(TD_MD + MD_RAS_START)] str r9, [r3, #4] ldr r9, [r1, #(TD_MD + MD_RAS_END)] str r9, [r3, #8] /* Get the user structure for the new process in r9 */ ldr r9, [r1, #(TD_PCB)] /* rem: r2 = old PCB */ /* rem: r9 = new PCB */ /* rem: interrupts are enabled */ #ifdef VFP fmrx r0, fpexc /* If the VFP is enabled */ tst r0, #(VFPEXC_EN) /* the current thread has */ movne r1, #1 /* used it, so go save */ addne r0, r2, #(PCB_VFPSTATE) /* the state into the PCB */ blne _C_LABEL(vfp_store) /* and disable the VFP. */ #endif /* r0-r3 now free! */ /* Third phase : restore saved context */ /* rem: r2 = old PCB */ /* rem: r9 = new PCB */ ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */ mov r2, #DOMAIN_CLIENT cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ beq .Lcs_context_switched /* Yup. Don't flush cache */ mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */ /* * Get the new L1 table pointer into r11. If we're switching to * an LWP with the same address space as the outgoing one, we can * skip the cache purge and the TTB load. * * To avoid data dep stalls that would happen anyway, we try * and get some useful work done in the mean time. */ mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ teq r10, r11 /* Same L1? */ cmpeq r0, r5 /* Same DACR? */ beq .Lcs_context_switched /* yes! */ /* * Definitely need to flush the cache. */ ldr r1, .Lcpufuncs mov lr, pc ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] .Lcs_cache_purge_skipped: /* rem: r6 = lock */ /* rem: r9 = new PCB */ /* rem: r10 = old L1 */ /* rem: r11 = new L1 */ mov r2, #0x00000000 ldr r7, [r9, #(PCB_PL1VEC)] /* * Ensure the vector table is accessible by fixing up the L1 */ cmp r7, #0 /* No need to fixup vector table? */ ldrne r2, [r7] /* But if yes, fetch current value */ ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */ cmpne r2, r0 /* Stuffing the same value? */ #ifndef PMAP_INCLUDE_PTE_SYNC strne r0, [r7] /* Nope, update it */ #else beq .Lcs_same_vector str r0, [r7] /* Otherwise, update it */ /* * Need to sync the cache to make sure that last store is * visible to the MMU. */ ldr r2, .Lcpufuncs mov r0, r7 mov r1, #4 mov lr, pc ldr pc, [r2, #CF_DCACHE_WB_RANGE] .Lcs_same_vector: #endif /* PMAP_INCLUDE_PTE_SYNC */ cmp r10, r11 /* Switching to the same L1? */ ldr r10, .Lcpufuncs beq .Lcs_same_l1 /* Yup. */ /* * Do a full context switch, including full TLB flush. */ mov r0, r11 mov lr, pc ldr pc, [r10, #CF_CONTEXT_SWITCH] b .Lcs_context_switched /* * We're switching to a different process in the same L1. * In this situation, we only need to flush the TLB for the * vector_page mapping, and even then only if r7 is non-NULL. */ .Lcs_same_l1: cmp r7, #0 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ movne lr, pc ldrne pc, [r10, #CF_TLB_FLUSHID_SE] .Lcs_context_switched: /* Release the old thread */ str r6, [r4, #TD_LOCK] /* XXXSCW: Safe to re-enable FIQs here */ /* rem: r9 = new PCB */ /* Restore all the saved registers and exit */ add r3, r9, #PCB_R4 ldmia r3, {r4-r12, sp, pc} END(cpu_switch) Index: head/sys/arm/arm/swtch-v6.S =================================================================== --- head/sys/arm/arm/swtch-v6.S (revision 331253) +++ head/sys/arm/arm/swtch-v6.S (revision 331254) @@ -1,508 +1,508 @@ /* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ /*- * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpuswitch.S * * cpu switching functions * * Created : 15/10/94 * */ -#include "assym.s" +#include "assym.inc" #include "opt_sched.h" #include #include #include #include #include __FBSDID("$FreeBSD$"); #if defined(SMP) #define GET_PCPU(tmp, tmp2) \ mrc CP15_MPIDR(tmp); \ and tmp, tmp, #0xf; \ ldr tmp2, .Lcurpcpu+4; \ mul tmp, tmp, tmp2; \ ldr tmp2, .Lcurpcpu; \ add tmp, tmp, tmp2; #else #define GET_PCPU(tmp, tmp2) \ ldr tmp, .Lcurpcpu #endif #ifdef VFP .fpu vfp /* allow VFP instructions */ #endif .Lcurpcpu: .word _C_LABEL(__pcpu) .word PCPU_SIZE .Lblocked_lock: .word _C_LABEL(blocked_lock) ENTRY(cpu_context_switch) DSB /* * We can directly switch between translation tables only when the * size of the mapping for any given virtual address is the same * in the old and new translation tables. * Thus, we must switch to kernel pmap translation table as * intermediate mapping because all sizes of these mappings are same * (or unmapped). The same is true for switch from kernel pmap * translation table to new pmap one. */ mov r2, #(CPU_ASID_KERNEL) ldr r1, =(_C_LABEL(pmap_kern_ttb)) ldr r1, [r1] mcr CP15_TTBR0(r1) /* switch to kernel TTB */ ISB mcr CP15_TLBIASID(r2) /* flush not global TLBs */ DSB mcr CP15_TTBR0(r0) /* switch to new TTB */ ISB /* * We must flush not global TLBs again because PT2MAP mapping * is different. */ mcr CP15_TLBIASID(r2) /* flush not global TLBs */ /* * Flush entire Branch Target Cache because of the branch predictor * is not architecturally invisible. See ARM Architecture Reference * Manual ARMv7-A and ARMv7-R edition, page B2-1264(65), Branch * predictors and Requirements for branch predictor maintenance * operations sections. */ /* * Additionally, to mitigate mistrained branch predictor attack * we must invalidate it on affected CPUs. Unfortunately, BPIALL * is effectively NOP on Cortex-A15 so it needs special treatment. */ ldr r0, [r8, #PC_BP_HARDEN_KIND] cmp r0, #PCPU_BP_HARDEN_KIND_ICIALLU mcrne CP15_BPIALL /* Flush entire Branch Target Cache */ mcreq CP15_ICIALLU /* This is the only way how to flush */ /* Branch Target Cache on Cortex-A15. */ DSB mov pc, lr END(cpu_context_switch) /* * cpu_throw(oldtd, newtd) * * Remove current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd */ ENTRY(cpu_throw) mov r10, r0 /* r10 = oldtd */ mov r11, r1 /* r11 = newtd */ #ifdef VFP /* This thread is dying, disable */ bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ #endif GET_PCPU(r8, r9) /* r8 = current pcpu */ ldr r4, [r8, #PC_CPUID] /* r4 = current cpu id */ cmp r10, #0 /* old thread? */ beq 2f /* no, skip */ /* Remove this CPU from the active list. */ ldr r5, [r8, #PC_CURPMAP] mov r0, #(PM_ACTIVE) add r5, r0 /* r5 = old pm_active */ /* Compute position and mask. */ #if _NCPUWORDS > 1 lsr r0, r4, #3 bic r0, #3 add r5, r0 /* r5 = position in old pm_active */ mov r2, #1 and r0, r4, #31 lsl r2, r0 /* r2 = mask */ #else mov r2, #1 lsl r2, r4 /* r2 = mask */ #endif /* Clear cpu from old active list. */ #ifdef SMP 1: ldrex r0, [r5] bic r0, r2 strex r1, r0, [r5] teq r1, #0 bne 1b #else ldr r0, [r5] bic r0, r2 str r0, [r5] #endif 2: #ifdef INVARIANTS cmp r11, #0 /* new thread? */ beq badsw1 /* no, panic */ #endif ldr r7, [r11, #(TD_PCB)] /* r7 = new PCB */ /* * Registers at this point * r4 = current cpu id * r7 = new PCB * r8 = current pcpu * r11 = newtd */ /* MMU switch to new thread. */ ldr r0, [r7, #(PCB_PAGEDIR)] #ifdef INVARIANTS cmp r0, #0 /* new thread? */ beq badsw4 /* no, panic */ #endif bl _C_LABEL(cpu_context_switch) /* * Set new PMAP as current one. * Insert cpu to new active list. */ ldr r6, [r11, #(TD_PROC)] /* newtd->proc */ ldr r6, [r6, #(P_VMSPACE)] /* newtd->proc->vmspace */ add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */ str r6, [r8, #PC_CURPMAP] /* store to curpmap */ mov r0, #PM_ACTIVE add r6, r0 /* r6 = new pm_active */ /* compute position and mask */ #if _NCPUWORDS > 1 lsr r0, r4, #3 bic r0, #3 add r6, r0 /* r6 = position in new pm_active */ mov r2, #1 and r0, r4, #31 lsl r2, r0 /* r2 = mask */ #else mov r2, #1 lsl r2, r4 /* r2 = mask */ #endif /* Set cpu to new active list. */ #ifdef SMP 1: ldrex r0, [r6] orr r0, r2 strex r1, r0, [r6] teq r1, #0 bne 1b #else ldr r0, [r6] orr r0, r2 str r0, [r6] #endif /* * Registers at this point. * r7 = new PCB * r8 = current pcpu * r11 = newtd * They must match the ones in sw1 position !!! */ DMB b sw1 /* share new thread init with cpu_switch() */ END(cpu_throw) /* * cpu_switch(oldtd, newtd, lock) * * Save the current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd * r2 = lock (new lock for old thread) */ ENTRY(cpu_switch) /* Interrupts are disabled. */ #ifdef INVARIANTS cmp r0, #0 /* old thread? */ beq badsw2 /* no, panic */ #endif /* Save all the registers in the old thread's pcb. */ ldr r3, [r0, #(TD_PCB)] add r3, #(PCB_R4) stmia r3, {r4-r12, sp, lr, pc} mrc CP15_TPIDRURW(r4) str r4, [r3, #(PCB_TPIDRURW - PCB_R4)] #ifdef INVARIANTS cmp r1, #0 /* new thread? */ beq badsw3 /* no, panic */ #endif /* * Save arguments. Note that we can now use r0-r14 until * it is time to restore them for the new thread. However, * some registers are not safe over function call. */ mov r9, r2 /* r9 = lock */ mov r10, r0 /* r10 = oldtd */ mov r11, r1 /* r11 = newtd */ GET_PCPU(r8, r3) /* r8 = current PCPU */ ldr r7, [r11, #(TD_PCB)] /* r7 = newtd->td_pcb */ #ifdef VFP ldr r3, [r10, #(TD_PCB)] fmrx r0, fpexc /* If the VFP is enabled */ tst r0, #(VFPEXC_EN) /* the current thread has */ movne r1, #1 /* used it, so go save */ addne r0, r3, #(PCB_VFPSTATE) /* the state into the PCB */ blne _C_LABEL(vfp_store) /* and disable the VFP. */ #endif /* * MMU switch. If we're switching to a thread with the same * address space as the outgoing one, we can skip the MMU switch. */ mrc CP15_TTBR0(r1) /* r1 = old TTB */ ldr r0, [r7, #(PCB_PAGEDIR)] /* r0 = new TTB */ cmp r0, r1 /* Switching to the TTB? */ beq sw0 /* same TTB, skip */ #ifdef INVARIANTS cmp r0, #0 /* new thread? */ beq badsw4 /* no, panic */ #endif bl cpu_context_switch /* new TTB as argument */ /* * Registers at this point * r7 = new PCB * r8 = current pcpu * r9 = lock * r10 = oldtd * r11 = newtd */ /* * Set new PMAP as current one. * Update active list on PMAPs. */ ldr r6, [r11, #TD_PROC] /* newtd->proc */ ldr r6, [r6, #P_VMSPACE] /* newtd->proc->vmspace */ add r6, #VM_PMAP /* newtd->proc->vmspace->pmap */ ldr r5, [r8, #PC_CURPMAP] /* get old curpmap */ str r6, [r8, #PC_CURPMAP] /* and save new one */ mov r0, #PM_ACTIVE add r5, r0 /* r5 = old pm_active */ add r6, r0 /* r6 = new pm_active */ /* Compute position and mask. */ ldr r4, [r8, #PC_CPUID] #if _NCPUWORDS > 1 lsr r0, r4, #3 bic r0, #3 add r5, r0 /* r5 = position in old pm_active */ add r6, r0 /* r6 = position in new pm_active */ mov r2, #1 and r0, r4, #31 lsl r2, r0 /* r2 = mask */ #else mov r2, #1 lsl r2, r4 /* r2 = mask */ #endif /* Clear cpu from old active list. */ #ifdef SMP 1: ldrex r0, [r5] bic r0, r2 strex r1, r0, [r5] teq r1, #0 bne 1b #else ldr r0, [r5] bic r0, r2 str r0, [r5] #endif /* Set cpu to new active list. */ #ifdef SMP 1: ldrex r0, [r6] orr r0, r2 strex r1, r0, [r6] teq r1, #0 bne 1b #else ldr r0, [r6] orr r0, r2 str r0, [r6] #endif sw0: /* * Registers at this point * r7 = new PCB * r8 = current pcpu * r9 = lock * r10 = oldtd * r11 = newtd */ /* Change the old thread lock. */ add r5, r10, #TD_LOCK DMB 1: ldrex r0, [r5] strex r1, r9, [r5] teq r1, #0 bne 1b DMB sw1: clrex /* * Registers at this point * r7 = new PCB * r8 = current pcpu * r11 = newtd */ #if defined(SMP) && defined(SCHED_ULE) /* * 386 and amd64 do the blocked lock test only for SMP and SCHED_ULE * QQQ: What does it mean in reality and why is it done? */ ldr r6, =blocked_lock 1: ldr r3, [r11, #TD_LOCK] /* atomic write regular read */ cmp r3, r6 beq 1b #endif /* We have a new curthread now so make a note it */ str r11, [r8, #PC_CURTHREAD] mcr CP15_TPIDRPRW(r11) /* store pcb in per cpu structure */ str r7, [r8, #PC_CURPCB] /* * Restore all saved registers and return. Note that some saved * registers can be changed when either cpu_fork(), cpu_copy_thread(), * cpu_fork_kthread_handler(), or makectx() was called. * * The value of TPIDRURW is also written into TPIDRURO, as * userspace still uses TPIDRURO, modifying it through * sysarch(ARM_SET_TP, addr). */ ldr r3, [r7, #PCB_TPIDRURW] mcr CP15_TPIDRURW(r3) /* write tls thread reg 2 */ mcr CP15_TPIDRURO(r3) /* write tls thread reg 3 */ add r3, r7, #PCB_R4 ldmia r3, {r4-r12, sp, pc} #ifdef INVARIANTS badsw1: ldr r0, =sw1_panic_str bl _C_LABEL(panic) 1: nop b 1b badsw2: ldr r0, =sw2_panic_str bl _C_LABEL(panic) 1: nop b 1b badsw3: ldr r0, =sw3_panic_str bl _C_LABEL(panic) 1: nop b 1b badsw4: ldr r0, =sw4_panic_str bl _C_LABEL(panic) 1: nop b 1b sw1_panic_str: .asciz "cpu_throw: no newthread supplied.\n" sw2_panic_str: .asciz "cpu_switch: no curthread supplied.\n" sw3_panic_str: .asciz "cpu_switch: no newthread supplied.\n" sw4_panic_str: .asciz "cpu_switch: new pagedir is NULL.\n" #endif END(cpu_switch) Index: head/sys/arm/arm/swtch.S =================================================================== --- head/sys/arm/arm/swtch.S (revision 331253) +++ head/sys/arm/arm/swtch.S (revision 331254) @@ -1,121 +1,121 @@ /* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ /*- * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpuswitch.S * * cpu switching functions * * Created : 15/10/94 * */ -#include "assym.s" +#include "assym.inc" #include #include #include #include __FBSDID("$FreeBSD$"); #ifdef VFP .fpu vfp /* allow VFP instructions */ #endif ENTRY(savectx) stmfd sp!, {lr} sub sp, sp, #4 /* Store all the registers in the thread's pcb */ add r3, r0, #(PCB_R4) stmia r3, {r4-r12, sp, lr, pc} #ifdef VFP fmrx r2, fpexc /* If the VFP is enabled */ tst r2, #(VFPEXC_EN) /* the current thread has */ movne r1, #1 /* used it, so go save */ addne r0, r0, #(PCB_VFPSTATE) /* the state into the PCB */ blne _C_LABEL(vfp_store) /* and disable the VFP. */ #endif add sp, sp, #4; ldmfd sp!, {pc} END(savectx) ENTRY(fork_trampoline) STOP_UNWINDING /* EABI: Don't unwind beyond the thread enty point. */ mov fp, #0 /* OABI: Stack traceback via fp stops here. */ mov r2, sp mov r1, r5 mov r0, r4 ldr lr, =swi_exit /* Go finish forking, then return */ b _C_LABEL(fork_exit) /* to userland via swi_exit code. */ END(fork_trampoline) Index: head/sys/arm64/arm64/bzero.S =================================================================== --- head/sys/arm64/arm64/bzero.S (revision 331253) +++ head/sys/arm64/arm64/bzero.S (revision 331254) @@ -1,206 +1,206 @@ /*- * Copyright (C) 2016 Cavium Inc. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); -#include "assym.s" +#include "assym.inc" /* * void bzero(void *p, size_t size) * * x0 - p * x1 - size */ ENTRY(bzero) cbz x1, ending /* * x5 is number of cache lines to zero - calculated later and * will become non-zero if buffer is long enough to zero by * cache lines (and if it is allowed.) * We need to zero it before proceeding with buffers of size * smaller than 16 bytes - otherwise the x5 will not be * calculated and will retain random value. * "normal" is used for buffers <= 16 bytes and to align buffer * to cache line for buffers bigger than cache line; non-0 x5 * after "normal" has completed indicates that it has been used * to align buffer to cache line and now zero by cache lines will * be performed, and x5 is amount of cache lines to loop through. */ mov x5, xzr /* No use of cache assisted zero for buffers with size <= 16 */ cmp x1, #0x10 b.le normal /* * Load size of line that will be cleaned by dc zva call. * 0 means that the instruction is not allowed */ ldr x7, =dczva_line_size ldr x7, [x7] cbz x7, normal /* * Buffer must be larger than cache line for using cache zeroing * (and cache line aligned but this is checked after jump) */ cmp x1, x7 b.lt normal /* * Calculate number of bytes to cache aligned address (x4) nad * number of full cache lines (x5). x6 is final address to zero. */ sub x2, x7, #0x01 mov x3, -1 eor x3, x3, x2 add x4, x0, x2 and x4, x4, x3 subs x4, x4, x0 b.eq normal /* Calculate number of "lines" in buffer */ sub x5, x1, x4 rbit x2, x7 clz x2, x2 lsr x5, x5, x2 /* * If number of cache lines is 0, we will not be able to zero * by cache lines, so go normal way. */ cbz x5, normal /* x6 is final address to zero */ add x6, x0, x1 /* * We are here because x5 is non-0 so normal will be used to * align buffer before cache zeroing. x4 holds number of bytes * needed for alignment. */ mov x1, x4 /* When jumping here: x0 holds pointer, x1 holds size */ normal: /* * Get buffer offset into 16 byte aligned address; 0 means pointer * is aligned. */ ands x2, x0, #0x0f b.eq aligned_to_16 /* Calculate one-byte loop runs to 8 byte aligned address. */ ands x2, x2, #0x07 mov x3, #0x08 sub x2, x3, x2 /* x2 is number of bytes missing for alignment, x1 is buffer size */ cmp x1, x2 csel x2, x1, x2, le sub x1, x1, x2 /* * Byte by byte copy will copy at least enough bytes to align * pointer and at most "size". */ align: strb wzr, [x0], #0x01 subs x2, x2, #0x01 b.ne align /* Now pointer is aligned to 8 bytes */ cmp x1, #0x10 b.lt lead_out /* * Check if copy of another 8 bytes is needed to align to 16 byte * address and do it */ tbz x0, #0x03, aligned_to_16 str xzr, [x0], #0x08 sub x1, x1, #0x08 /* While jumping here: x0 is 16 byte alligned address, x1 is size */ aligned_to_16: /* If size is less than 16 bytes, use lead_out to copy what remains */ cmp x1, #0x10 b.lt lead_out lsr x2, x1, #0x04 zero_by_16: stp xzr, xzr, [x0], #0x10 subs x2, x2, #0x01 b.ne zero_by_16 /* * Lead out requires addresses to be aligned to 8 bytes. It is used to * zero buffers with sizes < 16 and what can not be zeroed by * zero_by_16 loop. */ ands x1, x1, #0x0f b.eq lead_out_end lead_out: tbz x1, #0x03, lead_out_dword str xzr, [x0], #0x08 lead_out_dword: tbz x1, #0x02, lead_out_word str wzr, [x0], #0x04 lead_out_word: tbz x1, #0x01, lead_out_byte strh wzr, [x0], #0x02 lead_out_byte: tbz x1, #0x00, lead_out_end strb wzr, [x0], #0x01 lead_out_end: /* * If x5 is non-zero, this means that normal has been used as * a lead in to align buffer address to cache size */ cbz x5, ending /* * Here x5 holds number of lines to zero; x6 is final address of * buffer. x0 is cache line aligned pointer. x7 is cache line size * in bytes */ cache_line_zero: dc zva, x0 add x0, x0, x7 subs x5, x5, #0x01 b.ne cache_line_zero /* Need to zero remaining bytes? */ subs x1, x6, x0 b.ne normal ending: ret END(bzero) Index: head/sys/arm64/arm64/copyinout.S =================================================================== --- head/sys/arm64/arm64/copyinout.S (revision 331253) +++ head/sys/arm64/arm64/copyinout.S (revision 331254) @@ -1,226 +1,226 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include -#include "assym.s" +#include "assym.inc" /* * Fault handler for the copy{in,out} functions below. */ ENTRY(copyio_fault) SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */ EXIT_USER_ACCESS_CHECK(w0, x1) copyio_fault_nopcb: mov x0, #EFAULT ret END(copyio_fault) /* * Copies from a kernel to user address * * int copyout(const void *kaddr, void *udaddr, size_t len) */ ENTRY(copyout) cbz x2, 1f adds x3, x1, x2 b.cs copyio_fault_nopcb ldr x4, =VM_MAXUSER_ADDRESS cmp x3, x4 b.hi copyio_fault_nopcb b copycommon 1: mov x0, xzr /* return 0 */ ret END(copyout) /* * Copies from a user to kernel address * * int copyin(const void *uaddr, void *kdaddr, size_t len) */ ENTRY(copyin) cbz x2, 1f adds x3, x0, x2 b.cs copyio_fault_nopcb ldr x4, =VM_MAXUSER_ADDRESS cmp x3, x4 b.hi copyio_fault_nopcb b copycommon 1: mov x0, xzr /* return 0 */ ret END(copyin) /* * Copies a string from a user to kernel address * * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) */ ENTRY(copyinstr) mov x5, xzr /* count = 0 */ mov w4, #1 /* If zero return faulure */ cbz x2, 3f /* If len == 0 then skip loop */ adr x6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(x6, x7) /* Set the handler */ ENTER_USER_ACCESS(w6, x7) ldr x7, =VM_MAXUSER_ADDRESS 1: cmp x0, x7 b.cs copyio_fault ldtrb w4, [x0] /* Load from uaddr */ add x0, x0, #1 /* Next char */ strb w4, [x1], #1 /* Store in kaddr */ add x5, x5, #1 /* count++ */ cbz w4, 2f /* Break when NUL-terminated */ sub x2, x2, #1 /* len-- */ cbnz x2, 1b 2: EXIT_USER_ACCESS(w6) SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 3: cbz x3, 4f /* Check if done != NULL */ str x5, [x3] /* done = count */ 4: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */ cmp w4, #0 /* Check if we saved the NUL-terminator */ csel w0, wzr, w1, eq /* If so return success, else failure */ ret END(copyinstr) /* * Local helper * * x0 - src pointer * x1 - dst pointer * x2 - size * lr - the return address, so jump here instead of calling * * This function is optimized to minimize concurrent memory accesses. In * present form it is suited for cores with a single memory prefetching * unit. * ARM64TODO: * Consider using separate functions for each ARM64 core. Adding memory * access interleaving might increase a total throughput on A57 or A72. */ .text .align 4 .local copycommon .type copycommon,@function copycommon: adr x6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(x6, x7) /* Set the handler */ ENTER_USER_ACCESS(w6, x7) /* Check alignment */ orr x3, x0, x1 ands x3, x3, 0x07 b.eq aligned /* Unaligned is byte by byte copy */ byte_by_byte: ldrb w3, [x0], #0x01 strb w3, [x1], #0x01 subs x2, x2, #0x01 b.ne byte_by_byte b ending aligned: cmp x2, #0x10 b.lt lead_out cmp x2, #0x40 b.lt by_dwords_start /* Block copy */ lsr x15, x2, #0x06 by_blocks: ldp x3, x4, [x0], #0x10 ldp x5, x6, [x0], #0x10 ldp x7, x8, [x0], #0x10 ldp x9, x10, [x0], #0x10 stp x3, x4, [x1], #0x10 stp x5, x6, [x1], #0x10 stp x7, x8, [x1], #0x10 stp x9, x10, [x1], #0x10 subs x15, x15, #0x01 b.ne by_blocks and x2, x2, #0x3f by_dwords_start: lsr x15, x2, #0x04 cbz x15, lead_out by_dwords: ldp x3, x4, [x0], #0x10 stp x3, x4, [x1], #0x10 subs x15, x15, #0x01 b.ne by_dwords /* Less than 16 bytes to copy */ lead_out: tbz x2, #0x03, last_word ldr x3, [x0], #0x08 str x3, [x1], #0x08 last_word: tbz x2, #0x02, last_hword ldr w3, [x0], #0x04 str w3, [x1], #0x04 last_hword: tbz x2, #0x01, last_byte ldrh w3, [x0], #0x02 strh w3, [x1], #0x02 last_byte: tbz x2, #0x00, ending ldrb w3, [x0] strb w3, [x1] ending: EXIT_USER_ACCESS_CHECK(w6, x7) SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ mov x0, xzr /* return 0 */ ret .size copycommon, . - copycommon Index: head/sys/arm64/arm64/exception.S =================================================================== --- head/sys/arm64/arm64/exception.S (revision 331253) +++ head/sys/arm64/arm64/exception.S (revision 331254) @@ -1,229 +1,229 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include __FBSDID("$FreeBSD$"); -#include "assym.s" +#include "assym.inc" .text .macro save_registers el .if \el == 1 mov x18, sp sub sp, sp, #128 .endif sub sp, sp, #(TF_SIZE + 16) stp x29, x30, [sp, #(TF_SIZE)] stp x28, x29, [sp, #(TF_X + 28 * 8)] stp x26, x27, [sp, #(TF_X + 26 * 8)] stp x24, x25, [sp, #(TF_X + 24 * 8)] stp x22, x23, [sp, #(TF_X + 22 * 8)] stp x20, x21, [sp, #(TF_X + 20 * 8)] stp x18, x19, [sp, #(TF_X + 18 * 8)] stp x16, x17, [sp, #(TF_X + 16 * 8)] stp x14, x15, [sp, #(TF_X + 14 * 8)] stp x12, x13, [sp, #(TF_X + 12 * 8)] stp x10, x11, [sp, #(TF_X + 10 * 8)] stp x8, x9, [sp, #(TF_X + 8 * 8)] stp x6, x7, [sp, #(TF_X + 6 * 8)] stp x4, x5, [sp, #(TF_X + 4 * 8)] stp x2, x3, [sp, #(TF_X + 2 * 8)] stp x0, x1, [sp, #(TF_X + 0 * 8)] mrs x10, elr_el1 mrs x11, spsr_el1 mrs x12, esr_el1 .if \el == 0 mrs x18, sp_el0 .endif str x10, [sp, #(TF_ELR)] stp w11, w12, [sp, #(TF_SPSR)] stp x18, lr, [sp, #(TF_SP)] mrs x18, tpidr_el1 add x29, sp, #(TF_SIZE) .endm .macro restore_registers el .if \el == 1 msr daifset, #2 /* * Disable interrupts, x18 may change in the interrupt exception * handler. For EL0 exceptions, do_ast already did this. */ .endif ldp x18, lr, [sp, #(TF_SP)] ldp x10, x11, [sp, #(TF_ELR)] .if \el == 0 msr sp_el0, x18 .endif msr spsr_el1, x11 msr elr_el1, x10 ldp x0, x1, [sp, #(TF_X + 0 * 8)] ldp x2, x3, [sp, #(TF_X + 2 * 8)] ldp x4, x5, [sp, #(TF_X + 4 * 8)] ldp x6, x7, [sp, #(TF_X + 6 * 8)] ldp x8, x9, [sp, #(TF_X + 8 * 8)] ldp x10, x11, [sp, #(TF_X + 10 * 8)] ldp x12, x13, [sp, #(TF_X + 12 * 8)] ldp x14, x15, [sp, #(TF_X + 14 * 8)] ldp x16, x17, [sp, #(TF_X + 16 * 8)] .if \el == 0 /* * We only restore the callee saved registers when returning to * userland as they may have been updated by a system call or signal. */ ldp x18, x19, [sp, #(TF_X + 18 * 8)] ldp x20, x21, [sp, #(TF_X + 20 * 8)] ldp x22, x23, [sp, #(TF_X + 22 * 8)] ldp x24, x25, [sp, #(TF_X + 24 * 8)] ldp x26, x27, [sp, #(TF_X + 26 * 8)] ldp x28, x29, [sp, #(TF_X + 28 * 8)] .else ldr x29, [sp, #(TF_X + 29 * 8)] .endif .if \el == 0 add sp, sp, #(TF_SIZE + 16) .else mov sp, x18 mrs x18, tpidr_el1 .endif .endm .macro do_ast mrs x19, daif /* Make sure the IRQs are enabled before calling ast() */ bic x19, x19, #PSR_I 1: /* Disable interrupts */ msr daifset, #2 /* Read the current thread flags */ ldr x1, [x18, #PC_CURTHREAD] /* Load curthread */ ldr x2, [x1, #TD_FLAGS] /* Check if we have either bits set */ mov x3, #((TDF_ASTPENDING|TDF_NEEDRESCHED) >> 8) lsl x3, x3, #8 and x2, x2, x3 cbz x2, 2f /* Restore interrupts */ msr daif, x19 /* handle the ast */ mov x0, sp bl _C_LABEL(ast) /* Re-check for new ast scheduled */ b 1b 2: .endm ENTRY(handle_el1h_sync) save_registers 1 ldr x0, [x18, #PC_CURTHREAD] mov x1, sp bl do_el1h_sync restore_registers 1 eret END(handle_el1h_sync) ENTRY(handle_el1h_irq) save_registers 1 mov x0, sp bl intr_irq_handler restore_registers 1 eret END(handle_el1h_irq) ENTRY(handle_el1h_error) brk 0xf13 END(handle_el1h_error) ENTRY(handle_el0_sync) save_registers 0 ldr x0, [x18, #PC_CURTHREAD] mov x1, sp str x1, [x0, #TD_FRAME] bl do_el0_sync do_ast restore_registers 0 eret END(handle_el0_sync) ENTRY(handle_el0_irq) save_registers 0 mov x0, sp bl intr_irq_handler do_ast restore_registers 0 eret END(handle_el0_irq) ENTRY(handle_el0_error) save_registers 0 mov x0, sp bl do_el0_error brk 0xf23 1: b 1b END(handle_el0_error) .macro vempty .align 7 brk 0xfff 1: b 1b .endm .macro vector name .align 7 b handle_\name .endm .align 11 .globl exception_vectors exception_vectors: vempty /* Synchronous EL1t */ vempty /* IRQ EL1t */ vempty /* FIQ EL1t */ vempty /* Error EL1t */ vector el1h_sync /* Synchronous EL1h */ vector el1h_irq /* IRQ EL1h */ vempty /* FIQ EL1h */ vector el1h_error /* Error EL1h */ vector el0_sync /* Synchronous 64-bit EL0 */ vector el0_irq /* IRQ 64-bit EL0 */ vempty /* FIQ 64-bit EL0 */ vector el0_error /* Error 64-bit EL0 */ vector el0_sync /* Synchronous 32-bit EL0 */ vector el0_irq /* IRQ 32-bit EL0 */ vempty /* FIQ 32-bit EL0 */ vector el0_error /* Error 32-bit EL0 */ Index: head/sys/arm64/arm64/locore.S =================================================================== --- head/sys/arm64/arm64/locore.S (revision 331253) +++ head/sys/arm64/arm64/locore.S (revision 331254) @@ -1,701 +1,701 @@ /*- * Copyright (c) 2012-2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#include "assym.s" +#include "assym.inc" #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #define VIRT_BITS 48 #define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) .globl kernbase .set kernbase, KERNBASE #define DEVICE_MEM 0 #define NORMAL_UNCACHED 1 #define NORMAL_MEM 2 /* * We assume: * MMU on with an identity map, or off * D-Cache: off * I-Cache: on or off * We are loaded at a 2MiB aligned address */ .text .globl _start _start: /* Drop to EL1 */ bl drop_to_el1 /* * Disable the MMU. We may have entered the kernel with it on and * will need to update the tables later. If this has been set up * with anything other than a VA == PA map then this will fail, * but in this case the code to find where we are running from * would have also failed. */ dsb sy mrs x2, sctlr_el1 bic x2, x2, SCTLR_M msr sctlr_el1, x2 isb /* Set the context id */ msr contextidr_el1, xzr /* Get the virt -> phys offset */ bl get_virt_delta /* * At this point: * x29 = PA - VA * x28 = Our physical load address */ /* Create the page tables */ bl create_pagetables /* * At this point: * x27 = TTBR0 table * x26 = Kernel L1 table * x24 = TTBR1 table */ /* Enable the mmu */ bl start_mmu /* Jump to the virtual address space */ ldr x15, .Lvirtdone br x15 virtdone: /* Set up the stack */ adr x25, initstack_end mov sp, x25 sub sp, sp, #PCB_SIZE /* Zero the BSS */ ldr x15, .Lbss ldr x14, .Lend 1: str xzr, [x15], #8 cmp x15, x14 b.lo 1b /* Backup the module pointer */ mov x1, x0 /* Make the page table base a virtual address */ sub x26, x26, x29 sub x24, x24, x29 sub sp, sp, #(64 * 4) mov x0, sp /* Degate the delda so it is VA -> PA */ neg x29, x29 str x1, [x0] /* modulep */ str x26, [x0, 8] /* kern_l1pt */ str x29, [x0, 16] /* kern_delta */ str x25, [x0, 24] /* kern_stack */ str x24, [x0, 32] /* kern_l0pt */ /* trace back starts here */ mov fp, #0 /* Branch to C code */ bl initarm bl mi_startup /* We should not get here */ brk 0 .align 3 .Lvirtdone: .quad virtdone .Lbss: .quad __bss_start .Lend: .quad _end #ifdef SMP /* * mpentry(unsigned long) * * Called by a core when it is being brought online. * The data in x0 is passed straight to init_secondary. */ ENTRY(mpentry) /* Disable interrupts */ msr daifset, #2 /* Drop to EL1 */ bl drop_to_el1 /* Set the context id */ msr contextidr_el1, xzr /* Load the kernel page table */ adr x24, pagetable_l0_ttbr1 /* Load the identity page table */ adr x27, pagetable_l0_ttbr0 /* Enable the mmu */ bl start_mmu /* Jump to the virtual address space */ ldr x15, =mp_virtdone br x15 mp_virtdone: ldr x4, =secondary_stacks mov x5, #(PAGE_SIZE * KSTACK_PAGES) mul x5, x0, x5 add sp, x4, x5 b init_secondary END(mpentry) #endif /* * If we are started in EL2, configure the required hypervisor * registers and drop to EL1. */ drop_to_el1: mrs x1, CurrentEL lsr x1, x1, #2 cmp x1, #0x2 b.eq 1f ret 1: /* Configure the Hypervisor */ mov x2, #(HCR_RW) msr hcr_el2, x2 /* Load the Virtualization Process ID Register */ mrs x2, midr_el1 msr vpidr_el2, x2 /* Load the Virtualization Multiprocess ID Register */ mrs x2, mpidr_el1 msr vmpidr_el2, x2 /* Set the bits that need to be 1 in sctlr_el1 */ ldr x2, .Lsctlr_res1 msr sctlr_el1, x2 /* Don't trap to EL2 for exceptions */ mov x2, #CPTR_RES1 msr cptr_el2, x2 /* Don't trap to EL2 for CP15 traps */ msr hstr_el2, xzr /* Enable access to the physical timers at EL1 */ mrs x2, cnthctl_el2 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) msr cnthctl_el2, x2 /* Set the counter offset to a known value */ msr cntvoff_el2, xzr /* Hypervisor trap functions */ adr x2, hyp_vectors msr vbar_el2, x2 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) msr spsr_el2, x2 /* Configure GICv3 CPU interface */ mrs x2, id_aa64pfr0_el1 /* Extract GIC bits from the register */ ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) b.ne 2f mrs x2, icc_sre_el2 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ msr icc_sre_el2, x2 2: /* Set the address to return to our return address */ msr elr_el2, x30 isb eret .align 3 .Lsctlr_res1: .quad SCTLR_RES1 #define VECT_EMPTY \ .align 7; \ 1: b 1b .align 11 hyp_vectors: VECT_EMPTY /* Synchronous EL2t */ VECT_EMPTY /* IRQ EL2t */ VECT_EMPTY /* FIQ EL2t */ VECT_EMPTY /* Error EL2t */ VECT_EMPTY /* Synchronous EL2h */ VECT_EMPTY /* IRQ EL2h */ VECT_EMPTY /* FIQ EL2h */ VECT_EMPTY /* Error EL2h */ VECT_EMPTY /* Synchronous 64-bit EL1 */ VECT_EMPTY /* IRQ 64-bit EL1 */ VECT_EMPTY /* FIQ 64-bit EL1 */ VECT_EMPTY /* Error 64-bit EL1 */ VECT_EMPTY /* Synchronous 32-bit EL1 */ VECT_EMPTY /* IRQ 32-bit EL1 */ VECT_EMPTY /* FIQ 32-bit EL1 */ VECT_EMPTY /* Error 32-bit EL1 */ /* * Get the delta between the physical address we were loaded to and the * virtual address we expect to run from. This is used when building the * initial page table. */ get_virt_delta: /* Load the physical address of virt_map */ adr x29, virt_map /* Load the virtual address of virt_map stored in virt_map */ ldr x28, [x29] /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ sub x29, x29, x28 /* Find the load address for the kernel */ mov x28, #(KERNBASE) add x28, x28, x29 ret .align 3 virt_map: .quad virt_map /* * This builds the page tables containing the identity map, and the kernel * virtual map. * * It relys on: * We were loaded to an address that is on a 2MiB boundary * All the memory must not cross a 1GiB boundaty * x28 contains the physical address we were loaded from * * TODO: This is out of date. * There are at least 5 pages before that address for the page tables * The pages used are: * - The Kernel L2 table * - The Kernel L1 table * - The Kernel L0 table (TTBR1) * - The identity (PA = VA) L1 table * - The identity (PA = VA) L0 table (TTBR0) * - The DMAP L1 tables */ create_pagetables: /* Save the Link register */ mov x5, x30 /* Clean the page table */ adr x6, pagetable mov x26, x6 adr x27, pagetable_end 1: stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 stp xzr, xzr, [x6], #16 cmp x6, x27 b.lo 1b /* * Build the TTBR1 maps. */ /* Find the size of the kernel */ mov x6, #(KERNBASE) /* Find modulep - begin */ sub x8, x0, x6 /* Add a 2MiB page for the module data and round up */ ldr x7, =(2 * L2_SIZE - 1) add x8, x8, x7 /* Get the number of l2 pages to allocate, rounded down */ lsr x10, x8, #(L2_SHIFT) /* Create the kernel space L2 table */ mov x6, x26 mov x7, #NORMAL_MEM mov x8, #(KERNBASE & L2_BLOCK_MASK) mov x9, x28 bl build_l2_block_pagetable /* Move to the l1 table */ add x26, x26, #PAGE_SIZE /* Link the l1 -> l2 table */ mov x9, x6 mov x6, x26 bl link_l1_pagetable /* Move to the l0 table */ add x24, x26, #PAGE_SIZE /* Link the l0 -> l1 table */ mov x9, x6 mov x6, x24 mov x10, #1 bl link_l0_pagetable /* Link the DMAP tables */ ldr x8, =DMAP_MIN_ADDRESS adr x9, pagetable_dmap; mov x10, #DMAP_TABLES bl link_l0_pagetable /* * Build the TTBR0 maps. */ add x27, x24, #PAGE_SIZE mov x6, x27 /* The initial page table */ #if defined(SOCDEV_PA) && defined(SOCDEV_VA) /* Create a table for the UART */ mov x7, #DEVICE_MEM mov x8, #(SOCDEV_VA) /* VA start */ mov x9, #(SOCDEV_PA) /* PA start */ mov x10, #1 bl build_l1_block_pagetable #endif /* Create the VA = PA map */ mov x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */ mov x9, x27 mov x8, x9 /* VA start (== PA start) */ mov x10, #1 bl build_l1_block_pagetable /* Move to the l0 table */ add x27, x27, #PAGE_SIZE /* Link the l0 -> l1 table */ mov x9, x6 mov x6, x27 mov x10, #1 bl link_l0_pagetable /* Restore the Link register */ mov x30, x5 ret /* * Builds an L0 -> L1 table descriptor * * This is a link for a 512GiB block of memory with up to 1GiB regions mapped * within it by build_l1_block_pagetable. * * x6 = L0 table * x8 = Virtual Address * x9 = L1 PA (trashed) * x10 = Entry count * x11, x12 and x13 are trashed */ link_l0_pagetable: /* * Link an L0 -> L1 table entry. */ /* Find the table index */ lsr x11, x8, #L0_SHIFT and x11, x11, #L0_ADDR_MASK /* Build the L0 block entry */ mov x12, #L0_TABLE /* Only use the output address bits */ lsr x9, x9, #PAGE_SHIFT 1: orr x13, x12, x9, lsl #PAGE_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] sub x10, x10, #1 add x11, x11, #1 add x9, x9, #1 cbnz x10, 1b ret /* * Builds an L1 -> L2 table descriptor * * This is a link for a 1GiB block of memory with up to 2MiB regions mapped * within it by build_l2_block_pagetable. * * x6 = L1 table * x8 = Virtual Address * x9 = L2 PA (trashed) * x11, x12 and x13 are trashed */ link_l1_pagetable: /* * Link an L1 -> L2 table entry. */ /* Find the table index */ lsr x11, x8, #L1_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L1 block entry */ mov x12, #L1_TABLE /* Only use the output address bits */ lsr x9, x9, #PAGE_SHIFT orr x13, x12, x9, lsl #PAGE_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] ret /* * Builds count 1 GiB page table entry * x6 = L1 table * x7 = Type (0 = Device, 1 = Normal) * x8 = VA start * x9 = PA start (trashed) * x10 = Entry count * x11, x12 and x13 are trashed */ build_l1_block_pagetable: /* * Build the L1 table entry. */ /* Find the table index */ lsr x11, x8, #L1_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L1 block entry */ lsl x12, x7, #2 orr x12, x12, #L1_BLOCK orr x12, x12, #(ATTR_AF) #ifdef SMP orr x12, x12, ATTR_SH(ATTR_SH_IS) #endif /* Only use the output address bits */ lsr x9, x9, #L1_SHIFT /* Set the physical address for this virtual address */ 1: orr x13, x12, x9, lsl #L1_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] sub x10, x10, #1 add x11, x11, #1 add x9, x9, #1 cbnz x10, 1b ret /* * Builds count 2 MiB page table entry * x6 = L2 table * x7 = Type (0 = Device, 1 = Normal) * x8 = VA start * x9 = PA start (trashed) * x10 = Entry count * x11, x12 and x13 are trashed */ build_l2_block_pagetable: /* * Build the L2 table entry. */ /* Find the table index */ lsr x11, x8, #L2_SHIFT and x11, x11, #Ln_ADDR_MASK /* Build the L2 block entry */ lsl x12, x7, #2 orr x12, x12, #L2_BLOCK orr x12, x12, #(ATTR_AF) #ifdef SMP orr x12, x12, ATTR_SH(ATTR_SH_IS) #endif /* Only use the output address bits */ lsr x9, x9, #L2_SHIFT /* Set the physical address for this virtual address */ 1: orr x13, x12, x9, lsl #L2_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] sub x10, x10, #1 add x11, x11, #1 add x9, x9, #1 cbnz x10, 1b ret start_mmu: dsb sy /* Load the exception vectors */ ldr x2, =exception_vectors msr vbar_el1, x2 /* Load ttbr0 and ttbr1 */ msr ttbr0_el1, x27 msr ttbr1_el1, x24 isb /* Clear the Monitor Debug System control register */ msr mdscr_el1, xzr /* Invalidate the TLB */ tlbi vmalle1is ldr x2, mair msr mair_el1, x2 /* * Setup TCR according to PARange bits from ID_AA64MMFR0_EL1. */ ldr x2, tcr mrs x3, id_aa64mmfr0_el1 bfi x2, x3, #32, #3 msr tcr_el1, x2 /* Setup SCTLR */ ldr x2, sctlr_set ldr x3, sctlr_clear mrs x1, sctlr_el1 bic x1, x1, x3 /* Clear the required bits */ orr x1, x1, x2 /* Set the required bits */ msr sctlr_el1, x1 isb ret .align 3 mair: .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, 0) | \ MAIR_ATTR(MAIR_NORMAL_NC, 1) | \ MAIR_ATTR(MAIR_NORMAL_WB, 2) | \ MAIR_ATTR(MAIR_NORMAL_WT, 3) tcr: .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K | \ TCR_CACHE_ATTRS | TCR_SMP_ATTRS) sctlr_set: /* Bits to set */ .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ SCTLR_M | SCTLR_CP15BEN) sctlr_clear: /* Bits to clear */ .quad (SCTLR_EE | SCTLR_EOE | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ SCTLR_ITD | SCTLR_THEE | SCTLR_A) .globl abort abort: b abort //.section .init_pagetable .align 12 /* 4KiB aligned */ /* * 3 initial tables (in the following order): * L2 for kernel (High addresses) * L1 for kernel * L1 for user (Low addresses) */ pagetable: .space PAGE_SIZE pagetable_l1_ttbr1: .space PAGE_SIZE pagetable_l0_ttbr1: .space PAGE_SIZE pagetable_l1_ttbr0: .space PAGE_SIZE pagetable_l0_ttbr0: .space PAGE_SIZE .globl pagetable_dmap pagetable_dmap: .space PAGE_SIZE * DMAP_TABLES pagetable_end: el2_pagetable: .space PAGE_SIZE .globl init_pt_va init_pt_va: .quad pagetable /* XXX: Keep page tables VA */ .align 4 initstack: .space (PAGE_SIZE * KSTACK_PAGES) initstack_end: ENTRY(sigcode) mov x0, sp add x0, x0, #SF_UC 1: mov x8, #SYS_sigreturn svc 0 /* sigreturn failed, exit */ mov x8, #SYS_exit svc 0 b 1b END(sigcode) /* This may be copied to the stack, keep it 16-byte aligned */ .align 3 esigcode: .data .align 3 .global szsigcode szsigcode: .quad esigcode - sigcode Index: head/sys/arm64/arm64/support.S =================================================================== --- head/sys/arm64/arm64/support.S (revision 331253) +++ head/sys/arm64/arm64/support.S (revision 331254) @@ -1,335 +1,335 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014-2015 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by Andrew Turner * under sponsorship from the FreeBSD Foundation * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include -#include "assym.s" +#include "assym.inc" /* * One of the fu* or su* functions failed, return -1. */ ENTRY(fsu_fault) SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ EXIT_USER_ACCESS_CHECK(w0, x1) fsu_fault_nopcb: mov x0, #-1 ret END(fsu_fault) /* * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) */ ENTRY(casueword32) ldr x4, =(VM_MAXUSER_ADDRESS-3) cmp x0, x4 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x4) /* And set it */ ENTER_USER_ACCESS(w6, x4) 1: ldxr w4, [x0] /* Load-exclusive the data */ cmp w4, w1 /* Compare */ b.ne 2f /* Not equal, exit */ stxr w5, w3, [x0] /* Store the new data */ cbnz w5, 1b /* Retry on failure */ EXIT_USER_ACCESS(w6) 2: SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */ str w4, [x2] /* Store the read data */ mov x0, #0 /* Success */ ret /* Return */ END(casueword32) /* * int casueword(volatile u_long *, u_long, u_long *, u_long) */ ENTRY(casueword) ldr x4, =(VM_MAXUSER_ADDRESS-7) cmp x0, x4 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x4) /* And set it */ ENTER_USER_ACCESS(w6, x4) 1: ldxr x4, [x0] /* Load-exclusive the data */ cmp x4, x1 /* Compare */ b.ne 2f /* Not equal, exit */ stxr w5, x3, [x0] /* Store the new data */ cbnz w5, 1b /* Retry on failure */ EXIT_USER_ACCESS(w6) 2: SET_FAULT_HANDLER(xzr, x5) /* Reset the fault handler */ str x4, [x2] /* Store the read data */ mov x0, #0 /* Success */ ret /* Return */ END(casueword) /* * int fubyte(volatile const void *) */ ENTRY(fubyte) ldr x1, =VM_MAXUSER_ADDRESS cmp x0, x1 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x1) /* And set it */ ldtrb w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ ret /* Return */ END(fubyte) /* * int fuword(volatile const void *) */ ENTRY(fuword16) ldr x1, =(VM_MAXUSER_ADDRESS-1) cmp x0, x1 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x1) /* And set it */ ldtrh w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ ret /* Return */ END(fuword16) /* * int32_t fueword32(volatile const void *, int32_t *) */ ENTRY(fueword32) ldr x2, =(VM_MAXUSER_ADDRESS-3) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ ldtr w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ str w0, [x1] /* Save the data in kernel space */ mov w0, #0 /* Success */ ret /* Return */ END(fueword32) /* * long fueword(volatile const void *, int64_t *) * int64_t fueword64(volatile const void *, int64_t *) */ ENTRY(fueword) EENTRY(fueword64) ldr x2, =(VM_MAXUSER_ADDRESS-7) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ ldtr x0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ str x0, [x1] /* Save the data in kernel space */ mov x0, #0 /* Success */ ret /* Return */ EEND(fueword64) END(fueword) /* * int subyte(volatile void *, int) */ ENTRY(subyte) ldr x2, =VM_MAXUSER_ADDRESS cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ sttrb w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(subyte) /* * int suword16(volatile void *, int) */ ENTRY(suword16) ldr x2, =(VM_MAXUSER_ADDRESS-1) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ sttrh w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(suword16) /* * int suword32(volatile void *, int) */ ENTRY(suword32) ldr x2, =(VM_MAXUSER_ADDRESS-3) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ sttr w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(suword32) /* * int suword(volatile void *, long) */ ENTRY(suword) EENTRY(suword64) ldr x2, =(VM_MAXUSER_ADDRESS-7) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ sttr x1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ EEND(suword64) END(suword) /* * fuswintr and suswintr are just like fusword and susword except that if * the page is not in memory or would cause a trap, then we return an error. * The important thing is to prevent sleep() and switch(). */ /* * Special handler so the trap code knows not to sleep. */ ENTRY(fsu_intr_fault) SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ EXIT_USER_ACCESS_CHECK(w0, x1) mov x0, #-1 ret END(fsu_fault) /* * int fuswintr(void *) */ ENTRY(fuswintr) ldr x1, =(VM_MAXUSER_ADDRESS-3) cmp x0, x1 b.cs fsu_fault_nopcb adr x6, fsu_intr_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x1) /* And set it */ ldtr w0, [x0] /* Try loading the data */ SET_FAULT_HANDLER(xzr, x1) /* Reset the fault handler */ ret /* Return */ END(fuswintr) /* * int suswintr(void *base, int word) */ ENTRY(suswintr) ldr x2, =(VM_MAXUSER_ADDRESS-3) cmp x0, x2 b.cs fsu_fault_nopcb adr x6, fsu_intr_fault /* Load the fault handler */ SET_FAULT_HANDLER(x6, x2) /* And set it */ sttr w1, [x0] /* Try storing the data */ SET_FAULT_HANDLER(xzr, x2) /* Reset the fault handler */ mov x0, #0 /* Success */ ret /* Return */ END(suswintr) ENTRY(setjmp) /* Store the stack pointer */ mov x8, sp str x8, [x0], #8 /* Store the general purpose registers and lr */ stp x19, x20, [x0], #16 stp x21, x22, [x0], #16 stp x23, x24, [x0], #16 stp x25, x26, [x0], #16 stp x27, x28, [x0], #16 stp x29, lr, [x0], #16 /* Return value */ mov x0, #0 ret END(setjmp) ENTRY(longjmp) /* Restore the stack pointer */ ldr x8, [x0], #8 mov sp, x8 /* Restore the general purpose registers and lr */ ldp x19, x20, [x0], #16 ldp x21, x22, [x0], #16 ldp x23, x24, [x0], #16 ldp x25, x26, [x0], #16 ldp x27, x28, [x0], #16 ldp x29, lr, [x0], #16 /* Load the return value */ mov x0, x1 ret END(longjmp) /* * pagezero, simple implementation */ ENTRY(pagezero_simple) add x1, x0, #PAGE_SIZE 1: stp xzr, xzr, [x0], #0x10 stp xzr, xzr, [x0], #0x10 stp xzr, xzr, [x0], #0x10 stp xzr, xzr, [x0], #0x10 cmp x0, x1 b.ne 1b ret END(pagezero_simple) /* * pagezero, cache assisted */ ENTRY(pagezero_cache) add x1, x0, #PAGE_SIZE ldr x2, =dczva_line_size ldr x2, [x2] 1: dc zva, x0 add x0, x0, x2 cmp x0, x1 b.ne 1b ret END(pagezero_cache) Index: head/sys/arm64/arm64/swtch.S =================================================================== --- head/sys/arm64/arm64/swtch.S (revision 331253) +++ head/sys/arm64/arm64/swtch.S (revision 331254) @@ -1,293 +1,293 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ -#include "assym.s" +#include "assym.inc" #include "opt_kstack_pages.h" #include "opt_sched.h" #include __FBSDID("$FreeBSD$"); .macro clear_step_flag pcbflags, tmp tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f mrs \tmp, mdscr_el1 bic \tmp, \tmp, #1 msr mdscr_el1, \tmp isb 999: .endm .macro set_step_flag pcbflags, tmp tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f mrs \tmp, mdscr_el1 orr \tmp, \tmp, #1 msr mdscr_el1, \tmp isb 999: .endm /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) /* Of old == NULL skip disabling stepping */ cbz x0, 1f /* If we were single stepping, disable it */ ldr x4, [x0, #TD_PCB] ldr w5, [x4, #PCB_FLAGS] clear_step_flag w5, x6 1: #ifdef VFP /* Backup the new thread pointer around a call to C code */ mov x19, x0 mov x20, x1 bl vfp_discard mov x1, x20 mov x0, x19 #endif bl pmap_switch mov x4, x0 /* If we are single stepping, enable it */ ldr w5, [x4, #PCB_FLAGS] set_step_flag w5, x6 /* Restore the registers */ ldp x5, x6, [x4, #PCB_SP] mov sp, x5 msr tpidr_el0, x6 ldr x6, [x4, #PCB_TPIDRRO] msr tpidrro_el0, x6 ldp x8, x9, [x4, #PCB_REGS + 8 * 8] ldp x10, x11, [x4, #PCB_REGS + 10 * 8] ldp x12, x13, [x4, #PCB_REGS + 12 * 8] ldp x14, x15, [x4, #PCB_REGS + 14 * 8] ldp x16, x17, [x4, #PCB_REGS + 16 * 8] ldr x19, [x4, #PCB_REGS + 19 * 8] ldp x20, x21, [x4, #PCB_REGS + 20 * 8] ldp x22, x23, [x4, #PCB_REGS + 22 * 8] ldp x24, x25, [x4, #PCB_REGS + 24 * 8] ldp x26, x27, [x4, #PCB_REGS + 26 * 8] ldp x28, x29, [x4, #PCB_REGS + 28 * 8] ldr x30, [x4, #PCB_REGS + 30 * 8] ret END(cpu_throw) /* * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) * * x0 = old * x1 = new * x2 = mtx * x3 to x7, x16 and x17 are caller saved */ ENTRY(cpu_switch) /* * Save the old context. */ ldr x4, [x0, #TD_PCB] /* Store the callee-saved registers */ stp x8, x9, [x4, #PCB_REGS + 8 * 8] stp x10, x11, [x4, #PCB_REGS + 10 * 8] stp x12, x13, [x4, #PCB_REGS + 12 * 8] stp x14, x15, [x4, #PCB_REGS + 14 * 8] stp x16, x17, [x4, #PCB_REGS + 16 * 8] stp x18, x19, [x4, #PCB_REGS + 18 * 8] stp x20, x21, [x4, #PCB_REGS + 20 * 8] stp x22, x23, [x4, #PCB_REGS + 22 * 8] stp x24, x25, [x4, #PCB_REGS + 24 * 8] stp x26, x27, [x4, #PCB_REGS + 26 * 8] stp x28, x29, [x4, #PCB_REGS + 28 * 8] str x30, [x4, #PCB_REGS + 30 * 8] /* And the old stack pointer */ mov x5, sp mrs x6, tpidrro_el0 str x6, [x4, #PCB_TPIDRRO] mrs x6, tpidr_el0 stp x5, x6, [x4, #PCB_SP] /* If we were single stepping, disable it */ ldr w5, [x4, #PCB_FLAGS] clear_step_flag w5, x6 mov x19, x0 mov x20, x1 mov x21, x2 #ifdef VFP /* Load the pcb address */ mov x1, x4 bl vfp_save_state mov x1, x20 mov x0, x19 #endif bl pmap_switch /* Move the new pcb out of the way */ mov x4, x0 mov x2, x21 mov x1, x20 mov x0, x19 /* * Release the old thread. This doesn't need to be a store-release * as the above dsb instruction will provide release semantics. */ str x2, [x0, #TD_LOCK] #if defined(SCHED_ULE) && defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ ldr x2, =_C_LABEL(blocked_lock) 1: ldar x3, [x1, #TD_LOCK] cmp x3, x2 b.eq 1b #endif /* If we are single stepping, enable it */ ldr w5, [x4, #PCB_FLAGS] set_step_flag w5, x6 /* Restore the registers */ ldp x5, x6, [x4, #PCB_SP] mov sp, x5 msr tpidr_el0, x6 ldr x6, [x4, #PCB_TPIDRRO] msr tpidrro_el0, x6 ldp x8, x9, [x4, #PCB_REGS + 8 * 8] ldp x10, x11, [x4, #PCB_REGS + 10 * 8] ldp x12, x13, [x4, #PCB_REGS + 12 * 8] ldp x14, x15, [x4, #PCB_REGS + 14 * 8] ldp x16, x17, [x4, #PCB_REGS + 16 * 8] ldr x19, [x4, #PCB_REGS + 19 * 8] ldp x20, x21, [x4, #PCB_REGS + 20 * 8] ldp x22, x23, [x4, #PCB_REGS + 22 * 8] ldp x24, x25, [x4, #PCB_REGS + 24 * 8] ldp x26, x27, [x4, #PCB_REGS + 26 * 8] ldp x28, x29, [x4, #PCB_REGS + 28 * 8] ldr x30, [x4, #PCB_REGS + 30 * 8] str xzr, [x4, #PCB_REGS + 18 * 8] ret .Lcpu_switch_panic_str: .asciz "cpu_switch: %p\0" END(cpu_switch) ENTRY(fork_trampoline) mov x0, x8 mov x1, x9 mov x2, sp mov fp, #0 /* Stack traceback stops here. */ bl _C_LABEL(fork_exit) /* Restore the registers other than x0 and x1 */ ldp x2, x3, [sp, #TF_X + 2 * 8] ldp x4, x5, [sp, #TF_X + 4 * 8] ldp x6, x7, [sp, #TF_X + 6 * 8] ldp x8, x9, [sp, #TF_X + 8 * 8] ldp x10, x11, [sp, #TF_X + 10 * 8] ldp x12, x13, [sp, #TF_X + 12 * 8] ldp x14, x15, [sp, #TF_X + 14 * 8] ldp x16, x17, [sp, #TF_X + 16 * 8] ldr x19, [sp, #TF_X + 19 * 8] ldp x20, x21, [sp, #TF_X + 20 * 8] ldp x22, x23, [sp, #TF_X + 22 * 8] ldp x24, x25, [sp, #TF_X + 24 * 8] ldp x26, x27, [sp, #TF_X + 26 * 8] ldp x28, x29, [sp, #TF_X + 28 * 8] /* * Disable interrupts to avoid * overwriting spsr_el1 and sp_el0 by an IRQ exception. */ msr daifset, #2 /* Restore sp and lr */ ldp x0, x1, [sp] msr sp_el0, x0 mov lr, x1 /* Restore elr and spsr */ ldp x0, x1, [sp, #16] msr elr_el1, x0 msr spsr_el1, x1 /* Finally x0 and x1 */ ldp x0, x1, [sp, #TF_X + 0 * 8] ldr x18, [sp, #TF_X + 18 * 8] /* * No need for interrupts reenabling since PSR * will be set to the desired value anyway. */ eret END(fork_trampoline) ENTRY(savectx) /* Store the callee-saved registers */ stp x8, x9, [x0, #PCB_REGS + 8 * 8] stp x10, x11, [x0, #PCB_REGS + 10 * 8] stp x12, x13, [x0, #PCB_REGS + 12 * 8] stp x14, x15, [x0, #PCB_REGS + 14 * 8] stp x16, x17, [x0, #PCB_REGS + 16 * 8] stp x18, x19, [x0, #PCB_REGS + 18 * 8] stp x20, x21, [x0, #PCB_REGS + 20 * 8] stp x22, x23, [x0, #PCB_REGS + 22 * 8] stp x24, x25, [x0, #PCB_REGS + 24 * 8] stp x26, x27, [x0, #PCB_REGS + 26 * 8] stp x28, x29, [x0, #PCB_REGS + 28 * 8] str x30, [x0, #PCB_REGS + 30 * 8] /* And the old stack pointer */ mov x5, sp mrs x6, tpidrro_el0 str x6, [x0, #PCB_TPIDRRO] mrs x6, tpidr_el0 stp x5, x6, [x0, #PCB_SP] /* Store the VFP registers */ #ifdef VFP mov x28, lr mov x1, x0 /* move pcb to the correct register */ mov x0, xzr /* td = NULL */ bl vfp_save_state mov lr, x28 #endif ret END(savectx) Index: head/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S =================================================================== --- head/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S (revision 331253) +++ head/sys/cddl/dev/dtrace/aarch64/dtrace_asm.S (revision 331254) @@ -1,174 +1,174 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #define _ASM #define _LOCORE #include #include #include #include -#include "assym.s" +#include "assym.inc" /* void dtrace_membar_producer(void) */ ENTRY(dtrace_membar_producer) RET END(dtrace_membar_producer) /* void dtrace_membar_consumer(void) */ ENTRY(dtrace_membar_consumer) RET END(dtrace_membar_consumer) /* dtrace_icookie_t dtrace_interrupt_disable(void) */ ENTRY(dtrace_interrupt_disable) mrs x0, daif msr daifset, #2 RET END(dtrace_interrupt_disable) /* void dtrace_interrupt_enable(dtrace_icookie_t cookie) */ ENTRY(dtrace_interrupt_enable) msr daif, x0 RET END(dtrace_interrupt_enable) /* uint8_t dtrace_fuword8_nocheck(void *addr) */ ENTRY(dtrace_fuword8_nocheck) ldrb w0, [x0] RET END(dtrace_fuword8_nocheck) /* uint16_t dtrace_fuword16_nocheck(void *addr) */ ENTRY(dtrace_fuword16_nocheck) ldrh w0, [x0] RET END(dtrace_fuword16_nocheck) /* uint32_t dtrace_fuword32_nocheck(void *addr) */ ENTRY(dtrace_fuword32_nocheck) ldr w0, [x0] RET END(dtrace_fuword32_nocheck) /* uint64_t dtrace_fuword64_nocheck(void *addr) */ ENTRY(dtrace_fuword64_nocheck) ldr x0, [x0] RET END(dtrace_fuword64_nocheck) /* void dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) */ ENTRY(dtrace_copy) cbz x2, 2f /* If len == 0 then skip loop */ 1: ldrb w4, [x0], #1 /* Load from uaddr */ strb w4, [x1], #1 /* Store in kaddr */ sub x2, x2, #1 /* len-- */ cbnz x2, 1b 2: RET END(dtrace_copy) /* void dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, volatile uint16_t *flags) XXX: Check for flags? */ ENTRY(dtrace_copystr) cbz x2, 2f /* If len == 0 then skip loop */ 1: ldrb w4, [x0], #1 /* Load from uaddr */ strb w4, [x1], #1 /* Store in kaddr */ cbz w4, 2f /* If == 0 then break */ sub x2, x2, #1 /* len-- */ cbnz x2, 1b 2: RET END(dtrace_copystr) /* uintptr_t dtrace_caller(int aframes) */ ENTRY(dtrace_caller) mov x0, #-1 RET END(dtrace_caller) /* uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) */ ENTRY(dtrace_cas32) 1: ldxr w3, [x0] /* Load target */ cmp w3, w1 /* Check if *target == cmp */ bne 2f /* No, return */ stxr w12, w2, [x0] /* Store new to target */ cbnz w12, 1b /* Try again if store not succeed */ 2: mov w0, w3 /* Return the value loaded from target */ RET END(dtrace_cas32) /* void * dtrace_casptr(volatile void *target, volatile void *cmp, volatile void *new) */ ENTRY(dtrace_casptr) 1: ldxr x3, [x0] /* Load target */ cmp x3, x1 /* Check if *target == cmp */ bne 2f /* No, return */ stxr w12, x2, [x0] /* Store new to target */ cbnz w12, 1b /* Try again if store not succeed */ 2: mov x0, x3 /* Return the value loaded from target */ RET END(dtrace_casptr) Index: head/sys/cddl/dev/dtrace/amd64/dtrace_asm.S =================================================================== --- head/sys/cddl/dev/dtrace/amd64/dtrace_asm.S (revision 331253) +++ head/sys/cddl/dev/dtrace/amd64/dtrace_asm.S (revision 331254) @@ -1,360 +1,360 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2008 John Birrell * * $FreeBSD$ * */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #define _ASM #include #include #include -#include "assym.s" +#include "assym.inc" #define INTR_POP \ MEXITCOUNT; \ movq TF_RDI(%rsp),%rdi; \ movq TF_RSI(%rsp),%rsi; \ movq TF_RDX(%rsp),%rdx; \ movq TF_RCX(%rsp),%rcx; \ movq TF_R8(%rsp),%r8; \ movq TF_R9(%rsp),%r9; \ movq TF_RAX(%rsp),%rax; \ movq TF_RBX(%rsp),%rbx; \ movq TF_RBP(%rsp),%rbp; \ movq TF_R10(%rsp),%r10; \ movq TF_R11(%rsp),%r11; \ movq TF_R12(%rsp),%r12; \ movq TF_R13(%rsp),%r13; \ movq TF_R14(%rsp),%r14; \ movq TF_R15(%rsp),%r15; \ testb $SEL_RPL_MASK,TF_CS(%rsp); \ jz 1f; \ cli; \ swapgs; \ 1: addq $TF_RIP,%rsp; ENTRY(dtrace_invop_start) /* * #BP traps with %rip set to the next address. We need to decrement * the value to indicate the address of the int3 (0xcc) instruction * that we substituted. */ movq TF_RIP(%rsp), %rdi decq %rdi movq %rsp, %rsi movq TF_RAX(%rsp), %rdx call dtrace_invop ALTENTRY(dtrace_invop_callsite) cmpl $DTRACE_INVOP_PUSHL_EBP, %eax je bp_push cmpl $DTRACE_INVOP_LEAVE, %eax je bp_leave cmpl $DTRACE_INVOP_NOP, %eax je bp_nop cmpl $DTRACE_INVOP_RET, %eax je bp_ret /* When all else fails handle the trap in the usual way. */ jmpq *dtrace_invop_calltrap_addr bp_push: /* * We must emulate a "pushq %rbp". To do this, we pull the stack * down 8 bytes, and then store the base pointer. */ INTR_POP subq $16, %rsp /* make room for %rbp */ pushq %rax /* push temp */ movq 24(%rsp), %rax /* load calling RIP */ movq %rax, 8(%rsp) /* store calling RIP */ movq 32(%rsp), %rax /* load calling CS */ movq %rax, 16(%rsp) /* store calling CS */ movq 40(%rsp), %rax /* load calling RFLAGS */ movq %rax, 24(%rsp) /* store calling RFLAGS */ movq 48(%rsp), %rax /* load calling RSP */ subq $8, %rax /* make room for %rbp */ movq %rax, 32(%rsp) /* store calling RSP */ movq 56(%rsp), %rax /* load calling SS */ movq %rax, 40(%rsp) /* store calling SS */ movq 32(%rsp), %rax /* reload calling RSP */ movq %rbp, (%rax) /* store %rbp there */ popq %rax /* pop off temp */ iretq /* return from interrupt */ /*NOTREACHED*/ bp_leave: /* * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" * followed by a "popq %rbp". This is quite a bit simpler on amd64 * than it is on i386 -- we can exploit the fact that the %rsp is * explicitly saved to effect the pop without having to reshuffle * the other data pushed for the trap. */ INTR_POP pushq %rax /* push temp */ movq 8(%rsp), %rax /* load calling RIP */ movq %rax, 8(%rsp) /* store calling RIP */ movq (%rbp), %rax /* get new %rbp */ addq $8, %rbp /* adjust new %rsp */ movq %rbp, 32(%rsp) /* store new %rsp */ movq %rax, %rbp /* set new %rbp */ popq %rax /* pop off temp */ iretq /* return from interrupt */ /*NOTREACHED*/ bp_nop: /* We must emulate a "nop". */ INTR_POP iretq /*NOTREACHED*/ bp_ret: INTR_POP pushq %rax /* push temp */ movq 32(%rsp), %rax /* load %rsp */ movq (%rax), %rax /* load calling RIP */ movq %rax, 8(%rsp) /* store calling RIP */ addq $8, 32(%rsp) /* adjust new %rsp */ popq %rax /* pop off temp */ iretq /* return from interrupt */ /*NOTREACHED*/ END(dtrace_invop_start) /* void dtrace_invop_init(void) */ ENTRY(dtrace_invop_init) movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip) ret END(dtrace_invop_init) /* void dtrace_invop_uninit(void) */ ENTRY(dtrace_invop_uninit) movq $0, dtrace_invop_jump_addr(%rip) ret END(dtrace_invop_uninit) /* greg_t dtrace_getfp(void) */ ENTRY(dtrace_getfp) movq %rbp, %rax ret END(dtrace_getfp) /* uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) */ ENTRY(dtrace_cas32) movl %esi, %eax lock cmpxchgl %edx, (%rdi) ret END(dtrace_cas32) /* void * dtrace_casptr(void *target, void *cmp, void *new) */ ENTRY(dtrace_casptr) movq %rsi, %rax lock cmpxchgq %rdx, (%rdi) ret END(dtrace_casptr) /* uintptr_t dtrace_caller(int aframes) */ ENTRY(dtrace_caller) movq $-1, %rax ret END(dtrace_caller) /* void dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) */ ENTRY(dtrace_copy) pushq %rbp movq %rsp, %rbp xchgq %rdi, %rsi /* make %rsi source, %rdi dest */ movq %rdx, %rcx /* load count */ repz /* repeat for count ... */ smovb /* move from %ds:rsi to %ed:rdi */ leave ret END(dtrace_copy) /* void dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, volatile uint16_t *flags) */ ENTRY(dtrace_copystr) pushq %rbp movq %rsp, %rbp 0: movb (%rdi), %al /* load from source */ movb %al, (%rsi) /* store to destination */ addq $1, %rdi /* increment source pointer */ addq $1, %rsi /* increment destination pointer */ subq $1, %rdx /* decrement remaining count */ cmpb $0, %al je 2f testq $0xfff, %rdx /* test if count is 4k-aligned */ jnz 1f /* if not, continue with copying */ testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */ jnz 2f 1: cmpq $0, %rdx jne 0b 2: leave ret END(dtrace_copystr) /* uintptr_t dtrace_fulword(void *addr) */ ENTRY(dtrace_fulword) movq (%rdi), %rax ret END(dtrace_fulword) /* uint8_t dtrace_fuword8_nocheck(void *addr) */ ENTRY(dtrace_fuword8_nocheck) xorq %rax, %rax movb (%rdi), %al ret END(dtrace_fuword8_nocheck) /* uint16_t dtrace_fuword16_nocheck(void *addr) */ ENTRY(dtrace_fuword16_nocheck) xorq %rax, %rax movw (%rdi), %ax ret END(dtrace_fuword16_nocheck) /* uint32_t dtrace_fuword32_nocheck(void *addr) */ ENTRY(dtrace_fuword32_nocheck) xorq %rax, %rax movl (%rdi), %eax ret END(dtrace_fuword32_nocheck) /* uint64_t dtrace_fuword64_nocheck(void *addr) */ ENTRY(dtrace_fuword64_nocheck) movq (%rdi), %rax ret END(dtrace_fuword64_nocheck) /* void dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, int fault, int fltoffs, uintptr_t illval) */ ENTRY(dtrace_probe_error) pushq %rbp movq %rsp, %rbp subq $0x8, %rsp movq %r9, (%rsp) movq %r8, %r9 movq %rcx, %r8 movq %rdx, %rcx movq %rsi, %rdx movq %rdi, %rsi movl dtrace_probeid_error(%rip), %edi call dtrace_probe addq $0x8, %rsp leave ret END(dtrace_probe_error) /* void dtrace_membar_producer(void) */ ENTRY(dtrace_membar_producer) rep; ret /* use 2 byte return instruction when branch target */ /* AMD Software Optimization Guide - Section 6.2 */ END(dtrace_membar_producer) /* void dtrace_membar_consumer(void) */ ENTRY(dtrace_membar_consumer) rep; ret /* use 2 byte return instruction when branch target */ /* AMD Software Optimization Guide - Section 6.2 */ END(dtrace_membar_consumer) /* dtrace_icookie_t dtrace_interrupt_disable(void) */ ENTRY(dtrace_interrupt_disable) pushfq popq %rax cli ret END(dtrace_interrupt_disable) /* void dtrace_interrupt_enable(dtrace_icookie_t cookie) */ ENTRY(dtrace_interrupt_enable) pushq %rdi popfq ret END(dtrace_interrupt_enable) Index: head/sys/cddl/dev/dtrace/arm/dtrace_asm.S =================================================================== --- head/sys/cddl/dev/dtrace/arm/dtrace_asm.S (revision 331253) +++ head/sys/cddl/dev/dtrace/arm/dtrace_asm.S (revision 331254) @@ -1,198 +1,198 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #define _ASM #define _LOCORE #include #include #include #include -#include "assym.s" +#include "assym.inc" /* void dtrace_membar_producer(void) */ ENTRY(dtrace_membar_producer) RET END(dtrace_membar_producer) /* void dtrace_membar_consumer(void) */ ENTRY(dtrace_membar_consumer) RET END(dtrace_membar_consumer) /* dtrace_icookie_t dtrace_interrupt_disable(void) */ ENTRY(dtrace_interrupt_disable) mrs r0, cpsr mov r1, r0 orr r1, r1, #(PSR_I | PSR_F) msr cpsr_c, r1 RET END(dtrace_interrupt_disable) /* void dtrace_interrupt_enable(dtrace_icookie_t cookie) */ ENTRY(dtrace_interrupt_enable) and r0, r0, #(PSR_I | PSR_F) mrs r1, cpsr bic r1, r1, #(PSR_I | PSR_F) orr r1, r1, r0 msr cpsr_c, r1 RET END(dtrace_interrupt_enable) /* uint8_t dtrace_fuword8_nocheck(void *addr) */ ENTRY(dtrace_fuword8_nocheck) ldrb r3, [r0] mov r0, r3 RET END(dtrace_fuword8_nocheck) /* uint16_t dtrace_fuword16_nocheck(void *addr) */ ENTRY(dtrace_fuword16_nocheck) ldrh r3, [r0] mov r0, r3 RET END(dtrace_fuword16_nocheck) /* uint32_t dtrace_fuword32_nocheck(void *addr) */ ENTRY(dtrace_fuword32_nocheck) ldr r3, [r0] mov r0, r3 RET END(dtrace_fuword32_nocheck) /* uint64_t dtrace_fuword64_nocheck(void *addr) */ ENTRY(dtrace_fuword64_nocheck) ldm r0, {r2, r3} mov r0, r2 mov r1, r3 #if defined(__BIG_ENDIAN__) /* big endian */ mov r0, r3 mov r1, r2 #else /* little endian */ mov r0, r2 mov r1, r3 #endif RET END(dtrace_fuword64_nocheck) /* void dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) */ ENTRY(dtrace_copy) stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ teq r2, #0x00000000 mov r5, #0x00000000 beq 2f 1: ldrb r4, [r0], #0x0001 add r5, r5, #0x00000001 strb r4, [r1], #0x0001 teqne r5, r2 bne 1b 2: ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ RET END(dtrace_copy) /* void dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, volatile uint16_t *flags) XXX: Check for flags? */ ENTRY(dtrace_copystr) stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ teq r2, #0x00000000 mov r5, #0x00000000 beq 2f 1: ldrb r4, [r0], #0x0001 add r5, r5, #0x00000001 teq r4, #0x00000000 strb r4, [r1], #0x0001 teqne r5, r2 bne 1b 2: ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ RET END(dtrace_copystr) /* uintptr_t dtrace_caller(int aframes) */ ENTRY(dtrace_caller) mov r0, #-1 RET END(dtrace_caller) /* uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) void * dtrace_casptr(volatile void *target, volatile void *cmp, volatile void *new) */ ENTRY(dtrace_cas32) EENTRY(dtrace_casptr) 1: ldrex r3, [r0] /* Load target */ cmp r3, r1 /* Check if *target == cmp */ bne 2f /* No, return */ strex ip, r2, [r0] /* Store new to target */ cmp ip, #0 /* Did the store succeed? */ bne 1b /* No, try again */ 2: mov r0, r3 /* Return the value loaded from target */ RET EEND(dtrace_casptr) END(dtrace_cas32) Index: head/sys/cddl/dev/dtrace/i386/dtrace_asm.S =================================================================== --- head/sys/cddl/dev/dtrace/i386/dtrace_asm.S (revision 331253) +++ head/sys/cddl/dev/dtrace/i386/dtrace_asm.S (revision 331254) @@ -1,355 +1,355 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #define _ASM #include #include #include -#include "assym.s" +#include "assym.inc" ENTRY(dtrace_invop_start) pushl %eax /* push %eax -- may be return value */ pushl %esp /* push stack pointer */ subl $8, (%esp) /* skip first arg and segment regs */ pushl 40(%esp) /* push calling EIP */ /* * Call dtrace_invop to let it check if the exception was * a fbt one. The return value in %eax will tell us what * dtrace_invop wants us to do. */ call dtrace_invop ALTENTRY(dtrace_invop_callsite) addl $12, %esp cmpl $DTRACE_INVOP_PUSHL_EBP, %eax je invop_push cmpl $DTRACE_INVOP_POPL_EBP, %eax je invop_pop cmpl $DTRACE_INVOP_LEAVE, %eax je invop_leave cmpl $DTRACE_INVOP_NOP, %eax je invop_nop /* When all else fails handle the trap in the usual way. */ jmpl *dtrace_invop_calltrap_addr invop_push: /* * We must emulate a "pushl %ebp". To do this, we pull the stack * down 4 bytes, and then store the base pointer. */ popal subl $4, %esp /* make room for %ebp */ pushl %eax /* push temp */ movl 8(%esp), %eax /* load calling EIP */ incl %eax /* increment over LOCK prefix */ movl %eax, 4(%esp) /* store calling EIP */ movl 12(%esp), %eax /* load calling CS */ movl %eax, 8(%esp) /* store calling CS */ movl 16(%esp), %eax /* load calling EFLAGS */ movl %eax, 12(%esp) /* store calling EFLAGS */ movl %ebp, 16(%esp) /* push %ebp */ popl %eax /* pop off temp */ iret /* Return from interrupt. */ invop_pop: /* * We must emulate a "popl %ebp". To do this, we do the opposite of * the above: we remove the %ebp from the stack, and squeeze up the * saved state from the trap. */ popal pushl %eax /* push temp */ movl 16(%esp), %ebp /* pop %ebp */ movl 12(%esp), %eax /* load calling EFLAGS */ movl %eax, 16(%esp) /* store calling EFLAGS */ movl 8(%esp), %eax /* load calling CS */ movl %eax, 12(%esp) /* store calling CS */ movl 4(%esp), %eax /* load calling EIP */ incl %eax /* increment over LOCK prefix */ movl %eax, 8(%esp) /* store calling EIP */ popl %eax /* pop off temp */ addl $4, %esp /* adjust stack pointer */ iret /* Return from interrupt. */ invop_leave: /* * We must emulate a "leave", which is the same as a "movl %ebp, %esp" * followed by a "popl %ebp". This looks similar to the above, but * requires two temporaries: one for the new base pointer, and one * for the staging register. */ popa pushl %eax /* push temp */ pushl %ebx /* push temp */ movl %ebp, %ebx /* set temp to old %ebp */ movl (%ebx), %ebp /* pop %ebp */ movl 16(%esp), %eax /* load calling EFLAGS */ movl %eax, (%ebx) /* store calling EFLAGS */ movl 12(%esp), %eax /* load calling CS */ movl %eax, -4(%ebx) /* store calling CS */ movl 8(%esp), %eax /* load calling EIP */ incl %eax /* increment over LOCK prefix */ movl %eax, -8(%ebx) /* store calling EIP */ subl $8, %ebx /* adjust for three pushes, one pop */ movl %ebx, 8(%esp) /* temporarily store new %esp */ popl %ebx /* pop off temp */ popl %eax /* pop off temp */ movl (%esp), %esp /* set stack pointer */ iret /* return from interrupt */ invop_nop: /* * We must emulate a "nop". This is obviously not hard: we need only * advance the %eip by one. */ popa incl (%esp) iret /* return from interrupt */ END(dtrace_invop_start) /* void dtrace_invop_init(void) */ ENTRY(dtrace_invop_init) movl $dtrace_invop_start, dtrace_invop_jump_addr ret END(dtrace_invop_init) /* void dtrace_invop_uninit(void) */ ENTRY(dtrace_invop_uninit) movl $0, dtrace_invop_jump_addr ret END(dtrace_invop_uninit) /* greg_t dtrace_getfp(void) */ ENTRY(dtrace_getfp) movl %ebp, %eax ret END(dtrace_getfp) /* uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) */ ENTRY(dtrace_cas32) ALTENTRY(dtrace_casptr) movl 4(%esp), %edx movl 8(%esp), %eax movl 12(%esp), %ecx lock cmpxchgl %ecx, (%edx) ret END(dtrace_casptr) END(dtrace_cas32) /* uintptr_t dtrace_caller(int aframes) */ ENTRY(dtrace_caller) movl $-1, %eax ret END(dtrace_caller) /* void dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) */ ENTRY(dtrace_copy) pushl %ebp movl %esp, %ebp pushl %esi pushl %edi movl 8(%ebp), %esi /* Load source address */ movl 12(%ebp), %edi /* Load destination address */ movl 16(%ebp), %ecx /* Load count */ repz /* Repeat for count... */ smovb /* move from %ds:si to %es:di */ popl %edi popl %esi movl %ebp, %esp popl %ebp ret END(dtrace_copy) /* void dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size) */ ENTRY(dtrace_copystr) pushl %ebp /* Setup stack frame */ movl %esp, %ebp pushl %ebx /* Save registers */ movl 8(%ebp), %ebx /* Load source address */ movl 12(%ebp), %edx /* Load destination address */ movl 16(%ebp), %ecx /* Load count */ 0: movb (%ebx), %al /* Load from source */ movb %al, (%edx) /* Store to destination */ incl %ebx /* Increment source pointer */ incl %edx /* Increment destination pointer */ decl %ecx /* Decrement remaining count */ cmpb $0, %al je 1f cmpl $0, %ecx jne 0b 1: popl %ebx movl %ebp, %esp popl %ebp ret END(dtrace_copystr) /* uintptr_t dtrace_fulword(void *addr) */ ENTRY(dtrace_fulword) movl 4(%esp), %ecx xorl %eax, %eax movl (%ecx), %eax ret END(dtrace_fulword) /* uint8_t dtrace_fuword8_nocheck(void *addr) */ ENTRY(dtrace_fuword8_nocheck) movl 4(%esp), %ecx xorl %eax, %eax movzbl (%ecx), %eax ret END(dtrace_fuword8_nocheck) /* uint16_t dtrace_fuword16_nocheck(void *addr) */ ENTRY(dtrace_fuword16_nocheck) movl 4(%esp), %ecx xorl %eax, %eax movzwl (%ecx), %eax ret END(dtrace_fuword16_nocheck) /* uint32_t dtrace_fuword32_nocheck(void *addr) */ ENTRY(dtrace_fuword32_nocheck) movl 4(%esp), %ecx xorl %eax, %eax movl (%ecx), %eax ret END(dtrace_fuword32_nocheck) /* uint64_t dtrace_fuword64_nocheck(void *addr) */ ENTRY(dtrace_fuword64_nocheck) movl 4(%esp), %ecx xorl %eax, %eax xorl %edx, %edx movl (%ecx), %eax movl 4(%ecx), %edx ret END(dtrace_fuword64_nocheck) /* void dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which, int fault, int fltoffs, uintptr_t illval) */ ENTRY(dtrace_probe_error) pushl %ebp movl %esp, %ebp pushl 0x1c(%ebp) pushl 0x18(%ebp) pushl 0x14(%ebp) pushl 0x10(%ebp) pushl 0xc(%ebp) pushl 0x8(%ebp) pushl dtrace_probeid_error call dtrace_probe movl %ebp, %esp popl %ebp ret END(dtrace_probe_error) /* void dtrace_membar_producer(void) */ ENTRY(dtrace_membar_producer) rep; ret /* use 2 byte return instruction when branch target */ /* AMD Software Optimization Guide - Section 6.2 */ END(dtrace_membar_producer) /* void dtrace_membar_consumer(void) */ ENTRY(dtrace_membar_consumer) rep; ret /* use 2 byte return instruction when branch target */ /* AMD Software Optimization Guide - Section 6.2 */ END(dtrace_membar_consumer) /* dtrace_icookie_t dtrace_interrupt_disable(void) */ ENTRY(dtrace_interrupt_disable) pushfl popl %eax cli ret END(dtrace_interrupt_disable) /* void dtrace_interrupt_enable(dtrace_icookie_t cookie) */ ENTRY(dtrace_interrupt_enable) movl 4(%esp), %eax pushl %eax popfl ret END(dtrace_interrupt_enable) Index: head/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S =================================================================== --- head/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S (revision 331253) +++ head/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S (revision 331254) @@ -1,176 +1,176 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2012,2013 Justin Hibbits * * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#include "assym.s" +#include "assym.inc" #define _ASM #include #include #include /* #include */ /* * Primitives */ .text /* void dtrace_membar_producer(void) */ ASENTRY_NOPROF(dtrace_membar_producer) sync blr END(dtrace_membar_producer) /* void dtrace_membar_consumer(void) */ ASENTRY_NOPROF(dtrace_membar_consumer) isync blr END(dtrace_membar_consumer) /* dtrace_icookie_t dtrace_interrupt_disable(void) */ ASENTRY_NOPROF(dtrace_interrupt_disable) mfmsr %r3 #ifdef __powerpc64__ /* Two-instruction sequence to clear EE flag */ rldicl %r0,%r3,48,1 rotldi %r0,%r0,16 #else rlwinm %r0,%r3,0,~PSL_EE /* Clear EE flag */ #endif mtmsr %r0 blr END(dtrace_interrupt_disable) /* void dtrace_interrupt_enable(dtrace_icookie_t cookie) */ ASENTRY_NOPROF(dtrace_interrupt_enable) mtmsr %r3 blr END(dtrace_interrupt_enable) /* uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) */ ASENTRY_NOPROF(dtrace_cas32) 1: lwarx %r0,0,%r3 cmpw %r4,%r0 bne 2f stwcx. %r5,0,%r3 bne 1b 2: mr %r3,%r0 blr END(dtrace_cas32) /* void * dtrace_casptr(void *target, void *cmp, void *new) */ ASENTRY_NOPROF(dtrace_casptr) #ifdef __powerpc64__ 1: ldarx %r0,0,%r3 cmpd %r4,%r0 bne 2f stdcx. %r5,0,%r3 bne 1b #else 1: lwarx %r0,0,%r3 cmpw %r4,%r0 bne 2f stwcx. %r5,0,%r3 bne 1b #endif 2: mr %r3,%r0 blr END(dtrace_casptr) /* XXX: unoptimized void dtrace_copy(uintptr_t src, uintptr_t dest, size_t size) */ ASENTRY_NOPROF(dtrace_copy) subi %r7,%r3,1 subi %r8,%r4,1 mtctr %r5 1: lbzu %r3,1(%r7) stbu %r3,1(%r8) bdnz 1b 2: blr END(dtrace_copy) /* void dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, volatile uint16_t *flags) */ ASENTRY_NOPROF(dtrace_copystr) addme %r7,%r3 addme %r8,%r4 1: lbzu %r3,1(%r7) stbu %r3,1(%r8) addme %r5,%r5 beq 2f or %r3,%r3,%r3 beq 2f andi. %r0,%r5,0x0fff beq 2f lwz %r0,0(%r6) andi. %r0,%r0,CPU_DTRACE_BADADDR beq 1b 2: blr END(dtrace_copystr) /* uintptr_t dtrace_caller(int aframes) */ ASENTRY_NOPROF(dtrace_caller) li %r3, -1 blr END(dtrace_caller) Index: head/sys/cddl/dev/dtrace/riscv/dtrace_asm.S =================================================================== --- head/sys/cddl/dev/dtrace/riscv/dtrace_asm.S (revision 331253) +++ head/sys/cddl/dev/dtrace/riscv/dtrace_asm.S (revision 331254) @@ -1,177 +1,177 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Portions Copyright 2016 Ruslan Bukin * * $FreeBSD$ */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #define _ASM #define _LOCORE #include #include #include #include -#include "assym.s" +#include "assym.inc" /* void dtrace_membar_producer(void) */ ENTRY(dtrace_membar_producer) RET END(dtrace_membar_producer) /* void dtrace_membar_consumer(void) */ ENTRY(dtrace_membar_consumer) RET END(dtrace_membar_consumer) /* dtrace_icookie_t dtrace_interrupt_disable(void) */ ENTRY(dtrace_interrupt_disable) csrrci a0, sstatus, (SSTATUS_SIE) andi a0, a0, (SSTATUS_SIE) RET END(dtrace_interrupt_disable) /* void dtrace_interrupt_enable(dtrace_icookie_t cookie) */ ENTRY(dtrace_interrupt_enable) csrs sstatus, a0 RET END(dtrace_interrupt_enable) /* uint8_t dtrace_fuword8_nocheck(void *addr) */ ENTRY(dtrace_fuword8_nocheck) lb a0, 0(a0) RET END(dtrace_fuword8_nocheck) /* uint16_t dtrace_fuword16_nocheck(void *addr) */ ENTRY(dtrace_fuword16_nocheck) lh a0, 0(a0) RET END(dtrace_fuword16_nocheck) /* uint32_t dtrace_fuword32_nocheck(void *addr) */ ENTRY(dtrace_fuword32_nocheck) lw a0, 0(a0) RET END(dtrace_fuword32_nocheck) /* uint64_t dtrace_fuword64_nocheck(void *addr) */ ENTRY(dtrace_fuword64_nocheck) ld a0, 0(a0) RET END(dtrace_fuword64_nocheck) /* void dtrace_copy(uintptr_t uaddr, uintptr_t kaddr, size_t size) */ ENTRY(dtrace_copy) beqz a2, 2f /* If len == 0 then skip loop */ 1: lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in kaddr */ addi a1, a1, 1 addi a2, a2, -1 /* len-- */ bnez a2, 1b 2: RET END(dtrace_copy) /* void dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size, volatile uint16_t *flags) XXX: Check for flags? */ ENTRY(dtrace_copystr) beqz a2, 2f /* If len == 0 then skip loop */ lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in kaddr */ addi a1, a1, 1 beqz a4, 2f /* If == 0 then break */ addi a2, a2, -1 /* len-- */ bnez a2, 1b 2: RET END(dtrace_copystr) /* uintptr_t dtrace_caller(int aframes) */ ENTRY(dtrace_caller) li a0, -1 RET END(dtrace_caller) /* uint32_t dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new) */ ENTRY(dtrace_cas32) 1: lr.w a3, 0(a0) /* Load target */ bne a3, a1, 2f /* *target != cmp ? return */ sc.w a4, a2, 0(a0) /* Store new to target */ bnez a4, 1b /* Try again if store not succeed */ 2: mv a0, a3 /* Return the value loaded from target */ RET END(dtrace_cas32) /* void * dtrace_casptr(volatile void *target, volatile void *cmp, volatile void *new) */ ENTRY(dtrace_casptr) 1: lr.d a3, 0(a0) /* Load target */ bne a3, a1, 2f /* *target != cmp ? return */ sc.d a4, a2, 0(a0) /* Store new to target */ bnez a4, 1b /* Try again if store not succeed */ 2: mv a0, a3 /* Return the value loaded from target */ RET END(dtrace_casptr) Index: head/sys/conf/files.amd64 =================================================================== --- head/sys/conf/files.amd64 (revision 331253) +++ head/sys/conf/files.amd64 (revision 331254) @@ -1,730 +1,730 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # # cloudabi32_vdso.o optional compat_cloudabi32 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_i686_on_64bit.S" \ compile-with "${CC} -x assembler-with-cpp -m32 -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_i686_on_64bit.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi32_vdso.o" # cloudabi32_vdso_blob.o optional compat_cloudabi32 \ dependency "cloudabi32_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd --binary-architecture i386 cloudabi32_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi32_vdso_blob.o" # cloudabi64_vdso.o optional compat_cloudabi64 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_x86_64.S" \ compile-with "${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_x86_64.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi64_vdso.o" # cloudabi64_vdso_blob.o optional compat_cloudabi64 \ dependency "cloudabi64_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd --binary-architecture i386 cloudabi64_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi64_vdso_blob.o" # linux32_genassym.o optional compat_linux32 \ dependency "$S/amd64/linux32/linux32_genassym.c" \ compile-with "${CC} ${CFLAGS:N-flto:N-fno-common} -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "linux32_genassym.o" # linux32_assym.h optional compat_linux32 \ dependency "$S/kern/genassym.sh linux32_genassym.o" \ compile-with "sh $S/kern/genassym.sh linux32_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "linux32_assym.h" # linux32_locore.o optional compat_linux32 \ dependency "linux32_assym.h $S/amd64/linux32/linux32_locore.s" \ compile-with "${CC} -x assembler-with-cpp -DLOCORE -m32 -shared -s -pipe -I. -I$S -Werror -Wall -fPIC -fno-common -nostdinc -nostdlib -Wl,-T$S/amd64/linux32/linux32_vdso.lds.s -Wl,-soname=linux32_vdso.so,--eh-frame-hdr,-warn-common ${.IMPSRC} -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "linux32_locore.o" # linux32_vdso.so optional compat_linux32 \ dependency "linux32_locore.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd --binary-architecture i386 linux32_locore.o ${.TARGET}" \ no-implicit-rule \ clean "linux32_vdso.so" # ia32_genassym.o standard \ dependency "$S/compat/ia32/ia32_genassym.c" \ compile-with "${CC} ${CFLAGS:N-flto:N-fno-common} -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "ia32_genassym.o" # ia32_assym.h standard \ dependency "$S/kern/genassym.sh ia32_genassym.o" \ compile-with "env NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh ia32_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "ia32_assym.h" # font.h optional sc_dflt_font \ compile-with "uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'static u_char dflt_font_16[16*256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'static u_char dflt_font_14[14*256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'static u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" # atkbdmap.h optional atkbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${ATKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > atkbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" # ukbdmap.h optional ukbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${UKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > ukbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "ukbdmap.h" # hpt27xx_lib.o optional hpt27xx \ dependency "$S/dev/hpt27xx/amd64-elf.hpt27xx_lib.o.uu" \ compile-with "uudecode < $S/dev/hpt27xx/amd64-elf.hpt27xx_lib.o.uu" \ no-implicit-rule # hptmvraid.o optional hptmv \ dependency "$S/dev/hptmv/amd64-elf.raid.o.uu" \ compile-with "uudecode < $S/dev/hptmv/amd64-elf.raid.o.uu" \ no-implicit-rule # hptnr_lib.o optional hptnr \ dependency "$S/dev/hptnr/amd64-elf.hptnr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptnr/amd64-elf.hptnr_lib.o.uu" \ no-implicit-rule # hptrr_lib.o optional hptrr \ dependency "$S/dev/hptrr/amd64-elf.hptrr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptrr/amd64-elf.hptrr_lib.o.uu" \ no-implicit-rule # amd64/acpica/acpi_machdep.c optional acpi acpi_wakecode.o optional acpi \ - dependency "$S/amd64/acpica/acpi_wakecode.S assym.s" \ + dependency "$S/amd64/acpica/acpi_wakecode.S assym.inc" \ compile-with "${NORMAL_S}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.o" acpi_wakecode.bin optional acpi \ dependency "acpi_wakecode.o" \ compile-with "${OBJCOPY} -S -O binary acpi_wakecode.o ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.bin" acpi_wakecode.h optional acpi \ dependency "acpi_wakecode.bin" \ compile-with "file2c -sx 'static char wakecode[] = {' '};' < acpi_wakecode.bin > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.h" acpi_wakedata.h optional acpi \ dependency "acpi_wakecode.o" \ compile-with '${NM} -n --defined-only acpi_wakecode.o | while read offset dummy what; do echo "#define $${what} 0x$${offset}"; done > ${.TARGET}' \ no-obj no-implicit-rule before-depend \ clean "acpi_wakedata.h" # #amd64/amd64/apic_vector.S standard amd64/amd64/atomic.c standard amd64/amd64/bios.c standard amd64/amd64/bpf_jit_machdep.c optional bpf_jitter amd64/amd64/cpu_switch.S standard amd64/amd64/db_disasm.c optional ddb amd64/amd64/db_interface.c optional ddb amd64/amd64/db_trace.c optional ddb amd64/amd64/efirt_machdep.c optional efirt amd64/amd64/elf_machdep.c standard amd64/amd64/exception.S standard amd64/amd64/fpu.c standard amd64/amd64/gdb_machdep.c optional gdb amd64/amd64/in_cksum.c optional inet | inet6 amd64/amd64/initcpu.c standard amd64/amd64/io.c optional io amd64/amd64/locore.S standard no-obj amd64/amd64/xen-locore.S optional xenhvm amd64/amd64/machdep.c standard amd64/amd64/mem.c optional mem amd64/amd64/minidump_machdep.c standard amd64/amd64/mp_machdep.c optional smp amd64/amd64/mpboot.S optional smp amd64/amd64/pmap.c standard amd64/amd64/prof_machdep.c optional profiling-routine amd64/amd64/ptrace_machdep.c standard amd64/amd64/sigtramp.S standard amd64/amd64/support.S standard amd64/amd64/sys_machdep.c standard amd64/amd64/trap.c standard amd64/amd64/uio_machdep.c standard amd64/amd64/uma_machdep.c standard amd64/amd64/vm_machdep.c standard amd64/cloudabi32/cloudabi32_sysvec.c optional compat_cloudabi32 amd64/cloudabi64/cloudabi64_sysvec.c optional compat_cloudabi64 amd64/pci/pci_cfgreg.c optional pci cddl/contrib/opensolaris/common/atomic/amd64/opensolaris_atomic.S optional zfs | dtrace compile-with "${ZFS_S}" cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/x86/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/dtrace/x86/dis_tables.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" cddl/dev/dtrace/x86/instr_size.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/aesni/aesni.c optional aesni aesni_ghash.o optional aesni \ dependency "$S/crypto/aesni/aesni_ghash.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_wrap.o" crypto/blowfish/bf_enc.c optional crypto | ipsec | ipsec_support crypto/des/des_enc.c optional crypto | ipsec | \ ipsec_support | netsmb intel_sha1.o optional aesni \ dependency "$S/crypto/aesni/intel_sha1.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha1.o" intel_sha256.o optional aesni \ dependency "$S/crypto/aesni/intel_sha256.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha256.o" crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock dev/acpica/acpi_if.m standard dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/acpica/acpi_timer.c optional acpi dev/acpi_support/acpi_wmi_if.m standard dev/agp/agp_amd64.c optional agp dev/agp/agp_i810.c optional agp dev/agp/agp_via.c optional agp dev/amdsbwd/amdsbwd.c optional amdsbwd dev/amdsmn/amdsmn.c optional amdsmn | amdtemp dev/amdtemp/amdtemp.c optional amdtemp dev/arcmsr/arcmsr.c optional arcmsr pci dev/asmc/asmc.c optional asmc isa dev/atkbdc/atkbd.c optional atkbd atkbdc dev/atkbdc/atkbd_atkbdc.c optional atkbd atkbdc dev/atkbdc/atkbdc.c optional atkbdc dev/atkbdc/atkbdc_isa.c optional atkbdc isa dev/atkbdc/atkbdc_subr.c optional atkbdc dev/atkbdc/psm.c optional psm atkbdc dev/bxe/bxe.c optional bxe pci dev/bxe/bxe_stats.c optional bxe pci dev/bxe/bxe_debug.c optional bxe pci dev/bxe/ecore_sp.c optional bxe pci dev/bxe/bxe_elink.c optional bxe pci dev/bxe/57710_init_values.c optional bxe pci dev/bxe/57711_init_values.c optional bxe pci dev/bxe/57712_init_values.c optional bxe pci dev/coretemp/coretemp.c optional coretemp dev/cpuctl/cpuctl.c optional cpuctl dev/dpms/dpms.c optional dpms # There are no systems with isa slots, so all ed isa entries should go.. dev/ed/if_ed_3c503.c optional ed isa ed_3c503 dev/ed/if_ed_isa.c optional ed isa dev/ed/if_ed_wd80x3.c optional ed isa dev/ed/if_ed_hpp.c optional ed isa ed_hpp dev/ed/if_ed_sic.c optional ed isa ed_sic dev/fb/fb.c optional fb | vga dev/fb/s3_pci.c optional s3pci dev/fb/vesa.c optional vga vesa dev/fb/vga.c optional vga dev/ichwd/ichwd.c optional ichwd dev/if_ndis/if_ndis.c optional ndis dev/if_ndis/if_ndis_pccard.c optional ndis pccard dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb dev/imcsmb/imcsmb.c optional imcsmb dev/imcsmb/imcsmb_pci.c optional imcsmb pci dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/ioat/ioat.c optional ioat pci dev/ioat/ioat_test.c optional ioat pci dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_linux.c optional ipmi compat_linux32 dev/ixl/if_ixl.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_main.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_qmgr.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_iov.c optional ixl pci pci_iov \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_pf_i2c.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_iw.c optional ixl pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/if_ixlv.c optional ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixlvc.c optional ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/ixl_txrx.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_osdep.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_lan_hmc.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_hmc.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_common.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_nvm.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/ixl/i40e_adminq.c optional ixl pci | ixlv pci \ compile-with "${NORMAL_C} -I$S/dev/ixl" dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc dev/fdc/fdc_isa.c optional fdc isa dev/fdc/fdc_pccard.c optional fdc pccard dev/gpio/bytgpio.c optional bytgpio dev/gpio/chvgpio.c optional chvgpio dev/hpt27xx/hpt27xx_os_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_osm_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_config.c optional hpt27xx dev/hptmv/entry.c optional hptmv dev/hptmv/mv.c optional hptmv dev/hptmv/gui_lib.c optional hptmv dev/hptmv/hptproc.c optional hptmv dev/hptmv/ioctl.c optional hptmv dev/hptnr/hptnr_os_bsd.c optional hptnr dev/hptnr/hptnr_osm_bsd.c optional hptnr dev/hptnr/hptnr_config.c optional hptnr dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/input/hv_kbd.c optional hyperv dev/hyperv/input/hv_kbdc.c optional hyperv dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci dev/hyperv/netvsc/hn_nvs.c optional hyperv dev/hyperv/netvsc/hn_rndis.c optional hyperv dev/hyperv/netvsc/if_hn.c optional hyperv dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv dev/hyperv/utilities/hv_kvp.c optional hyperv dev/hyperv/utilities/hv_snapshot.c optional hyperv dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv dev/hyperv/utilities/vmbus_ic.c optional hyperv dev/hyperv/utilities/vmbus_shutdown.c optional hyperv dev/hyperv/utilities/vmbus_timesync.c optional hyperv dev/hyperv/vmbus/hyperv.c optional hyperv dev/hyperv/vmbus/hyperv_busdma.c optional hyperv dev/hyperv/vmbus/vmbus.c optional hyperv pci dev/hyperv/vmbus/vmbus_br.c optional hyperv dev/hyperv/vmbus/vmbus_chan.c optional hyperv dev/hyperv/vmbus/vmbus_et.c optional hyperv dev/hyperv/vmbus/vmbus_if.m optional hyperv dev/hyperv/vmbus/vmbus_res.c optional hyperv dev/hyperv/vmbus/vmbus_xact.c optional hyperv dev/hyperv/vmbus/amd64/hyperv_machdep.c optional hyperv dev/hyperv/vmbus/amd64/vmbus_vector.S optional hyperv dev/nctgpio/nctgpio.c optional nctgpio dev/nfe/if_nfe.c optional nfe pci dev/ntb/if_ntb/if_ntb.c optional if_ntb dev/ntb/ntb_transport.c optional ntb_transport | if_ntb dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_hw/ntb_hw_intel.c optional ntb_hw_intel | ntb_hw dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/nvd/nvd.c optional nvd nvme dev/nvme/nvme.c optional nvme dev/nvme/nvme_ctrlr.c optional nvme dev/nvme/nvme_ctrlr_cmd.c optional nvme dev/nvme/nvme_ns.c optional nvme dev/nvme/nvme_ns_cmd.c optional nvme dev/nvme/nvme_qpair.c optional nvme dev/nvme/nvme_sim.c optional nvme scbus dev/nvme/nvme_sysctl.c optional nvme dev/nvme/nvme_test.c optional nvme dev/nvme/nvme_util.c optional nvme dev/nvram/nvram.c optional nvram isa dev/random/ivy.c optional rdrand_rng dev/random/nehemiah.c optional padlock_rng dev/qlxge/qls_dbg.c optional qlxge pci dev/qlxge/qls_dump.c optional qlxge pci dev/qlxge/qls_hw.c optional qlxge pci dev/qlxge/qls_ioctl.c optional qlxge pci dev/qlxge/qls_isr.c optional qlxge pci dev/qlxge/qls_os.c optional qlxge pci dev/qlxgb/qla_dbg.c optional qlxgb pci dev/qlxgb/qla_hw.c optional qlxgb pci dev/qlxgb/qla_ioctl.c optional qlxgb pci dev/qlxgb/qla_isr.c optional qlxgb pci dev/qlxgb/qla_misc.c optional qlxgb pci dev/qlxgb/qla_os.c optional qlxgb pci dev/qlxgbe/ql_dbg.c optional qlxgbe pci dev/qlxgbe/ql_hw.c optional qlxgbe pci dev/qlxgbe/ql_ioctl.c optional qlxgbe pci dev/qlxgbe/ql_isr.c optional qlxgbe pci dev/qlxgbe/ql_misc.c optional qlxgbe pci dev/qlxgbe/ql_os.c optional qlxgbe pci dev/qlxgbe/ql_reset.c optional qlxgbe pci dev/qlnx/qlnxe/ecore_cxt.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dbg_fw_funcs.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dcbx.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_dev.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_hw.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_init_fw_funcs.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_init_ops.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_int.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_l2.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_mcp.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_sp_commands.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/ecore_spq.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/qlnx_ioctl.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/qlnx/qlnxe/qlnx_os.c optional qlnxe pci \ compile-with "${LINUXKPI_C}" dev/sfxge/common/ef10_ev.c optional sfxge pci dev/sfxge/common/ef10_filter.c optional sfxge pci dev/sfxge/common/ef10_intr.c optional sfxge pci dev/sfxge/common/ef10_mac.c optional sfxge pci dev/sfxge/common/ef10_mcdi.c optional sfxge pci dev/sfxge/common/ef10_nic.c optional sfxge pci dev/sfxge/common/ef10_nvram.c optional sfxge pci dev/sfxge/common/ef10_phy.c optional sfxge pci dev/sfxge/common/ef10_rx.c optional sfxge pci dev/sfxge/common/ef10_tx.c optional sfxge pci dev/sfxge/common/ef10_vpd.c optional sfxge pci dev/sfxge/common/efx_bootcfg.c optional sfxge pci dev/sfxge/common/efx_crc32.c optional sfxge pci dev/sfxge/common/efx_ev.c optional sfxge pci dev/sfxge/common/efx_filter.c optional sfxge pci dev/sfxge/common/efx_hash.c optional sfxge pci dev/sfxge/common/efx_intr.c optional sfxge pci dev/sfxge/common/efx_lic.c optional sfxge pci dev/sfxge/common/efx_mac.c optional sfxge pci dev/sfxge/common/efx_mcdi.c optional sfxge pci dev/sfxge/common/efx_mon.c optional sfxge pci dev/sfxge/common/efx_nic.c optional sfxge pci dev/sfxge/common/efx_nvram.c optional sfxge pci dev/sfxge/common/efx_phy.c optional sfxge pci dev/sfxge/common/efx_port.c optional sfxge pci dev/sfxge/common/efx_rx.c optional sfxge pci dev/sfxge/common/efx_sram.c optional sfxge pci dev/sfxge/common/efx_tx.c optional sfxge pci dev/sfxge/common/efx_vpd.c optional sfxge pci dev/sfxge/common/hunt_nic.c optional sfxge pci dev/sfxge/common/mcdi_mon.c optional sfxge pci dev/sfxge/common/medford_nic.c optional sfxge pci dev/sfxge/common/siena_mac.c optional sfxge pci dev/sfxge/common/siena_mcdi.c optional sfxge pci dev/sfxge/common/siena_nic.c optional sfxge pci dev/sfxge/common/siena_nvram.c optional sfxge pci dev/sfxge/common/siena_phy.c optional sfxge pci dev/sfxge/common/siena_sram.c optional sfxge pci dev/sfxge/common/siena_vpd.c optional sfxge pci dev/sfxge/sfxge.c optional sfxge pci dev/sfxge/sfxge_dma.c optional sfxge pci dev/sfxge/sfxge_ev.c optional sfxge pci dev/sfxge/sfxge_intr.c optional sfxge pci dev/sfxge/sfxge_mcdi.c optional sfxge pci dev/sfxge/sfxge_nvram.c optional sfxge pci dev/sfxge/sfxge_port.c optional sfxge pci dev/sfxge/sfxge_rx.c optional sfxge pci dev/sfxge/sfxge_tx.c optional sfxge pci dev/sio/sio.c optional sio dev/sio/sio_isa.c optional sio isa dev/sio/sio_pccard.c optional sio pccard dev/sio/sio_pci.c optional sio pci dev/sio/sio_puc.c optional sio puc dev/speaker/spkr.c optional speaker dev/syscons/apm/apm_saver.c optional apm_saver apm dev/syscons/scterm-teken.c optional sc dev/syscons/scvesactl.c optional sc vga vesa dev/syscons/scvgarndr.c optional sc vga dev/syscons/scvtb.c optional sc dev/tpm/tpm.c optional tpm dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/uart/uart_cpu_x86.c optional uart dev/viawd/viawd.c optional viawd dev/vmware/vmxnet3/if_vmx.c optional vmx dev/wbwd/wbwd.c optional wbwd dev/xen/pci/xen_acpi_pci.c optional xenhvm dev/xen/pci/xen_pci.c optional xenhvm dev/isci/isci.c optional isci dev/isci/isci_controller.c optional isci dev/isci/isci_domain.c optional isci dev/isci/isci_interrupt.c optional isci dev/isci/isci_io_request.c optional isci dev/isci/isci_logger.c optional isci dev/isci/isci_oem_parameters.c optional isci dev/isci/isci_remote_device.c optional isci dev/isci/isci_sysctl.c optional isci dev/isci/isci_task_request.c optional isci dev/isci/isci_timer.c optional isci dev/isci/scil/sati.c optional isci dev/isci/scil/sati_abort_task_set.c optional isci dev/isci/scil/sati_atapi.c optional isci dev/isci/scil/sati_device.c optional isci dev/isci/scil/sati_inquiry.c optional isci dev/isci/scil/sati_log_sense.c optional isci dev/isci/scil/sati_lun_reset.c optional isci dev/isci/scil/sati_mode_pages.c optional isci dev/isci/scil/sati_mode_select.c optional isci dev/isci/scil/sati_mode_sense.c optional isci dev/isci/scil/sati_mode_sense_10.c optional isci dev/isci/scil/sati_mode_sense_6.c optional isci dev/isci/scil/sati_move.c optional isci dev/isci/scil/sati_passthrough.c optional isci dev/isci/scil/sati_read.c optional isci dev/isci/scil/sati_read_buffer.c optional isci dev/isci/scil/sati_read_capacity.c optional isci dev/isci/scil/sati_reassign_blocks.c optional isci dev/isci/scil/sati_report_luns.c optional isci dev/isci/scil/sati_request_sense.c optional isci dev/isci/scil/sati_start_stop_unit.c optional isci dev/isci/scil/sati_synchronize_cache.c optional isci dev/isci/scil/sati_test_unit_ready.c optional isci dev/isci/scil/sati_unmap.c optional isci dev/isci/scil/sati_util.c optional isci dev/isci/scil/sati_verify.c optional isci dev/isci/scil/sati_write.c optional isci dev/isci/scil/sati_write_and_verify.c optional isci dev/isci/scil/sati_write_buffer.c optional isci dev/isci/scil/sati_write_long.c optional isci dev/isci/scil/sci_abstract_list.c optional isci dev/isci/scil/sci_base_controller.c optional isci dev/isci/scil/sci_base_domain.c optional isci dev/isci/scil/sci_base_iterator.c optional isci dev/isci/scil/sci_base_library.c optional isci dev/isci/scil/sci_base_logger.c optional isci dev/isci/scil/sci_base_memory_descriptor_list.c optional isci dev/isci/scil/sci_base_memory_descriptor_list_decorator.c optional isci dev/isci/scil/sci_base_object.c optional isci dev/isci/scil/sci_base_observer.c optional isci dev/isci/scil/sci_base_phy.c optional isci dev/isci/scil/sci_base_port.c optional isci dev/isci/scil/sci_base_remote_device.c optional isci dev/isci/scil/sci_base_request.c optional isci dev/isci/scil/sci_base_state_machine.c optional isci dev/isci/scil/sci_base_state_machine_logger.c optional isci dev/isci/scil/sci_base_state_machine_observer.c optional isci dev/isci/scil/sci_base_subject.c optional isci dev/isci/scil/sci_util.c optional isci dev/isci/scil/scic_sds_controller.c optional isci dev/isci/scil/scic_sds_library.c optional isci dev/isci/scil/scic_sds_pci.c optional isci dev/isci/scil/scic_sds_phy.c optional isci dev/isci/scil/scic_sds_port.c optional isci dev/isci/scil/scic_sds_port_configuration_agent.c optional isci dev/isci/scil/scic_sds_remote_device.c optional isci dev/isci/scil/scic_sds_remote_node_context.c optional isci dev/isci/scil/scic_sds_remote_node_table.c optional isci dev/isci/scil/scic_sds_request.c optional isci dev/isci/scil/scic_sds_sgpio.c optional isci dev/isci/scil/scic_sds_smp_remote_device.c optional isci dev/isci/scil/scic_sds_smp_request.c optional isci dev/isci/scil/scic_sds_ssp_request.c optional isci dev/isci/scil/scic_sds_stp_packet_request.c optional isci dev/isci/scil/scic_sds_stp_remote_device.c optional isci dev/isci/scil/scic_sds_stp_request.c optional isci dev/isci/scil/scic_sds_unsolicited_frame_control.c optional isci dev/isci/scil/scif_sas_controller.c optional isci dev/isci/scil/scif_sas_controller_state_handlers.c optional isci dev/isci/scil/scif_sas_controller_states.c optional isci dev/isci/scil/scif_sas_domain.c optional isci dev/isci/scil/scif_sas_domain_state_handlers.c optional isci dev/isci/scil/scif_sas_domain_states.c optional isci dev/isci/scil/scif_sas_high_priority_request_queue.c optional isci dev/isci/scil/scif_sas_internal_io_request.c optional isci dev/isci/scil/scif_sas_io_request.c optional isci dev/isci/scil/scif_sas_io_request_state_handlers.c optional isci dev/isci/scil/scif_sas_io_request_states.c optional isci dev/isci/scil/scif_sas_library.c optional isci dev/isci/scil/scif_sas_remote_device.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substates.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substates.c optional isci dev/isci/scil/scif_sas_remote_device_state_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_states.c optional isci dev/isci/scil/scif_sas_request.c optional isci dev/isci/scil/scif_sas_smp_activity_clear_affiliation.c optional isci dev/isci/scil/scif_sas_smp_io_request.c optional isci dev/isci/scil/scif_sas_smp_phy.c optional isci dev/isci/scil/scif_sas_smp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_io_request.c optional isci dev/isci/scil/scif_sas_stp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_task_request.c optional isci dev/isci/scil/scif_sas_task_request.c optional isci dev/isci/scil/scif_sas_task_request_state_handlers.c optional isci dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci isa/syscons_isa.c optional sc isa/vga_isa.c optional vga kern/kern_clocksource.c standard kern/link_elf_obj.c standard libkern/x86/crc32_sse42.c standard libkern/memmove.c standard libkern/memset.c standard # # IA32 binary support # #amd64/ia32/ia32_exception.S optional compat_freebsd32 amd64/ia32/ia32_reg.c optional compat_freebsd32 amd64/ia32/ia32_signal.c optional compat_freebsd32 amd64/ia32/ia32_sigtramp.S optional compat_freebsd32 amd64/ia32/ia32_syscall.c optional compat_freebsd32 amd64/ia32/ia32_misc.c optional compat_freebsd32 compat/ia32/ia32_sysvec.c optional compat_freebsd32 compat/linprocfs/linprocfs.c optional linprocfs compat/linsysfs/linsysfs.c optional linsysfs # # Linux/i386 binary support # amd64/linux32/linux32_dummy.c optional compat_linux32 amd64/linux32/linux32_machdep.c optional compat_linux32 amd64/linux32/linux32_support.s optional compat_linux32 \ dependency "linux32_assym.h" amd64/linux32/linux32_sysent.c optional compat_linux32 amd64/linux32/linux32_sysvec.c optional compat_linux32 compat/linux/linux_emul.c optional compat_linux32 compat/linux/linux_errno.c optional compat_linux32 compat/linux/linux_file.c optional compat_linux32 compat/linux/linux_fork.c optional compat_linux32 compat/linux/linux_futex.c optional compat_linux32 compat/linux/linux_getcwd.c optional compat_linux32 compat/linux/linux_ioctl.c optional compat_linux32 compat/linux/linux_ipc.c optional compat_linux32 compat/linux/linux_mib.c optional compat_linux32 compat/linux/linux_misc.c optional compat_linux32 compat/linux/linux_mmap.c optional compat_linux32 compat/linux/linux_signal.c optional compat_linux32 compat/linux/linux_socket.c optional compat_linux32 compat/linux/linux_stats.c optional compat_linux32 compat/linux/linux_sysctl.c optional compat_linux32 compat/linux/linux_time.c optional compat_linux32 compat/linux/linux_timer.c optional compat_linux32 compat/linux/linux_uid16.c optional compat_linux32 compat/linux/linux_util.c optional compat_linux32 compat/linux/linux_vdso.c optional compat_linux32 compat/linux/linux_common.c optional compat_linux32 compat/linux/linux_event.c optional compat_linux32 compat/linux/linux.c optional compat_linux32 dev/amr/amr_linux.c optional compat_linux32 amr dev/mfi/mfi_linux.c optional compat_linux32 mfi # # Windows NDIS driver support # compat/ndis/kern_ndis.c optional ndisapi pci compat/ndis/kern_windrv.c optional ndisapi pci compat/ndis/subr_hal.c optional ndisapi pci compat/ndis/subr_ndis.c optional ndisapi pci compat/ndis/subr_ntoskrnl.c optional ndisapi pci compat/ndis/subr_pe.c optional ndisapi pci compat/ndis/subr_usbd.c optional ndisapi pci compat/ndis/winx64_wrap.S optional ndisapi pci # # x86 real mode BIOS emulator, required by dpms/pci/vesa # compat/x86bios/x86bios.c optional x86bios | dpms | pci | vesa contrib/x86emu/x86emu.c optional x86bios | dpms | pci | vesa # # bvm console # dev/bvm/bvm_console.c optional bvmconsole dev/bvm/bvm_dbg.c optional bvmdebug # # x86 shared code between IA32 and AMD64 architectures # x86/acpica/OsdEnvironment.c optional acpi x86/acpica/acpi_apm.c optional acpi x86/acpica/acpi_wakeup.c optional acpi x86/acpica/madt.c optional acpi x86/acpica/srat.c optional acpi x86/bios/smbios.c optional smbios x86/bios/vpd.c optional vpd x86/cpufreq/powernow.c optional cpufreq x86/cpufreq/est.c optional cpufreq x86/cpufreq/hwpstate.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/iommu/busdma_dmar.c optional acpi acpi_dmar pci x86/iommu/intel_ctx.c optional acpi acpi_dmar pci x86/iommu/intel_drv.c optional acpi acpi_dmar pci x86/iommu/intel_fault.c optional acpi acpi_dmar pci x86/iommu/intel_gas.c optional acpi acpi_dmar pci x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci x86/iommu/intel_qi.c optional acpi acpi_dmar pci x86/iommu/intel_quirks.c optional acpi acpi_dmar pci x86/iommu/intel_utils.c optional acpi acpi_dmar pci x86/isa/atpic.c optional atpic isa x86/isa/atrtc.c standard x86/isa/clock.c standard x86/isa/elcr.c optional atpic isa | mptable x86/isa/isa.c standard x86/isa/isa_dma.c standard x86/isa/nmi.c standard x86/isa/orm.c optional isa x86/pci/pci_bus.c optional pci x86/pci/qpi.c optional pci x86/x86/autoconf.c standard x86/x86/bus_machdep.c standard x86/x86/busdma_bounce.c standard x86/x86/busdma_machdep.c standard x86/x86/cpu_machdep.c standard x86/x86/dump_machdep.c standard x86/x86/fdt_machdep.c optional fdt x86/x86/identcpu.c standard x86/x86/intr_machdep.c standard x86/x86/io_apic.c standard x86/x86/legacy.c standard x86/x86/local_apic.c standard x86/x86/mca.c standard x86/x86/x86_mem.c optional mem x86/x86/mptable.c optional mptable x86/x86/mptable_pci.c optional mptable pci x86/x86/mp_x86.c optional smp x86/x86/mp_watchdog.c optional mp_watchdog smp x86/x86/msi.c optional pci x86/x86/nexus.c standard x86/x86/pvclock.c standard x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/pv.c optional xenhvm x86/xen/pvcpu_enum.c optional xenhvm x86/xen/xen_apic.c optional xenhvm x86/xen/xenpv.c optional xenhvm x86/xen/xen_nexus.c optional xenhvm x86/xen/xen_msi.c optional xenhvm x86/xen/xen_pci_bus.c optional xenhvm Index: head/sys/conf/files.i386 =================================================================== --- head/sys/conf/files.i386 (revision 331253) +++ head/sys/conf/files.i386 (revision 331254) @@ -1,628 +1,628 @@ # This file tells config what files go into building a kernel, # files marked standard are always included. # # $FreeBSD$ # # The long compile-with and dependency lines are required because of # limitations in config: backslash-newline doesn't work in strings, and # dependency lines other than the first are silently ignored. # cloudabi32_vdso.o optional compat_cloudabi32 \ dependency "$S/contrib/cloudabi/cloudabi_vdso_i686.S" \ compile-with "${CC} -x assembler-with-cpp -shared -nostdinc -nostdlib -Wl,-T$S/compat/cloudabi/cloudabi_vdso.lds $S/contrib/cloudabi/cloudabi_vdso_i686.S -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "cloudabi32_vdso.o" # cloudabi32_vdso_blob.o optional compat_cloudabi32 \ dependency "cloudabi32_vdso.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf32-i386-freebsd --binary-architecture i386 cloudabi32_vdso.o ${.TARGET}" \ no-implicit-rule \ clean "cloudabi32_vdso_blob.o" # linux_genassym.o optional compat_linux \ dependency "$S/i386/linux/linux_genassym.c" \ compile-with "${CC} ${CFLAGS:N-flto:N-fno-common} -c ${.IMPSRC}" \ no-obj no-implicit-rule \ clean "linux_genassym.o" # linux_assym.h optional compat_linux \ dependency "$S/kern/genassym.sh linux_genassym.o" \ compile-with "sh $S/kern/genassym.sh linux_genassym.o > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "linux_assym.h" # linux_locore.o optional compat_linux \ dependency "linux_assym.h $S/i386/linux/linux_locore.s" \ compile-with "${CC} -x assembler-with-cpp -DLOCORE -shared -s -pipe -I. -I$S -Werror -Wall -fPIC -fno-common -nostdinc -nostdlib -Wl,-T$S/i386/linux/linux_vdso.lds.s -Wl,-soname=linux_vdso.so,--eh-frame-hdr,-warn-common ${.IMPSRC} -o ${.TARGET}" \ no-obj no-implicit-rule \ clean "linux_locore.o" # linux_vdso.so optional compat_linux \ dependency "linux_locore.o" \ compile-with "${OBJCOPY} --input-target binary --output-target elf32-i386-freebsd --binary-architecture i386 linux_locore.o ${.TARGET}" \ no-implicit-rule \ clean "linux_vdso.so" # font.h optional sc_dflt_font \ compile-with "uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'static u_char dflt_font_16[16*256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'static u_char dflt_font_14[14*256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'static u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" # atkbdmap.h optional atkbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${ATKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > atkbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "atkbdmap.h" # ukbdmap.h optional ukbd_dflt_keymap \ compile-with "kbdcontrol -P ${S:S/sys$/share/}/vt/keymaps -P ${S:S/sys$/share/}/syscons/keymaps -L ${UKBD_DFLT_KEYMAP} | sed -e 's/^static keymap_t.* = /static keymap_t key_map = /' -e 's/^static accentmap_t.* = /static accentmap_t accent_map = /' > ukbdmap.h" \ no-obj no-implicit-rule before-depend \ clean "ukbdmap.h" # hpt27xx_lib.o optional hpt27xx \ dependency "$S/dev/hpt27xx/i386-elf.hpt27xx_lib.o.uu" \ compile-with "uudecode < $S/dev/hpt27xx/i386-elf.hpt27xx_lib.o.uu" \ no-implicit-rule # hptmvraid.o optional hptmv \ dependency "$S/dev/hptmv/i386-elf.raid.o.uu" \ compile-with "uudecode < $S/dev/hptmv/i386-elf.raid.o.uu" \ no-implicit-rule # hptnr_lib.o optional hptnr \ dependency "$S/dev/hptnr/i386-elf.hptnr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptnr/i386-elf.hptnr_lib.o.uu" \ no-implicit-rule # hptrr_lib.o optional hptrr \ dependency "$S/dev/hptrr/i386-elf.hptrr_lib.o.uu" \ compile-with "uudecode < $S/dev/hptrr/i386-elf.hptrr_lib.o.uu" \ no-implicit-rule # cddl/contrib/opensolaris/common/atomic/i386/opensolaris_atomic.S optional zfs | dtrace compile-with "${ZFS_S}" cddl/dev/dtrace/i386/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/i386/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/x86/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" cddl/dev/dtrace/x86/dis_tables.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" cddl/dev/dtrace/x86/instr_size.c optional dtrace_fbt | dtraceall compile-with "${DTRACE_C}" compat/linprocfs/linprocfs.c optional linprocfs compat/linsysfs/linsysfs.c optional linsysfs compat/linux/linux_event.c optional compat_linux compat/linux/linux_emul.c optional compat_linux compat/linux/linux_errno.c optional compat_linux compat/linux/linux_file.c optional compat_linux compat/linux/linux_fork.c optional compat_linux compat/linux/linux_futex.c optional compat_linux compat/linux/linux_getcwd.c optional compat_linux compat/linux/linux_ioctl.c optional compat_linux compat/linux/linux_ipc.c optional compat_linux compat/linux/linux_mib.c optional compat_linux compat/linux/linux_misc.c optional compat_linux compat/linux/linux_mmap.c optional compat_linux compat/linux/linux_signal.c optional compat_linux compat/linux/linux_socket.c optional compat_linux compat/linux/linux_stats.c optional compat_linux compat/linux/linux_sysctl.c optional compat_linux compat/linux/linux_time.c optional compat_linux compat/linux/linux_timer.c optional compat_linux compat/linux/linux_uid16.c optional compat_linux compat/linux/linux_util.c optional compat_linux compat/linux/linux_vdso.c optional compat_linux compat/linux/linux.c optional compat_linux compat/ndis/kern_ndis.c optional ndisapi pci compat/ndis/kern_windrv.c optional ndisapi pci compat/ndis/subr_hal.c optional ndisapi pci compat/ndis/subr_ndis.c optional ndisapi pci compat/ndis/subr_ntoskrnl.c optional ndisapi pci compat/ndis/subr_pe.c optional ndisapi pci compat/ndis/subr_usbd.c optional ndisapi pci compat/ndis/winx32_wrap.S optional ndisapi pci bf_enc.o optional crypto | ipsec | ipsec_support \ dependency "$S/crypto/blowfish/arch/i386/bf_enc.S $S/crypto/blowfish/arch/i386/bf_enc_586.S $S/crypto/blowfish/arch/i386/bf_enc_686.S" \ compile-with "${CC} -c -I$S/crypto/blowfish/arch/i386 ${ASM_CFLAGS} ${WERROR} ${.IMPSRC}" \ no-implicit-rule crypto/aesni/aeskeys_i386.S optional aesni crypto/aesni/aesni.c optional aesni aesni_ghash.o optional aesni \ dependency "$S/crypto/aesni/aesni_ghash.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes -mpclmul ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_ghash.o" aesni_wrap.o optional aesni \ dependency "$S/crypto/aesni/aesni_wrap.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${NO_WCAST_QUAL} ${PROF} -mmmx -msse -msse4 -maes ${.IMPSRC}" \ no-implicit-rule \ clean "aesni_wrap.o" crypto/des/arch/i386/des_enc.S optional crypto | ipsec | ipsec_support | netsmb intel_sha1.o optional aesni \ dependency "$S/crypto/aesni/intel_sha1.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha1.o" intel_sha256.o optional aesni \ dependency "$S/crypto/aesni/intel_sha256.c" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc} ${WERROR} ${PROF} -mmmx -msse -msse4 -msha ${.IMPSRC}" \ no-implicit-rule \ clean "intel_sha256.o" crypto/via/padlock.c optional padlock crypto/via/padlock_cipher.c optional padlock crypto/via/padlock_hash.c optional padlock dev/acpica/acpi_pci.c optional acpi pci dev/acpica/acpi_pci_link.c optional acpi pci dev/acpica/acpi_pcib.c optional acpi pci dev/acpica/acpi_pcib_acpi.c optional acpi pci dev/acpica/acpi_pcib_pci.c optional acpi pci dev/advansys/adv_isa.c optional adv isa dev/agp/agp_ali.c optional agp dev/agp/agp_amd.c optional agp dev/agp/agp_amd64.c optional agp dev/agp/agp_ati.c optional agp dev/agp/agp_i810.c optional agp dev/agp/agp_intel.c optional agp dev/agp/agp_nvidia.c optional agp dev/agp/agp_sis.c optional agp dev/agp/agp_via.c optional agp dev/aic/aic_isa.c optional aic isa dev/amdsbwd/amdsbwd.c optional amdsbwd dev/amdsmn/amdsmn.c optional amdsmn | amdtemp dev/amdtemp/amdtemp.c optional amdtemp dev/arcmsr/arcmsr.c optional arcmsr pci dev/asmc/asmc.c optional asmc isa dev/atkbdc/atkbd.c optional atkbd atkbdc dev/atkbdc/atkbd_atkbdc.c optional atkbd atkbdc dev/atkbdc/atkbdc.c optional atkbdc dev/atkbdc/atkbdc_isa.c optional atkbdc isa dev/atkbdc/atkbdc_subr.c optional atkbdc dev/atkbdc/psm.c optional psm atkbdc dev/bxe/bxe.c optional bxe pci dev/bxe/bxe_stats.c optional bxe pci dev/bxe/bxe_debug.c optional bxe pci dev/bxe/ecore_sp.c optional bxe pci dev/bxe/bxe_elink.c optional bxe pci dev/bxe/57710_init_values.c optional bxe pci dev/bxe/57711_init_values.c optional bxe pci dev/bxe/57712_init_values.c optional bxe pci dev/ce/ceddk.c optional ce dev/ce/if_ce.c optional ce dev/ce/tau32-ddk.c optional ce \ compile-with "${NORMAL_C} ${NO_WCONSTANT_CONVERSION}" dev/cm/if_cm_isa.c optional cm isa dev/coretemp/coretemp.c optional coretemp dev/cp/cpddk.c optional cp dev/cp/if_cp.c optional cp dev/cpuctl/cpuctl.c optional cpuctl dev/ctau/ctau.c optional ctau dev/ctau/ctddk.c optional ctau dev/ctau/if_ct.c optional ctau dev/cx/csigma.c optional cx dev/cx/cxddk.c optional cx dev/cx/if_cx.c optional cx dev/dpms/dpms.c optional dpms dev/ed/if_ed_3c503.c optional ed isa ed_3c503 dev/ed/if_ed_isa.c optional ed isa dev/ed/if_ed_wd80x3.c optional ed isa dev/ed/if_ed_hpp.c optional ed isa ed_hpp dev/ed/if_ed_sic.c optional ed isa ed_sic dev/ep/elink.c optional ep dev/fb/fb.c optional fb | vga dev/fb/s3_pci.c optional s3pci dev/fb/vesa.c optional vga vesa dev/fb/vga.c optional vga dev/fdc/fdc.c optional fdc dev/fdc/fdc_acpi.c optional fdc dev/fdc/fdc_isa.c optional fdc isa dev/fdc/fdc_pccard.c optional fdc pccard dev/fe/if_fe_isa.c optional fe isa dev/glxiic/glxiic.c optional glxiic dev/glxsb/glxsb.c optional glxsb dev/glxsb/glxsb_hash.c optional glxsb dev/gpio/bytgpio.c optional bytgpio dev/gpio/chvgpio.c optional chvgpio dev/hpt27xx/hpt27xx_os_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_osm_bsd.c optional hpt27xx dev/hpt27xx/hpt27xx_config.c optional hpt27xx dev/hptmv/entry.c optional hptmv dev/hptmv/mv.c optional hptmv dev/hptmv/gui_lib.c optional hptmv dev/hptmv/hptproc.c optional hptmv dev/hptmv/ioctl.c optional hptmv dev/hptnr/hptnr_os_bsd.c optional hptnr dev/hptnr/hptnr_osm_bsd.c optional hptnr dev/hptnr/hptnr_config.c optional hptnr dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc dev/hwpmc/hwpmc_pentium.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc dev/hwpmc/hwpmc_ppro.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/pcib/vmbus_pcib.c optional hyperv pci dev/hyperv/netvsc/hn_nvs.c optional hyperv dev/hyperv/netvsc/hn_rndis.c optional hyperv dev/hyperv/netvsc/if_hn.c optional hyperv dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c optional hyperv dev/hyperv/utilities/hv_kvp.c optional hyperv dev/hyperv/utilities/hv_snapshot.c optional hyperv dev/hyperv/utilities/vmbus_heartbeat.c optional hyperv dev/hyperv/utilities/vmbus_ic.c optional hyperv dev/hyperv/utilities/vmbus_shutdown.c optional hyperv dev/hyperv/utilities/vmbus_timesync.c optional hyperv dev/hyperv/vmbus/hyperv.c optional hyperv dev/hyperv/vmbus/hyperv_busdma.c optional hyperv dev/hyperv/vmbus/vmbus.c optional hyperv pci dev/hyperv/vmbus/vmbus_br.c optional hyperv dev/hyperv/vmbus/vmbus_chan.c optional hyperv dev/hyperv/vmbus/vmbus_et.c optional hyperv dev/hyperv/vmbus/vmbus_if.m optional hyperv dev/hyperv/vmbus/vmbus_res.c optional hyperv dev/hyperv/vmbus/vmbus_xact.c optional hyperv dev/hyperv/vmbus/i386/hyperv_machdep.c optional hyperv dev/hyperv/vmbus/i386/vmbus_vector.S optional hyperv dev/ichwd/ichwd.c optional ichwd dev/if_ndis/if_ndis.c optional ndis dev/if_ndis/if_ndis_pccard.c optional ndis pccard dev/if_ndis/if_ndis_pci.c optional ndis cardbus | ndis pci dev/if_ndis/if_ndis_usb.c optional ndis usb dev/imcsmb/imcsmb.c optional imcsmb dev/imcsmb/imcsmb_pci.c optional imcsmb pci dev/intel/spi.c optional intelspi dev/io/iodev.c optional io dev/ipmi/ipmi.c optional ipmi dev/ipmi/ipmi_acpi.c optional ipmi acpi dev/ipmi/ipmi_isa.c optional ipmi isa dev/ipmi/ipmi_kcs.c optional ipmi dev/ipmi/ipmi_smic.c optional ipmi dev/ipmi/ipmi_smbus.c optional ipmi smbus dev/ipmi/ipmi_smbios.c optional ipmi dev/ipmi/ipmi_ssif.c optional ipmi smbus dev/ipmi/ipmi_pci.c optional ipmi pci dev/ipmi/ipmi_linux.c optional ipmi compat_linux dev/le/if_le_isa.c optional le isa dev/mse/mse.c optional mse dev/mse/mse_isa.c optional mse isa dev/nctgpio/nctgpio.c optional nctgpio dev/nfe/if_nfe.c optional nfe pci dev/ntb/if_ntb/if_ntb.c optional if_ntb dev/ntb/ntb_transport.c optional ntb_transport | if_ntb dev/ntb/ntb.c optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_if.m optional ntb | ntb_transport | if_ntb | ntb_hw_intel | ntb_hw_plx | ntb_hw dev/ntb/ntb_hw/ntb_hw_intel.c optional ntb_hw_intel | ntb_hw dev/ntb/ntb_hw/ntb_hw_plx.c optional ntb_hw_plx | ntb_hw dev/nvd/nvd.c optional nvd nvme dev/nvme/nvme.c optional nvme dev/nvme/nvme_ctrlr.c optional nvme dev/nvme/nvme_ctrlr_cmd.c optional nvme dev/nvme/nvme_ns.c optional nvme dev/nvme/nvme_ns_cmd.c optional nvme dev/nvme/nvme_qpair.c optional nvme dev/nvme/nvme_sysctl.c optional nvme dev/nvme/nvme_test.c optional nvme dev/nvme/nvme_util.c optional nvme dev/nvram/nvram.c optional nvram isa dev/ofw/ofwpci.c optional fdt pci dev/pcf/pcf_isa.c optional pcf dev/random/ivy.c optional rdrand_rng dev/random/nehemiah.c optional padlock_rng dev/sbni/if_sbni.c optional sbni dev/sbni/if_sbni_isa.c optional sbni isa dev/sbni/if_sbni_pci.c optional sbni pci dev/sio/sio.c optional sio dev/sio/sio_isa.c optional sio isa dev/sio/sio_pccard.c optional sio pccard dev/sio/sio_pci.c optional sio pci dev/sio/sio_puc.c optional sio puc dev/speaker/spkr.c optional speaker dev/syscons/apm/apm_saver.c optional apm_saver apm dev/syscons/scterm-teken.c optional sc dev/syscons/scvesactl.c optional sc vga vesa dev/syscons/scvgarndr.c optional sc vga dev/syscons/scvtb.c optional sc dev/tpm/tpm.c optional tpm dev/tpm/tpm_acpi.c optional tpm acpi dev/tpm/tpm_isa.c optional tpm isa dev/uart/uart_cpu_x86.c optional uart dev/viawd/viawd.c optional viawd dev/vmware/vmxnet3/if_vmx.c optional vmx dev/acpica/acpi_if.m standard dev/acpica/acpi_hpet.c optional acpi dev/acpica/acpi_timer.c optional acpi dev/acpi_support/acpi_wmi_if.m standard dev/wbwd/wbwd.c optional wbwd dev/isci/isci.c optional isci dev/isci/isci_controller.c optional isci dev/isci/isci_domain.c optional isci dev/isci/isci_interrupt.c optional isci dev/isci/isci_io_request.c optional isci dev/isci/isci_logger.c optional isci dev/isci/isci_oem_parameters.c optional isci dev/isci/isci_remote_device.c optional isci dev/isci/isci_sysctl.c optional isci dev/isci/isci_task_request.c optional isci dev/isci/isci_timer.c optional isci dev/isci/scil/sati.c optional isci dev/isci/scil/sati_abort_task_set.c optional isci dev/isci/scil/sati_atapi.c optional isci dev/isci/scil/sati_device.c optional isci dev/isci/scil/sati_inquiry.c optional isci dev/isci/scil/sati_log_sense.c optional isci dev/isci/scil/sati_lun_reset.c optional isci dev/isci/scil/sati_mode_pages.c optional isci dev/isci/scil/sati_mode_select.c optional isci dev/isci/scil/sati_mode_sense.c optional isci dev/isci/scil/sati_mode_sense_10.c optional isci dev/isci/scil/sati_mode_sense_6.c optional isci dev/isci/scil/sati_move.c optional isci dev/isci/scil/sati_passthrough.c optional isci dev/isci/scil/sati_read.c optional isci dev/isci/scil/sati_read_buffer.c optional isci dev/isci/scil/sati_read_capacity.c optional isci dev/isci/scil/sati_reassign_blocks.c optional isci dev/isci/scil/sati_report_luns.c optional isci dev/isci/scil/sati_request_sense.c optional isci dev/isci/scil/sati_start_stop_unit.c optional isci dev/isci/scil/sati_synchronize_cache.c optional isci dev/isci/scil/sati_test_unit_ready.c optional isci dev/isci/scil/sati_unmap.c optional isci dev/isci/scil/sati_util.c optional isci dev/isci/scil/sati_verify.c optional isci dev/isci/scil/sati_write.c optional isci dev/isci/scil/sati_write_and_verify.c optional isci dev/isci/scil/sati_write_buffer.c optional isci dev/isci/scil/sati_write_long.c optional isci dev/isci/scil/sci_abstract_list.c optional isci dev/isci/scil/sci_base_controller.c optional isci dev/isci/scil/sci_base_domain.c optional isci dev/isci/scil/sci_base_iterator.c optional isci dev/isci/scil/sci_base_library.c optional isci dev/isci/scil/sci_base_logger.c optional isci dev/isci/scil/sci_base_memory_descriptor_list.c optional isci dev/isci/scil/sci_base_memory_descriptor_list_decorator.c optional isci dev/isci/scil/sci_base_object.c optional isci dev/isci/scil/sci_base_observer.c optional isci dev/isci/scil/sci_base_phy.c optional isci dev/isci/scil/sci_base_port.c optional isci dev/isci/scil/sci_base_remote_device.c optional isci dev/isci/scil/sci_base_request.c optional isci dev/isci/scil/sci_base_state_machine.c optional isci dev/isci/scil/sci_base_state_machine_logger.c optional isci dev/isci/scil/sci_base_state_machine_observer.c optional isci dev/isci/scil/sci_base_subject.c optional isci dev/isci/scil/sci_util.c optional isci dev/isci/scil/scic_sds_controller.c optional isci dev/isci/scil/scic_sds_library.c optional isci dev/isci/scil/scic_sds_pci.c optional isci dev/isci/scil/scic_sds_phy.c optional isci dev/isci/scil/scic_sds_port.c optional isci dev/isci/scil/scic_sds_port_configuration_agent.c optional isci dev/isci/scil/scic_sds_remote_device.c optional isci dev/isci/scil/scic_sds_remote_node_context.c optional isci dev/isci/scil/scic_sds_remote_node_table.c optional isci dev/isci/scil/scic_sds_request.c optional isci dev/isci/scil/scic_sds_sgpio.c optional isci dev/isci/scil/scic_sds_smp_remote_device.c optional isci dev/isci/scil/scic_sds_smp_request.c optional isci dev/isci/scil/scic_sds_ssp_request.c optional isci dev/isci/scil/scic_sds_stp_packet_request.c optional isci dev/isci/scil/scic_sds_stp_remote_device.c optional isci dev/isci/scil/scic_sds_stp_request.c optional isci dev/isci/scil/scic_sds_unsolicited_frame_control.c optional isci dev/isci/scil/scif_sas_controller.c optional isci dev/isci/scil/scif_sas_controller_state_handlers.c optional isci dev/isci/scil/scif_sas_controller_states.c optional isci dev/isci/scil/scif_sas_domain.c optional isci dev/isci/scil/scif_sas_domain_state_handlers.c optional isci dev/isci/scil/scif_sas_domain_states.c optional isci dev/isci/scil/scif_sas_high_priority_request_queue.c optional isci dev/isci/scil/scif_sas_internal_io_request.c optional isci dev/isci/scil/scif_sas_io_request.c optional isci dev/isci/scil/scif_sas_io_request_state_handlers.c optional isci dev/isci/scil/scif_sas_io_request_states.c optional isci dev/isci/scil/scif_sas_library.c optional isci dev/isci/scil/scif_sas_remote_device.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_ready_substates.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substate_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_starting_substates.c optional isci dev/isci/scil/scif_sas_remote_device_state_handlers.c optional isci dev/isci/scil/scif_sas_remote_device_states.c optional isci dev/isci/scil/scif_sas_request.c optional isci dev/isci/scil/scif_sas_smp_activity_clear_affiliation.c optional isci dev/isci/scil/scif_sas_smp_io_request.c optional isci dev/isci/scil/scif_sas_smp_phy.c optional isci dev/isci/scil/scif_sas_smp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_io_request.c optional isci dev/isci/scil/scif_sas_stp_remote_device.c optional isci dev/isci/scil/scif_sas_stp_task_request.c optional isci dev/isci/scil/scif_sas_task_request.c optional isci dev/isci/scil/scif_sas_task_request_state_handlers.c optional isci dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci i386/acpica/acpi_machdep.c optional acpi acpi_wakecode.o optional acpi \ - dependency "$S/i386/acpica/acpi_wakecode.S assym.s" \ + dependency "$S/i386/acpica/acpi_wakecode.S assym.inc" \ compile-with "${NORMAL_S}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.o" acpi_wakecode.bin optional acpi \ dependency "acpi_wakecode.o" \ compile-with "${OBJCOPY} -S -O binary acpi_wakecode.o ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.bin" acpi_wakecode.h optional acpi \ dependency "acpi_wakecode.bin" \ compile-with "file2c -sx 'static char wakecode[] = {' '};' < acpi_wakecode.bin > ${.TARGET}" \ no-obj no-implicit-rule before-depend \ clean "acpi_wakecode.h" acpi_wakedata.h optional acpi \ dependency "acpi_wakecode.o" \ compile-with '${NM} -n --defined-only acpi_wakecode.o | while read offset dummy what; do echo "#define $${what} 0x$${offset}"; done > ${.TARGET}' \ no-obj no-implicit-rule before-depend \ clean "acpi_wakedata.h" # i386/bios/apm.c optional apm i386/bios/smapi.c optional smapi i386/bios/smapi_bios.S optional smapi i386/cloudabi32/cloudabi32_sysvec.c optional compat_cloudabi32 #i386/i386/apic_vector.s optional apic i386/i386/atomic.c standard \ compile-with "${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} ${.IMPSRC}" i386/i386/bios.c standard i386/i386/bioscall.s standard i386/i386/bpf_jit_machdep.c optional bpf_jitter i386/i386/db_disasm.c optional ddb i386/i386/db_interface.c optional ddb i386/i386/db_trace.c optional ddb i386/i386/elan-mmcr.c optional cpu_elan | cpu_soekris i386/i386/elf_machdep.c standard i386/i386/exception.s standard i386/i386/gdb_machdep.c optional gdb i386/i386/geode.c optional cpu_geode i386/i386/in_cksum.c optional inet | inet6 i386/i386/initcpu.c standard i386/i386/io.c optional io i386/i386/k6_mem.c optional mem i386/i386/locore.s standard no-obj i386/i386/longrun.c optional cpu_enable_longrun i386/i386/machdep.c standard i386/i386/mem.c optional mem i386/i386/minidump_machdep.c standard i386/i386/mp_clock.c optional smp i386/i386/mp_machdep.c optional smp i386/i386/mpboot.s optional smp i386/i386/npx.c standard i386/i386/perfmon.c optional perfmon i386/i386/pmap.c standard i386/i386/prof_machdep.c optional profiling-routine i386/i386/ptrace_machdep.c standard i386/i386/sigtramp.s standard i386/i386/support.s standard i386/i386/swtch.s standard i386/i386/sys_machdep.c standard i386/i386/trap.c standard i386/i386/uio_machdep.c standard i386/i386/vm86.c standard i386/i386/vm_machdep.c standard i386/ibcs2/ibcs2_errno.c optional ibcs2 i386/ibcs2/ibcs2_fcntl.c optional ibcs2 i386/ibcs2/ibcs2_ioctl.c optional ibcs2 i386/ibcs2/ibcs2_ipc.c optional ibcs2 i386/ibcs2/ibcs2_isc.c optional ibcs2 i386/ibcs2/ibcs2_isc_sysent.c optional ibcs2 i386/ibcs2/ibcs2_misc.c optional ibcs2 i386/ibcs2/ibcs2_msg.c optional ibcs2 i386/ibcs2/ibcs2_other.c optional ibcs2 i386/ibcs2/ibcs2_signal.c optional ibcs2 i386/ibcs2/ibcs2_socksys.c optional ibcs2 i386/ibcs2/ibcs2_stat.c optional ibcs2 i386/ibcs2/ibcs2_sysent.c optional ibcs2 i386/ibcs2/ibcs2_sysi86.c optional ibcs2 i386/ibcs2/ibcs2_sysvec.c optional ibcs2 i386/ibcs2/ibcs2_util.c optional ibcs2 i386/ibcs2/ibcs2_xenix.c optional ibcs2 i386/ibcs2/ibcs2_xenix_sysent.c optional ibcs2 i386/ibcs2/imgact_coff.c optional ibcs2 i386/linux/imgact_linux.c optional compat_linux i386/linux/linux_dummy.c optional compat_linux i386/linux/linux_machdep.c optional compat_linux i386/linux/linux_ptrace.c optional compat_linux i386/linux/linux_support.s optional compat_linux \ dependency "linux_assym.h" i386/linux/linux_sysent.c optional compat_linux i386/linux/linux_sysvec.c optional compat_linux i386/pci/pci_cfgreg.c optional pci i386/pci/pci_pir.c optional pci isa/syscons_isa.c optional sc isa/vga_isa.c optional vga kern/kern_clocksource.c standard kern/imgact_aout.c optional compat_aout kern/imgact_gzip.c optional gzip kern/subr_sfbuf.c standard libkern/divdi3.c standard libkern/ffsll.c standard libkern/flsll.c standard libkern/memmove.c standard libkern/memset.c standard libkern/moddi3.c standard libkern/qdivrem.c standard libkern/ucmpdi2.c standard libkern/udivdi3.c standard libkern/umoddi3.c standard libkern/x86/crc32_sse42.c standard # # x86 real mode BIOS support, required by dpms/pci/vesa # compat/x86bios/x86bios.c optional x86bios | dpms | pci | vesa # # bvm console # dev/bvm/bvm_console.c optional bvmconsole dev/bvm/bvm_dbg.c optional bvmdebug # # x86 shared code between IA32 and AMD64 architectures # x86/acpica/OsdEnvironment.c optional acpi x86/acpica/acpi_apm.c optional acpi x86/acpica/acpi_wakeup.c optional acpi x86/acpica/madt.c optional acpi apic x86/acpica/srat.c optional acpi x86/bios/smbios.c optional smbios x86/bios/vpd.c optional vpd x86/cpufreq/est.c optional cpufreq x86/cpufreq/hwpstate.c optional cpufreq x86/cpufreq/p4tcc.c optional cpufreq x86/cpufreq/powernow.c optional cpufreq x86/cpufreq/smist.c optional cpufreq x86/iommu/busdma_dmar.c optional acpi acpi_dmar pci x86/iommu/intel_ctx.c optional acpi acpi_dmar pci x86/iommu/intel_drv.c optional acpi acpi_dmar pci x86/iommu/intel_fault.c optional acpi acpi_dmar pci x86/iommu/intel_gas.c optional acpi acpi_dmar pci x86/iommu/intel_idpgtbl.c optional acpi acpi_dmar pci x86/iommu/intel_intrmap.c optional acpi acpi_dmar pci x86/iommu/intel_qi.c optional acpi acpi_dmar pci x86/iommu/intel_quirks.c optional acpi acpi_dmar pci x86/iommu/intel_utils.c optional acpi acpi_dmar pci x86/isa/atpic.c optional atpic x86/isa/atrtc.c standard x86/isa/clock.c standard x86/isa/elcr.c optional atpic | apic x86/isa/isa.c optional isa x86/isa/isa_dma.c optional isa x86/isa/nmi.c standard x86/isa/orm.c optional isa x86/pci/pci_bus.c optional pci x86/pci/qpi.c optional pci x86/x86/autoconf.c standard x86/x86/bus_machdep.c standard x86/x86/busdma_bounce.c standard x86/x86/busdma_machdep.c standard x86/x86/cpu_machdep.c standard x86/x86/dump_machdep.c standard x86/x86/fdt_machdep.c optional fdt x86/x86/identcpu.c standard x86/x86/intr_machdep.c standard x86/x86/io_apic.c optional apic x86/x86/legacy.c standard x86/x86/local_apic.c optional apic x86/x86/mca.c standard x86/x86/x86_mem.c optional mem x86/x86/mptable.c optional apic x86/x86/mptable_pci.c optional apic pci x86/x86/mp_x86.c optional smp x86/x86/mp_watchdog.c optional mp_watchdog smp x86/x86/msi.c optional apic pci x86/x86/nexus.c standard x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/pvclock.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm x86/xen/xen_intr.c optional xenhvm x86/xen/xen_apic.c optional xenhvm x86/xen/xenpv.c optional xenhvm x86/xen/xen_nexus.c optional xenhvm x86/xen/xen_msi.c optional xenhvm Index: head/sys/conf/kern.post.mk =================================================================== --- head/sys/conf/kern.post.mk (revision 331253) +++ head/sys/conf/kern.post.mk (revision 331254) @@ -1,411 +1,411 @@ # $FreeBSD$ # Part of a unified Makefile for building kernels. This part includes all # the definitions that need to be after all the % directives except %RULES # and ones that act like they are part of %RULES. # # Most make variables should not be defined in this file. Instead, they # should be defined in the kern.pre.mk so that port makefiles can # override or augment them. # In case the config had a makeoptions DESTDIR... .if defined(DESTDIR) MKMODULESENV+= DESTDIR="${DESTDIR}" .endif SYSDIR?= ${S:C;^[^/];${.CURDIR}/&;:tA} MKMODULESENV+= KERNBUILDDIR="${.CURDIR}" SYSDIR="${SYSDIR}" .if defined(CONF_CFLAGS) MKMODULESENV+= CONF_CFLAGS="${CONF_CFLAGS}" .endif .if defined(WITH_CTF) MKMODULESENV+= WITH_CTF="${WITH_CTF}" .endif .if defined(WITH_EXTRA_TCP_STACKS) MKMODULESENV+= WITH_EXTRA_TCP_STACKS="${WITH_EXTRA_TCP_STACKS}" .endif # Allow overriding the kernel debug directory, so kernel and user debug may be # installed in different directories. Setting it to "" restores the historical # behavior of installing debug files in the kernel directory. KERN_DEBUGDIR?= ${DEBUGDIR} .MAIN: all .for target in all clean cleandepend cleandir clobber depend install \ ${_obj} reinstall tags ${target}: kernel-${target} .if !defined(MODULES_WITH_WORLD) && !defined(NO_MODULES) && exists($S/modules) ${target}: modules-${target} modules-${target}: cd $S/modules; ${MKMODULESENV} ${MAKE} \ ${target:S/^reinstall$/install/:S/^clobber$/cleandir/} .endif .endfor # Handle ports (as defined by the user) that build kernel modules .if !defined(NO_MODULES) && defined(PORTS_MODULES) # # The ports tree needs some environment variables defined to match the new kernel # # Ports search for some dependencies in PATH, so add the location of the installed files LOCALBASE?= /usr/local # SRC_BASE is how the ports tree refers to the location of the base source files .if !defined(SRC_BASE) SRC_BASE= ${SYSDIR:H:tA} .endif # OSVERSION is used by some ports to determine build options .if !defined(OSRELDATE) # Definition copied from src/Makefile.inc1 OSRELDATE!= awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \ ${MAKEOBJDIRPREFIX}${SRC_BASE}/include/osreldate.h .endif # Keep the related ports builds in the obj directory so that they are only rebuilt once per kernel build WRKDIRPREFIX?= ${.OBJDIR} PORTSMODULESENV=\ env \ -u CC \ -u CXX \ -u CPP \ -u MAKESYSPATH \ -u MAKEOBJDIR \ MAKEFLAGS="${MAKEFLAGS:M*:tW:S/^-m /-m_/g:S/ -m / -m_/g:tw:N-m_*}" \ SYSDIR=${SYSDIR} \ PATH=${PATH}:${LOCALBASE}/bin:${LOCALBASE}/sbin \ SRC_BASE=${SRC_BASE} \ OSVERSION=${OSRELDATE} \ WRKDIRPREFIX=${WRKDIRPREFIX} # The WRKDIR needs to be cleaned before building, and trying to change the target # with a :C pattern below results in install -> instclean all: .for __i in ${PORTS_MODULES} @${ECHO} "===> Ports module ${__i} (all)" cd $${PORTSDIR:-/usr/ports}/${__i}; ${PORTSMODULESENV} ${MAKE} -B clean build .endfor .for __target in install reinstall clean ${__target}: ports-${__target} ports-${__target}: .for __i in ${PORTS_MODULES} @${ECHO} "===> Ports module ${__i} (${__target})" cd $${PORTSDIR:-/usr/ports}/${__i}; ${PORTSMODULESENV} ${MAKE} -B ${__target:C/(re)?install/deinstall reinstall/} .endfor .endfor .endif .ORDER: kernel-install modules-install beforebuild: .PHONY kernel-all: beforebuild .WAIT ${KERNEL_KO} ${KERNEL_EXTRA} kernel-cleandir: kernel-clean kernel-cleandepend kernel-clobber: find . -maxdepth 1 ! -type d ! -name version -delete kernel-obj: .if !defined(MODULES_WITH_WORLD) && !defined(NO_MODULES) && exists($S/modules) modules: modules-all .if !defined(NO_MODULES_OBJ) modules-all modules-depend: modules-obj .endif .endif .if !defined(DEBUG) FULLKERNEL= ${KERNEL_KO} .else FULLKERNEL= ${KERNEL_KO}.full ${KERNEL_KO}: ${FULLKERNEL} ${KERNEL_KO}.debug ${OBJCOPY} --strip-debug --add-gnu-debuglink=${KERNEL_KO}.debug \ ${FULLKERNEL} ${.TARGET} ${KERNEL_KO}.debug: ${FULLKERNEL} ${OBJCOPY} --only-keep-debug ${FULLKERNEL} ${.TARGET} install.debug reinstall.debug: gdbinit cd ${.CURDIR}; ${MAKE} ${.TARGET:R} # Install gdbinit files for kernel debugging. gdbinit: grep -v '# XXX' ${S}/../tools/debugscripts/dot.gdbinit | \ sed "s:MODPATH:${.OBJDIR}/modules:" > .gdbinit cp ${S}/../tools/debugscripts/gdbinit.kernel ${.CURDIR} .if exists(${S}/../tools/debugscripts/gdbinit.${MACHINE_CPUARCH}) cp ${S}/../tools/debugscripts/gdbinit.${MACHINE_CPUARCH} \ ${.CURDIR}/gdbinit.machine .endif .endif ${FULLKERNEL}: ${SYSTEM_DEP} vers.o @rm -f ${.TARGET} @echo linking ${.TARGET} ${SYSTEM_LD} .if !empty(MD_ROOT_SIZE_CONFIGURED) && defined(MFS_IMAGE) @sh ${S}/tools/embed_mfs.sh ${.TARGET} ${MFS_IMAGE} .endif .if ${MK_CTF} != "no" @echo ${CTFMERGE} ${CTFFLAGS} -o ${.TARGET} ... @${CTFMERGE} ${CTFFLAGS} -o ${.TARGET} ${SYSTEM_OBJS} vers.o .endif .if !defined(DEBUG) ${OBJCOPY} --strip-debug ${.TARGET} .endif ${SYSTEM_LD_TAIL} -OBJS_DEPEND_GUESS+= assym.s vnode_if.h ${BEFORE_DEPEND:M*.h} \ +OBJS_DEPEND_GUESS+= assym.inc vnode_if.h ${BEFORE_DEPEND:M*.h} \ ${MFILES:T:S/.m$/.h/} .for mfile in ${MFILES} # XXX the low quality .m.o rules gnerated by config are normally used # instead of the .m.c rules here. ${mfile:T:S/.m$/.c/}: ${mfile} ${AWK} -f $S/tools/makeobjops.awk ${mfile} -c ${mfile:T:S/.m$/.h/}: ${mfile} ${AWK} -f $S/tools/makeobjops.awk ${mfile} -h .endfor kernel-clean: rm -f *.o *.so *.pico *.ko *.s eddep errs \ ${FULLKERNEL} ${KERNEL_KO} ${KERNEL_KO}.debug \ tags vers.c \ vnode_if.c vnode_if.h vnode_if_newproto.h vnode_if_typedef.h \ ${MFILES:T:S/.m$/.c/} ${MFILES:T:S/.m$/.h/} \ ${CLEAN} # This is a hack. BFD "optimizes" away dynamic mode if there are no # dynamic references. We could probably do a '-Bforcedynamic' mode like # in the a.out ld. For now, this works. HACK_EXTRA_FLAGS?= -shared hack.pico: Makefile :> hack.c ${CC} ${HACK_EXTRA_FLAGS} -nostdlib hack.c -o hack.pico rm -f hack.c -assym.s: $S/kern/genassym.sh genassym.o +assym.inc: $S/kern/genassym.sh genassym.o NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh genassym.o > ${.TARGET} genassym.o: $S/$M/$M/genassym.c ${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/$M/$M/genassym.c ${SYSTEM_OBJS} genassym.o vers.o: opt_global.h .if !empty(.MAKE.MODE:Unormal:Mmeta) && empty(.MAKE.MODE:Unormal:Mnofilemon) _meta_filemon= 1 .endif # Skip reading .depend when not needed to speed up tree-walks and simple # lookups. For install, only do this if no other targets are specified. # Also skip generating or including .depend.* files if in meta+filemon mode # since it will track dependencies itself. OBJS_DEPEND_GUESS is still used # for _meta_filemon but not for _SKIP_DEPEND. .if !defined(NO_SKIP_DEPEND) && \ ((!empty(.MAKEFLAGS:M-V) && empty(.MAKEFLAGS:M*DEP*)) || \ ${.TARGETS:M*obj} == ${.TARGETS} || \ ${.TARGETS:M*clean*} == ${.TARGETS} || \ ${.TARGETS:M*install*} == ${.TARGETS}) _SKIP_DEPEND= 1 .endif .if defined(_SKIP_DEPEND) || defined(_meta_filemon) .MAKE.DEPENDFILE= /dev/null .endif kernel-depend: .depend -SRCS= assym.s vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ +SRCS= assym.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \ ${SYSTEM_CFILES} ${GEN_CFILES} ${SFILES} \ ${MFILES:T:S/.m$/.h/} DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o DEPENDFILES= ${DEPENDOBJS:O:u:C/^/.depend./} .if ${MAKE_VERSION} < 20160220 DEPEND_MP?= -MP .endif .if defined(_SKIP_DEPEND) # Don't bother reading any .meta files ${DEPENDOBJS}: .NOMETA .depend: .NOMETA # Unset these to avoid looping/statting on them later. .undef DEPENDOBJS .undef DEPENDFILES .endif # defined(_SKIP_DEPEND) DEPEND_CFLAGS+= -MD ${DEPEND_MP} -MF.depend.${.TARGET} DEPEND_CFLAGS+= -MT${.TARGET} .if !defined(_meta_filemon) .if !empty(DEPEND_CFLAGS) # Only add in DEPEND_CFLAGS for CFLAGS on files we expect from DEPENDOBJS # as those are the only ones we will include. DEPEND_CFLAGS_CONDITION= "${DEPENDOBJS:M${.TARGET}}" != "" CFLAGS+= ${${DEPEND_CFLAGS_CONDITION}:?${DEPEND_CFLAGS}:} .endif .for __depend_obj in ${DEPENDFILES} .if ${MAKE_VERSION} < 20160220 .sinclude "${.OBJDIR}/${__depend_obj}" .else .dinclude "${.OBJDIR}/${__depend_obj}" .endif .endfor .endif # !defined(_meta_filemon) # Always run 'make depend' to generate dependencies early and to avoid the # need for manually running it. For the kernel this is mostly a NOP since # all dependencies are correctly added or accounted for. This is mostly to # ensure downstream uses of kernel-depend are handled. beforebuild: kernel-depend # Guess some dependencies for when no ${DEPENDFILE}.OBJ is generated yet. # For meta+filemon the .meta file is checked for since it is the dependency # file used. .for __obj in ${DEPENDOBJS:O:u} .if defined(_meta_filemon) _depfile= ${.OBJDIR}/${__obj}.meta .else _depfile= ${.OBJDIR}/.depend.${__obj} .endif .if !exists(${_depfile}) .if ${SYSTEM_OBJS:M${__obj}} ${__obj}: ${OBJS_DEPEND_GUESS} .endif ${__obj}: ${OBJS_DEPEND_GUESS.${__obj}} .elif defined(_meta_filemon) # For meta mode we still need to know which file to depend on to avoid # ambiguous suffix transformation rules from .PATH. Meta mode does not # use .depend files. We really only need source files, not headers since # they are typically in SRCS/beforebuild already. For target-specific # guesses do include headers though since they may not be in SRCS. .if ${SYSTEM_OBJS:M${__obj}} ${__obj}: ${OBJS_DEPEND_GUESS:N*.h} .endif ${__obj}: ${OBJS_DEPEND_GUESS.${__obj}} .endif # !exists(${_depfile}) .endfor .NOPATH: .depend ${DEPENDFILES} .depend: .PRECIOUS ${SRCS} _ILINKS= machine .if ${MACHINE} != ${MACHINE_CPUARCH} && ${MACHINE} != "arm64" _ILINKS+= ${MACHINE_CPUARCH} .endif .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" _ILINKS+= x86 .endif # Ensure that the link exists without depending on it when it exists. .for _link in ${_ILINKS} .if !exists(${.OBJDIR}/${_link}) ${SRCS} ${CLEAN:M*.o}: ${_link} .endif .endfor ${_ILINKS}: @case ${.TARGET} in \ machine) \ path=${S}/${MACHINE}/include ;; \ *) \ path=${S}/${.TARGET}/include ;; \ esac ; \ ${ECHO} ${.TARGET} "->" $$path ; \ ln -fns $$path ${.TARGET} # .depend needs include links so we remove them only together. kernel-cleandepend: .PHONY rm -f .depend .depend.* ${_ILINKS} kernel-tags: @[ -f .depend ] || { echo "you must make depend first"; exit 1; } sh $S/conf/systags.sh kernel-install: .PHONY @if [ ! -f ${KERNEL_KO} ] ; then \ echo "You must build a kernel first." ; \ exit 1 ; \ fi .if exists(${DESTDIR}${KODIR}) -thiskernel=`sysctl -n kern.bootfile` ; \ if [ ! "`dirname "$$thiskernel"`" -ef ${DESTDIR}${KODIR} ] ; then \ chflags -R noschg ${DESTDIR}${KODIR} ; \ rm -rf ${DESTDIR}${KODIR} ; \ rm -rf ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ; \ else \ if [ -d ${DESTDIR}${KODIR}.old ] ; then \ chflags -R noschg ${DESTDIR}${KODIR}.old ; \ rm -rf ${DESTDIR}${KODIR}.old ; \ fi ; \ mv ${DESTDIR}${KODIR} ${DESTDIR}${KODIR}.old ; \ if [ -n "${KERN_DEBUGDIR}" -a \ -d ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ]; then \ rm -rf ${DESTDIR}${KERN_DEBUGDIR}${KODIR}.old ; \ mv ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ${DESTDIR}${KERN_DEBUGDIR}${KODIR}.old ; \ fi ; \ sysctl kern.bootfile=${DESTDIR}${KODIR}.old/"`basename "$$thiskernel"`" ; \ fi .endif mkdir -p ${DESTDIR}${KODIR} ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}/ .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no" mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/ .endif .if defined(KERNEL_EXTRA_INSTALL) ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}/ .endif kernel-reinstall: @-chflags -R noschg ${DESTDIR}${KODIR} ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}/ .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no" ${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}/ .endif config.o env.o hints.o vers.o vnode_if.o: ${NORMAL_C} ${NORMAL_CTFCONVERT} .if ${MK_REPRODUCIBLE_BUILD} != "no" REPRO_FLAG="-r" .endif vers.c: $S/conf/newvers.sh $S/sys/param.h ${SYSTEM_DEP} MAKE="${MAKE}" sh $S/conf/newvers.sh ${REPRO_FLAG} ${KERN_IDENT} vnode_if.c: $S/tools/vnode_if.awk $S/kern/vnode_if.src ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -c vnode_if.h vnode_if_newproto.h vnode_if_typedef.h: $S/tools/vnode_if.awk \ $S/kern/vnode_if.src vnode_if.h: vnode_if_newproto.h vnode_if_typedef.h ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -h vnode_if_newproto.h: ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -p vnode_if_typedef.h: ${AWK} -f $S/tools/vnode_if.awk $S/kern/vnode_if.src -q .if ${MFS_IMAGE:Uno} != "no" .if empty(MD_ROOT_SIZE_CONFIGURED) # Generate an object file from the file system image to embed in the kernel # via linking. Make sure the contents are in the mfs section and rename the # start/end/size variables to __start_mfs, __stop_mfs, and mfs_size, # respectively. embedfs_${MFS_IMAGE:T:R}.o: ${MFS_IMAGE} ${OBJCOPY} --input-target binary \ --output-target ${EMBEDFS_FORMAT.${MACHINE_ARCH}} \ --binary-architecture ${EMBEDFS_ARCH.${MACHINE_ARCH}} \ ${MFS_IMAGE} ${.TARGET} ${OBJCOPY} \ --rename-section .data=mfs,contents,alloc,load,readonly,data \ --redefine-sym \ _binary_${MFS_IMAGE:C,[^[:alnum:]],_,g}_size=__mfs_root_size \ --redefine-sym \ _binary_${MFS_IMAGE:C,[^[:alnum:]],_,g}_start=mfs_root \ --redefine-sym \ _binary_${MFS_IMAGE:C,[^[:alnum:]],_,g}_end=mfs_root_end \ ${.TARGET} .endif .endif # XXX strictly, everything depends on Makefile because changes to ${PROF} # only appear there, but we don't handle that. .include "kern.mk" Index: head/sys/conf/kmod.mk =================================================================== --- head/sys/conf/kmod.mk (revision 331253) +++ head/sys/conf/kmod.mk (revision 331254) @@ -1,486 +1,486 @@ # From: @(#)bsd.prog.mk 5.26 (Berkeley) 6/25/91 # $FreeBSD$ # # The include file handles building and installing loadable # kernel modules. # # # +++ variables +++ # # CLEANFILES Additional files to remove for the clean and cleandir targets. # # EXPORT_SYMS A list of symbols that should be exported from the module, # or the name of a file containing a list of symbols, or YES # to export all symbols. If not defined, no symbols are # exported. # # KMOD The name of the kernel module to build. # # KMODDIR Base path for kernel modules (see kld(4)). [/boot/kernel] # # KMODOWN Module file owner. [${BINOWN}] # # KMODGRP Module file group. [${BINGRP}] # # KMODMODE Module file mode. [${BINMODE}] # # KMODLOAD Command to load a kernel module [/sbin/kldload] # # KMODUNLOAD Command to unload a kernel module [/sbin/kldunload] # # KMODISLOADED Command to check whether a kernel module is # loaded [/sbin/kldstat -q -n] # # PROG The name of the kernel module to build. # If not supplied, ${KMOD}.ko is used. # # SRCS List of source files. # # FIRMWS List of firmware images in format filename:shortname:version # # FIRMWARE_LICENSE # Set to the name of the license the user has to agree on in # order to use this firmware. See /usr/share/doc/legal # # DESTDIR The tree where the module gets installed. [not set] # # KERNBUILDDIR # Set to the location of the kernel build directory where # the opt_*.h files, .o's and kernel winds up. # # +++ targets +++ # # install: # install the kernel module; if the Makefile # does not itself define the target install, the targets # beforeinstall and afterinstall may also be used to cause # actions immediately before and after the install target # is executed. # # load: # Load a module. # # unload: # Unload a module. # # reload: # Unload if loaded, then load. # AWK?= awk KMODLOAD?= /sbin/kldload KMODUNLOAD?= /sbin/kldunload KMODISLOADED?= /sbin/kldstat -q -n OBJCOPY?= objcopy .include # Grab all the options for a kernel build. For backwards compat, we need to # do this after bsd.own.mk. .include "kern.opts.mk" .include .include "config.mk" # Search for kernel source tree in standard places. .for _dir in ${.CURDIR}/../.. ${.CURDIR}/../../.. /sys /usr/src/sys .if !defined(SYSDIR) && exists(${_dir}/kern/) SYSDIR= ${_dir:tA} .endif .endfor .if !defined(SYSDIR) || !exists(${SYSDIR}/kern/) .error "can't find kernel source tree" .endif .SUFFIXES: .out .o .c .cc .cxx .C .y .l .s .S .m # amd64 and mips use direct linking for kmod, all others use shared binaries .if ${MACHINE_CPUARCH} != amd64 && ${MACHINE_CPUARCH} != mips __KLD_SHARED=yes .else __KLD_SHARED=no .endif .if !empty(CFLAGS:M-O[23s]) && empty(CFLAGS:M-fno-strict-aliasing) CFLAGS+= -fno-strict-aliasing .endif WERROR?= -Werror CFLAGS+= ${WERROR} CFLAGS+= -D_KERNEL CFLAGS+= -DKLD_MODULE # Don't use any standard or source-relative include directories. NOSTDINC= -nostdinc CFLAGS:= ${CFLAGS:N-I*} ${NOSTDINC} ${INCLMAGIC} ${CFLAGS:M-I*} .if defined(KERNBUILDDIR) CFLAGS+= -DHAVE_KERNEL_OPTION_HEADERS -include ${KERNBUILDDIR}/opt_global.h .endif # Add -I paths for system headers. Individual module makefiles don't # need any -I paths for this. Similar defaults for .PATH can't be # set because there are no standard paths for non-headers. CFLAGS+= -I. -I${SYSDIR} CFLAGS.gcc+= -finline-limit=${INLINE_LIMIT} CFLAGS.gcc+= -fms-extensions CFLAGS.gcc+= --param inline-unit-growth=100 CFLAGS.gcc+= --param large-function-growth=1000 # Disallow common variables, and if we end up with commons from # somewhere unexpected, allocate storage for them in the module itself. CFLAGS+= -fno-common LDFLAGS+= -d -warn-common .if defined(LINKER_FEATURES) && ${LINKER_FEATURES:Mbuild-id} LDFLAGS+= -Wl,--build-id=sha1 .endif CFLAGS+= ${DEBUG_FLAGS} .if ${MACHINE_CPUARCH} == amd64 CFLAGS+= -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer .endif .if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "riscv" CFLAGS+= -fPIC .endif # Temporary workaround for PR 196407, which contains the fascinating details. # Don't allow clang to use fpu instructions or registers in kernel modules. .if ${MACHINE_CPUARCH} == arm .if ${COMPILER_VERSION} < 30800 CFLAGS.clang+= -mllvm -arm-use-movt=0 .else CFLAGS.clang+= -mno-movt .endif CFLAGS.clang+= -mfpu=none CFLAGS+= -funwind-tables .endif .if ${MACHINE_CPUARCH} == powerpc CFLAGS+= -mlongcall -fno-omit-frame-pointer .endif .if ${MACHINE_CPUARCH} == mips CFLAGS+= -G0 -fno-pic -mno-abicalls -mlong-calls .endif .if defined(DEBUG) || defined(DEBUG_FLAGS) CTFFLAGS+= -g .endif .if defined(FIRMWS) ${KMOD:S/$/.c/}: ${SYSDIR}/tools/fw_stub.awk ${AWK} -f ${SYSDIR}/tools/fw_stub.awk ${FIRMWS} -m${KMOD} -c${KMOD:S/$/.c/g} \ ${FIRMWARE_LICENSE:C/.+/-l/}${FIRMWARE_LICENSE} SRCS+= ${KMOD:S/$/.c/} CLEANFILES+= ${KMOD:S/$/.c/} .for _firmw in ${FIRMWS} ${_firmw:C/\:.*$/.fwo/:T}: ${_firmw:C/\:.*$//} @${ECHO} ${_firmw:C/\:.*$//} ${.ALLSRC:M*${_firmw:C/\:.*$//}} @if [ -e ${_firmw:C/\:.*$//} ]; then \ ${LD} -b binary --no-warn-mismatch ${_LDFLAGS} \ -m ${LD_EMULATION} -r -d \ -o ${.TARGET} ${_firmw:C/\:.*$//}; \ else \ ln -s ${.ALLSRC:M*${_firmw:C/\:.*$//}} ${_firmw:C/\:.*$//}; \ ${LD} -b binary --no-warn-mismatch ${_LDFLAGS} \ -m ${LD_EMULATION} -r -d \ -o ${.TARGET} ${_firmw:C/\:.*$//}; \ rm ${_firmw:C/\:.*$//}; \ fi OBJS+= ${_firmw:C/\:.*$/.fwo/:T} .endfor .endif # Conditionally include SRCS based on kernel config options. .for _o in ${KERN_OPTS} SRCS+=${SRCS.${_o}} .endfor OBJS+= ${SRCS:N*.h:R:S/$/.o/g} .if !defined(PROG) PROG= ${KMOD}.ko .endif .if !defined(DEBUG_FLAGS) FULLPROG= ${PROG} .else FULLPROG= ${PROG}.full ${PROG}: ${FULLPROG} ${PROG}.debug ${OBJCOPY} --strip-debug --add-gnu-debuglink=${PROG}.debug \ ${FULLPROG} ${.TARGET} ${PROG}.debug: ${FULLPROG} ${OBJCOPY} --only-keep-debug ${FULLPROG} ${.TARGET} .endif .if ${__KLD_SHARED} == yes ${FULLPROG}: ${KMOD}.kld ${LD} -m ${LD_EMULATION} -Bshareable -znotext ${_LDFLAGS} \ -o ${.TARGET} ${KMOD}.kld .if !defined(DEBUG_FLAGS) ${OBJCOPY} --strip-debug ${.TARGET} .endif .endif EXPORT_SYMS?= NO .if ${EXPORT_SYMS} != YES CLEANFILES+= export_syms .endif .if ${__KLD_SHARED} == yes ${KMOD}.kld: ${OBJS} .else ${FULLPROG}: ${OBJS} .endif ${LD} -m ${LD_EMULATION} ${_LDFLAGS} -r -d -o ${.TARGET} ${OBJS} .if ${MK_CTF} != "no" ${CTFMERGE} ${CTFFLAGS} -o ${.TARGET} ${OBJS} .endif .if defined(EXPORT_SYMS) .if ${EXPORT_SYMS} != YES .if ${EXPORT_SYMS} == NO :> export_syms .elif !exists(${.CURDIR}/${EXPORT_SYMS}) echo -n "${EXPORT_SYMS:@s@$s${.newline}@}" > export_syms .else grep -v '^#' < ${EXPORT_SYMS} > export_syms .endif ${AWK} -f ${SYSDIR}/conf/kmod_syms.awk ${.TARGET} \ export_syms | xargs -J% ${OBJCOPY} % ${.TARGET} .endif .endif # defined(EXPORT_SYMS) .if defined(PREFIX_SYMS) ${AWK} -v prefix=${PREFIX_SYMS} -f ${SYSDIR}/conf/kmod_syms_prefix.awk \ ${.TARGET} /dev/null | xargs -J% ${OBJCOPY} % ${.TARGET} .endif .if !defined(DEBUG_FLAGS) && ${__KLD_SHARED} == no ${OBJCOPY} --strip-debug ${.TARGET} .endif _ILINKS=machine .if ${MACHINE} != ${MACHINE_CPUARCH} && ${MACHINE} != "arm64" _ILINKS+=${MACHINE_CPUARCH} .endif .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" _ILINKS+=x86 .endif CLEANFILES+=${_ILINKS} all: ${PROG} beforedepend: ${_ILINKS} beforebuild: ${_ILINKS} # Ensure that the links exist without depending on it when it exists which # causes all the modules to be rebuilt when the directory pointed to changes. .for _link in ${_ILINKS} .if !exists(${.OBJDIR}/${_link}) OBJS_DEPEND_GUESS+= ${_link} .endif .endfor .NOPATH: ${_ILINKS} ${_ILINKS}: @case ${.TARGET} in \ machine) \ path=${SYSDIR}/${MACHINE}/include ;; \ *) \ path=${SYSDIR}/${.TARGET:T}/include ;; \ esac ; \ path=`(cd $$path && /bin/pwd)` ; \ ${ECHO} ${.TARGET:T} "->" $$path ; \ ln -fns $$path ${.TARGET:T} CLEANFILES+= ${PROG} ${KMOD}.kld ${OBJS} .if defined(DEBUG_FLAGS) CLEANFILES+= ${FULLPROG} ${PROG}.debug .endif .if !target(install) _INSTALLFLAGS:= ${INSTALLFLAGS} .for ie in ${INSTALLFLAGS_EDIT} _INSTALLFLAGS:= ${_INSTALLFLAGS${ie}} .endfor .if !target(realinstall) KERN_DEBUGDIR?= ${DEBUGDIR} realinstall: _kmodinstall .ORDER: beforeinstall _kmodinstall _kmodinstall: .PHONY ${INSTALL} -T release -o ${KMODOWN} -g ${KMODGRP} -m ${KMODMODE} \ ${_INSTALLFLAGS} ${PROG} ${DESTDIR}${KMODDIR}/ .if defined(DEBUG_FLAGS) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no" ${INSTALL} -T debug -o ${KMODOWN} -g ${KMODGRP} -m ${KMODMODE} \ ${_INSTALLFLAGS} ${PROG}.debug ${DESTDIR}${KERN_DEBUGDIR}${KMODDIR}/ .endif .include .if !defined(NO_XREF) afterinstall: _kldxref .ORDER: realinstall _kldxref .ORDER: _installlinks _kldxref _kldxref: .PHONY @if type kldxref >/dev/null 2>&1; then \ ${ECHO} kldxref ${DESTDIR}${KMODDIR}; \ kldxref ${DESTDIR}${KMODDIR}; \ fi .endif .endif # !target(realinstall) .endif # !target(install) .if !target(load) load: ${PROG} .PHONY ${KMODLOAD} -v ${.OBJDIR}/${PROG} .endif .if !target(unload) unload: .PHONY if ${KMODISLOADED} ${PROG} ; then ${KMODUNLOAD} -v ${PROG} ; fi .endif .if !target(reload) reload: unload load .PHONY .endif .if defined(KERNBUILDDIR) .PATH: ${KERNBUILDDIR} CFLAGS+= -I${KERNBUILDDIR} .for _src in ${SRCS:Mopt_*.h} CLEANFILES+= ${_src} .if !target(${_src}) ${_src}: ln -sf ${KERNBUILDDIR}/${_src} ${.TARGET} .endif .endfor .else .for _src in ${SRCS:Mopt_*.h} CLEANFILES+= ${_src} .if !target(${_src}) ${_src}: :> ${.TARGET} .endif .endfor .endif # Respect configuration-specific C flags. CFLAGS+= ${ARCH_FLAGS} ${CONF_CFLAGS} .if !empty(SRCS:Mvnode_if.c) CLEANFILES+= vnode_if.c vnode_if.c: ${SYSDIR}/tools/vnode_if.awk ${SYSDIR}/kern/vnode_if.src ${AWK} -f ${SYSDIR}/tools/vnode_if.awk ${SYSDIR}/kern/vnode_if.src -c .endif .if !empty(SRCS:Mvnode_if.h) CLEANFILES+= vnode_if.h vnode_if_newproto.h vnode_if_typedef.h vnode_if.h vnode_if_newproto.h vnode_if_typedef.h: ${SYSDIR}/tools/vnode_if.awk \ ${SYSDIR}/kern/vnode_if.src vnode_if.h: vnode_if_newproto.h vnode_if_typedef.h ${AWK} -f ${SYSDIR}/tools/vnode_if.awk ${SYSDIR}/kern/vnode_if.src -h vnode_if_newproto.h: ${AWK} -f ${SYSDIR}/tools/vnode_if.awk ${SYSDIR}/kern/vnode_if.src -p vnode_if_typedef.h: ${AWK} -f ${SYSDIR}/tools/vnode_if.awk ${SYSDIR}/kern/vnode_if.src -q .endif # Build _if.[ch] from _if.m, and clean them when we're done. # __MPATH defined in config.mk _MFILES=${__MPATH:T:O} _MPATH=${__MPATH:H:O:u} .PATH.m: ${_MPATH} .for _i in ${SRCS:M*_if.[ch]} _MATCH=M${_i:R:S/$/.m/} _MATCHES=${_MFILES:${_MATCH}} .if !empty(_MATCHES) CLEANFILES+= ${_i} .endif .endfor # _i .m.c: ${SYSDIR}/tools/makeobjops.awk ${AWK} -f ${SYSDIR}/tools/makeobjops.awk ${.IMPSRC} -c .m.h: ${SYSDIR}/tools/makeobjops.awk ${AWK} -f ${SYSDIR}/tools/makeobjops.awk ${.IMPSRC} -h .for _i in mii pccard .if !empty(SRCS:M${_i}devs.h) CLEANFILES+= ${_i}devs.h ${_i}devs.h: ${SYSDIR}/tools/${_i}devs2h.awk ${SYSDIR}/dev/${_i}/${_i}devs ${AWK} -f ${SYSDIR}/tools/${_i}devs2h.awk ${SYSDIR}/dev/${_i}/${_i}devs .endif .endfor # _i .if !empty(SRCS:Mbhnd_nvram_map.h) CLEANFILES+= bhnd_nvram_map.h bhnd_nvram_map.h: ${SYSDIR}/dev/bhnd/tools/nvram_map_gen.awk \ ${SYSDIR}/dev/bhnd/tools/nvram_map_gen.sh \ ${SYSDIR}/dev/bhnd/nvram/nvram_map bhnd_nvram_map.h: sh ${SYSDIR}/dev/bhnd/tools/nvram_map_gen.sh \ ${SYSDIR}/dev/bhnd/nvram/nvram_map -h .endif .if !empty(SRCS:Mbhnd_nvram_map_data.h) CLEANFILES+= bhnd_nvram_map_data.h bhnd_nvram_map_data.h: ${SYSDIR}/dev/bhnd/tools/nvram_map_gen.awk \ ${SYSDIR}/dev/bhnd/tools/nvram_map_gen.sh \ ${SYSDIR}/dev/bhnd/nvram/nvram_map bhnd_nvram_map_data.h: sh ${SYSDIR}/dev/bhnd/tools/nvram_map_gen.sh \ ${SYSDIR}/dev/bhnd/nvram/nvram_map -d .endif .if !empty(SRCS:Musbdevs.h) CLEANFILES+= usbdevs.h usbdevs.h: ${SYSDIR}/tools/usbdevs2h.awk ${SYSDIR}/dev/usb/usbdevs ${AWK} -f ${SYSDIR}/tools/usbdevs2h.awk ${SYSDIR}/dev/usb/usbdevs -h .endif .if !empty(SRCS:Musbdevs_data.h) CLEANFILES+= usbdevs_data.h usbdevs_data.h: ${SYSDIR}/tools/usbdevs2h.awk ${SYSDIR}/dev/usb/usbdevs ${AWK} -f ${SYSDIR}/tools/usbdevs2h.awk ${SYSDIR}/dev/usb/usbdevs -d .endif .if !empty(SRCS:Macpi_quirks.h) CLEANFILES+= acpi_quirks.h acpi_quirks.h: ${SYSDIR}/tools/acpi_quirks2h.awk ${SYSDIR}/dev/acpica/acpi_quirks ${AWK} -f ${SYSDIR}/tools/acpi_quirks2h.awk ${SYSDIR}/dev/acpica/acpi_quirks .endif -.if !empty(SRCS:Massym.s) || !empty(DPSRCS:Massym.s) -CLEANFILES+= assym.s genassym.o +.if !empty(SRCS:Massym.inc) || !empty(DPSRCS:Massym.inc) +CLEANFILES+= assym.inc genassym.o DEPENDOBJS+= genassym.o -assym.s: genassym.o +assym.inc: genassym.o .if defined(KERNBUILDDIR) genassym.o: opt_global.h .endif -assym.s: ${SYSDIR}/kern/genassym.sh +assym.inc: ${SYSDIR}/kern/genassym.sh sh ${SYSDIR}/kern/genassym.sh genassym.o > ${.TARGET} genassym.o: ${SYSDIR}/${MACHINE}/${MACHINE}/genassym.c genassym.o: ${SRCS:Mopt_*.h} ${CC} -c ${CFLAGS:N-flto:N-fno-common} \ ${SYSDIR}/${MACHINE}/${MACHINE}/genassym.c .endif .if defined(KERNBUILDDIR) ${OBJS}: opt_global.h .endif CLEANDEPENDFILES+= ${_ILINKS} # .depend needs include links so we remove them only together. cleanilinks: rm -f ${_ILINKS} OBJS_DEPEND_GUESS+= ${SRCS:M*.h} .include .include .include .include "kern.mk" Index: head/sys/dev/hyperv/vmbus/amd64/vmbus_vector.S =================================================================== --- head/sys/dev/hyperv/vmbus/amd64/vmbus_vector.S (revision 331253) +++ head/sys/dev/hyperv/vmbus/amd64/vmbus_vector.S (revision 331254) @@ -1,45 +1,45 @@ /*- * Copyright (c) 2016 Microsoft Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ -#include "assym.s" +#include "assym.inc" #include #include /* * This is the Hyper-V vmbus channel direct callback interrupt. * Only used when it is running on Hyper-V. */ .text SUPERALIGN_TEXT INTR_HANDLER vmbus_isr FAKE_MCOUNT(TF_RIP(%rsp)) movq %rsp, %rdi call vmbus_handle_intr MEXITCOUNT jmp doreti Index: head/sys/dev/hyperv/vmbus/i386/vmbus_vector.S =================================================================== --- head/sys/dev/hyperv/vmbus/i386/vmbus_vector.S (revision 331253) +++ head/sys/dev/hyperv/vmbus/i386/vmbus_vector.S (revision 331254) @@ -1,50 +1,50 @@ /*- * Copyright (c) 2016 Microsoft Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #include #include -#include "assym.s" +#include "assym.inc" /* * This is the Hyper-V vmbus channel direct callback interrupt. * Only used when it is running on Hyper-V. */ .text SUPERALIGN_TEXT IDTVEC(vmbus_isr_pti) IDTVEC(vmbus_isr) PUSH_FRAME SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) pushl %esp call vmbus_handle_intr add $4, %esp MEXITCOUNT jmp doreti Index: head/sys/i386/acpica/acpi_wakecode.S =================================================================== --- head/sys/i386/acpica/acpi_wakecode.S (revision 331253) +++ head/sys/i386/acpica/acpi_wakecode.S (revision 331254) @@ -1,201 +1,201 @@ /*- * Copyright (c) 2001 Takanori Watanabe * Copyright (c) 2001-2012 Mitsuru IWASAKI * Copyright (c) 2003 Peter Wemm * Copyright (c) 2008-2012 Jung-uk Kim * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include -#include "assym.s" +#include "assym.inc" /* * Resume entry point. The BIOS enters here in real mode after POST with * CS set to the page where we stored this code. It should configure the * segment registers with a flat 4 GB address space and EFLAGS.IF = 0. * Depending on the previous sleep state, we may need to initialize more * of the system (i.e., S3 suspend-to-RAM vs. S4 suspend-to-disk). */ .data /* So we can modify it */ ALIGN_TEXT .code16 wakeup_start: /* * Set up segment registers for real mode and a small stack for * any calls we make. Set up full 32-bit bootstrap kernel flags * since resumectx() doesn't restore flags. PSL_KERNEL gives * bootstrap kernel flags (with interrupts disabled), not normal * kernel flags. */ cli /* make sure no interrupts */ mov %cs, %ax /* copy %cs to %ds. Remember these */ mov %ax, %ds /* are offsets rather than selectors */ mov %ax, %ss movw $PAGE_SIZE, %sp pushl $PSL_KERNEL popfl /* To debug resume hangs, beep the speaker if the user requested. */ testb $~0, resume_beep - wakeup_start jz 1f movb $0, resume_beep - wakeup_start /* Set PIC timer2 to beep. */ movb $(TIMER_SEL2 | TIMER_SQWAVE | TIMER_16BIT), %al outb %al, $TIMER_MODE /* Turn on speaker. */ inb $IO_PPI, %al orb $PIT_SPKR, %al outb %al, $IO_PPI /* Set frequency. */ movw $0x4c0, %ax outb %al, $TIMER_CNTR2 shrw $8, %ax outb %al, $TIMER_CNTR2 1: /* Re-initialize video BIOS if the reset_video tunable is set. */ testb $~0, reset_video - wakeup_start jz 1f movb $0, reset_video - wakeup_start lcall $0xc000, $3 /* When we reach here, int 0x10 should be ready. Hide cursor. */ movb $0x01, %ah movb $0x20, %ch int $0x10 /* Re-start in case the previous BIOS call clobbers them. */ jmp wakeup_start 1: /* * Find relocation base and patch the gdt descript and ljmp targets */ xorl %ebx, %ebx mov %cs, %bx sall $4, %ebx /* %ebx is now our relocation base */ /* * Load the descriptor table pointer. We'll need it when running * in 16-bit protected mode. */ lgdtl bootgdtdesc - wakeup_start /* Enable protected mode */ movl $CR0_PE, %eax mov %eax, %cr0 /* * Now execute a far jump to turn on protected mode. This * causes the segment registers to turn into selectors and causes * %cs to be loaded from the gdt. * * The following instruction is: * ljmpl $bootcode32 - bootgdt, $wakeup_32 - wakeup_start * but gas cannot assemble that. And besides, we patch the targets * in early startup and its a little clearer what we are patching. */ wakeup_sw32: .byte 0x66 /* size override to 32 bits */ .byte 0xea /* opcode for far jump */ .long wakeup_32 - wakeup_start /* offset in segment */ .word bootcode32 - bootgdt /* index in gdt for 32 bit code */ /* * At this point, we are running in 32 bit legacy protected mode. */ ALIGN_TEXT .code32 wakeup_32: mov $bootdata32 - bootgdt, %eax mov %ax, %ds /* Get PCB and return address. */ movl wakeup_pcb - wakeup_start(%ebx), %ecx movl wakeup_ret - wakeup_start(%ebx), %edx /* Restore CR4 and CR3. */ movl wakeup_cr4 - wakeup_start(%ebx), %eax mov %eax, %cr4 movl wakeup_cr3 - wakeup_start(%ebx), %eax mov %eax, %cr3 /* Enable paging. */ mov %cr0, %eax orl $CR0_PG, %eax mov %eax, %cr0 /* Jump to return address. */ jmp *%edx .data resume_beep: .byte 0 reset_video: .byte 0 ALIGN_DATA bootgdt: .long 0x00000000 .long 0x00000000 bootcode32: .long 0x0000ffff .long 0x00cf9b00 bootdata32: .long 0x0000ffff .long 0x00cf9300 bootgdtend: bootgdtdesc: .word bootgdtend - bootgdt /* Length */ .long bootgdt - wakeup_start /* Offset plus %ds << 4 */ ALIGN_DATA wakeup_cr4: .long 0 wakeup_cr3: .long 0 wakeup_pcb: .long 0 wakeup_ret: .long 0 wakeup_gdt: /* not used */ .word 0 .long 0 dummy: Index: head/sys/i386/i386/apic_vector.s =================================================================== --- head/sys/i386/i386/apic_vector.s (revision 331253) +++ head/sys/i386/i386/apic_vector.s (revision 331254) @@ -1,318 +1,318 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD$ */ /* * Interrupt entry points for external interrupts triggered by I/O APICs * as well as IPI handlers. */ #include "opt_smp.h" #include #include #include -#include "assym.s" +#include "assym.inc" .text SUPERALIGN_TEXT /* End Of Interrupt to APIC */ as_lapic_eoi: cmpl $0,x2apic_mode jne 1f movl lapic_map,%eax movl $0,LA_EOI(%eax) ret 1: movl $MSR_APIC_EOI,%ecx xorl %eax,%eax xorl %edx,%edx wrmsr ret /* * I/O Interrupt Entry Point. Rather than having one entry point for * each interrupt source, we use one entry point for each 32-bit word * in the ISR. The handler determines the highest bit set in the ISR, * translates that into a vector, and passes the vector to the * lapic_handle_intr() function. */ #define ISR_VEC(index, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name ## _pti) ; \ IDTVEC(vec_name) ; \ PUSH_FRAME ; \ SET_KERNEL_SREGS ; \ cld ; \ FAKE_MCOUNT(TF_EIP(%esp)) ; \ cmpl $0,x2apic_mode ; \ je 1f ; \ movl $(MSR_APIC_ISR0 + index),%ecx ; \ rdmsr ; \ jmp 2f ; \ 1: ; \ movl lapic_map, %edx ;/* pointer to local APIC */ \ movl LA_ISR + 16 * (index)(%edx), %eax ; /* load ISR */ \ 2: ; \ bsrl %eax, %eax ; /* index of highest set bit in ISR */ \ jz 3f ; \ addl $(32 * index),%eax ; \ pushl %esp ; \ pushl %eax ; /* pass the IRQ */ \ call lapic_handle_intr ; \ addl $8, %esp ; /* discard parameter */ \ 3: ; \ MEXITCOUNT ; \ jmp doreti /* * Handle "spurious INTerrupts". * Notes: * This is different than the "spurious INTerrupt" generated by an * 8259 PIC for missing INTs. See the APIC documentation for details. * This routine should NOT do an 'EOI' cycle. */ .text SUPERALIGN_TEXT IDTVEC(spuriousint) /* No EOI cycle used here */ iret ISR_VEC(1, apic_isr1) ISR_VEC(2, apic_isr2) ISR_VEC(3, apic_isr3) ISR_VEC(4, apic_isr4) ISR_VEC(5, apic_isr5) ISR_VEC(6, apic_isr6) ISR_VEC(7, apic_isr7) /* * Local APIC periodic timer handler. */ .text SUPERALIGN_TEXT IDTVEC(timerint_pti) IDTVEC(timerint) PUSH_FRAME SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) pushl %esp call lapic_handle_timer add $4, %esp MEXITCOUNT jmp doreti /* * Local APIC CMCI handler. */ .text SUPERALIGN_TEXT IDTVEC(cmcint_pti) IDTVEC(cmcint) PUSH_FRAME SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) call lapic_handle_cmc MEXITCOUNT jmp doreti /* * Local APIC error interrupt handler. */ .text SUPERALIGN_TEXT IDTVEC(errorint_pti) IDTVEC(errorint) PUSH_FRAME SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) call lapic_handle_error MEXITCOUNT jmp doreti #ifdef XENHVM /* * Xen event channel upcall interrupt handler. * Only used when the hypervisor supports direct vector callbacks. */ .text SUPERALIGN_TEXT IDTVEC(xen_intr_upcall) PUSH_FRAME SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) pushl %esp call xen_intr_handle_upcall add $4, %esp MEXITCOUNT jmp doreti #endif #ifdef SMP /* * Global address space TLB shootdown. */ .text SUPERALIGN_TEXT invltlb_ret: call as_lapic_eoi jmp doreti SUPERALIGN_TEXT IDTVEC(invltlb) PUSH_FRAME SET_KERNEL_SREGS cld call invltlb_handler jmp invltlb_ret /* * Single page TLB shootdown */ .text SUPERALIGN_TEXT IDTVEC(invlpg) PUSH_FRAME SET_KERNEL_SREGS cld call invlpg_handler jmp invltlb_ret /* * Page range TLB shootdown. */ .text SUPERALIGN_TEXT IDTVEC(invlrng) PUSH_FRAME SET_KERNEL_SREGS cld call invlrng_handler jmp invltlb_ret /* * Invalidate cache. */ .text SUPERALIGN_TEXT IDTVEC(invlcache) PUSH_FRAME SET_KERNEL_SREGS cld call invlcache_handler jmp invltlb_ret /* * Handler for IPIs sent via the per-cpu IPI bitmap. */ .text SUPERALIGN_TEXT IDTVEC(ipi_intr_bitmap_handler) PUSH_FRAME SET_KERNEL_SREGS cld call as_lapic_eoi FAKE_MCOUNT(TF_EIP(%esp)) call ipi_bitmap_handler MEXITCOUNT jmp doreti /* * Executed by a CPU when it receives an IPI_STOP from another CPU. */ .text SUPERALIGN_TEXT IDTVEC(cpustop) PUSH_FRAME SET_KERNEL_SREGS cld call as_lapic_eoi call cpustop_handler jmp doreti /* * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. */ .text SUPERALIGN_TEXT IDTVEC(cpususpend) PUSH_FRAME SET_KERNEL_SREGS cld call as_lapic_eoi call cpususpend_handler jmp doreti /* * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. * * - Calls the generic rendezvous action function. */ .text SUPERALIGN_TEXT IDTVEC(rendezvous) PUSH_FRAME SET_KERNEL_SREGS cld #ifdef COUNT_IPIS movl PCPU(CPUID), %eax movl ipi_rendezvous_counts(,%eax,4), %eax incl (%eax) #endif call smp_rendezvous_action call as_lapic_eoi jmp doreti #endif /* SMP */ Index: head/sys/i386/i386/atpic_vector.s =================================================================== --- head/sys/i386/i386/atpic_vector.s (revision 331253) +++ head/sys/i386/i386/atpic_vector.s (revision 331254) @@ -1,79 +1,79 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD$ */ /* * Interrupt entry points for external interrupts triggered by the 8259A * master and slave interrupt controllers. */ #include -#include "assym.s" +#include "assym.inc" /* * Macros for interrupt entry, call to handler, and exit. */ #define INTR(irq_num, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name ##_pti) ; \ IDTVEC(vec_name) ; \ PUSH_FRAME ; \ SET_KERNEL_SREGS ; \ cld ; \ ; \ FAKE_MCOUNT(TF_EIP(%esp)) ; \ pushl %esp ; \ pushl $irq_num; /* pass the IRQ */ \ call atpic_handle_intr ; \ addl $8, %esp ; /* discard the parameters */ \ ; \ MEXITCOUNT ; \ jmp doreti INTR(0, atpic_intr0) INTR(1, atpic_intr1) INTR(2, atpic_intr2) INTR(3, atpic_intr3) INTR(4, atpic_intr4) INTR(5, atpic_intr5) INTR(6, atpic_intr6) INTR(7, atpic_intr7) INTR(8, atpic_intr8) INTR(9, atpic_intr9) INTR(10, atpic_intr10) INTR(11, atpic_intr11) INTR(12, atpic_intr12) INTR(13, atpic_intr13) INTR(14, atpic_intr14) INTR(15, atpic_intr15) Index: head/sys/i386/i386/bioscall.s =================================================================== --- head/sys/i386/i386/bioscall.s (revision 331253) +++ head/sys/i386/i386/bioscall.s (revision 331254) @@ -1,160 +1,160 @@ /*- * Copyright (c) 1997 Jonathan Lemon * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * Functions for calling x86 BIOS functions from the BSD kernel */ #include -#include "assym.s" +#include "assym.inc" .data ALIGN_DATA bioscall_frame: .long 0 bioscall_stack: .long 0 .text /* * bios32(regs, offset, segment) * struct bios_regs *regs; * u_int offset; * u_short segment; */ ENTRY(bios32) pushl %ebp movl 16(%esp),%ebp mov %bp,bioscall_vector+4 movl 12(%esp),%ebp movl %ebp,bioscall_vector movl 8(%esp),%ebp pushl %ebx pushl %esi pushl %edi movl 0(%ebp),%eax movl 4(%ebp),%ebx movl 8(%ebp),%ecx movl 12(%ebp),%edx movl 16(%ebp),%esi movl 20(%ebp),%edi pushl %ebp lcall *bioscall_vector popl %ebp movl %eax,0(%ebp) movl %ebx,4(%ebp) movl %ecx,8(%ebp) movl %edx,12(%ebp) movl %esi,16(%ebp) movl %edi,20(%ebp) movl $0,%eax /* presume success */ jnc 1f movl $1,%eax /* nope */ 1: popl %edi popl %esi popl %ebx popl %ebp ret /* * bios16_call(regs, stack) * struct bios_regs *regs; * char *stack; */ ENTRY(bios16_call) pushl %ebp movl %esp,%ebp addl $4,%ebp /* frame pointer */ movl %ebp,bioscall_frame /* ... save it */ pushl %ebx pushl %esi pushl %edi /* * the problem with a full 32-bit stack segment is that 16-bit code * tends to do a pushf, which only pushes %sp, not %esp. This value * is then popped off (into %esp) which causes a page fault because * it is the wrong address. * * the reverse problem happens for 16-bit stack addresses; the kernel * code attempts to get the address of something on the stack, and the * value returned is the address relative to %ss, not %ds. * * we fix this by installing a temporary stack at page 0, so the * addresses are always valid in both 32 bit and 16 bit modes. */ movl %esp,bioscall_stack /* save current stack location */ movl 8(%ebp),%esp /* switch to page 0 stack */ movl 4(%ebp),%ebp /* regs */ movl 0(%ebp),%eax movl 4(%ebp),%ebx movl 8(%ebp),%ecx movl 12(%ebp),%edx movl 16(%ebp),%esi movl 20(%ebp),%edi pushl $BC32SEL leal CNAME(bios16_jmp),%ebp andl $PAGE_MASK,%ebp pushl %ebp /* reload %cs and */ lret /* ...continue below */ .globl CNAME(bios16_jmp) CNAME(bios16_jmp): lcallw *bioscall_vector /* 16-bit call */ jc 1f pushl $0 /* success */ jmp 2f 1: pushl $1 /* failure */ 2: movl bioscall_frame,%ebp movl 4(%ebp),%ebp /* regs */ movl %eax,0(%ebp) movl %ebx,4(%ebp) movl %ecx,8(%ebp) movl %edx,12(%ebp) movl %esi,16(%ebp) movl %edi,20(%ebp) popl %eax /* recover return value */ movl bioscall_stack,%esp /* return to normal stack */ popl %edi popl %esi popl %ebx popl %ebp movl (%esp),%ecx pushl %ecx /* return address */ movl $KCSEL,4(%esp) lret /* reload %cs on the way out */ Index: head/sys/i386/i386/exception.s =================================================================== --- head/sys/i386/i386/exception.s (revision 331253) +++ head/sys/i386/i386/exception.s (revision 331254) @@ -1,515 +1,515 @@ /*- * Copyright (c) 1989, 1990 William F. Jolitz. * Copyright (c) 1990 The Regents of the University of California. * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_apic.h" #include "opt_atpic.h" #include "opt_hwpmc_hooks.h" #include #include #include -#include "assym.s" +#include "assym.inc #define SEL_RPL_MASK 0x0003 #define GSEL_KPL 0x0020 /* GSEL(GCODE_SEL, SEL_KPL) */ #ifdef KDTRACE_HOOKS .bss .globl dtrace_invop_jump_addr .align 4 .type dtrace_invop_jump_addr, @object .size dtrace_invop_jump_addr, 4 dtrace_invop_jump_addr: .zero 4 .globl dtrace_invop_calltrap_addr .align 4 .type dtrace_invop_calltrap_addr, @object .size dtrace_invop_calltrap_addr, 4 dtrace_invop_calltrap_addr: .zero 8 #endif .text #ifdef HWPMC_HOOKS ENTRY(start_exceptions) #endif /*****************************************************************************/ /* Trap handling */ /*****************************************************************************/ /* * Trap and fault vector routines. * * Most traps are 'trap gates', SDT_SYS386TGT. A trap gate pushes state on * the stack that mostly looks like an interrupt, but does not disable * interrupts. A few of the traps we are use are interrupt gates, * SDT_SYS386IGT, which are nearly the same thing except interrupts are * disabled on entry. * * The cpu will push a certain amount of state onto the kernel stack for * the current process. The amount of state depends on the type of trap * and whether the trap crossed rings or not. See i386/include/frame.h. * At the very least the current EFLAGS (status register, which includes * the interrupt disable state prior to the trap), the code segment register, * and the return instruction pointer are pushed by the cpu. The cpu * will also push an 'error' code for certain traps. We push a dummy * error code for those traps where the cpu doesn't in order to maintain * a consistent frame. We also push a contrived 'trap number'. * * The cpu does not push the general registers, we must do that, and we * must restore them prior to calling 'iret'. The cpu adjusts the %cs and * %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we * must load them with appropriate values for supervisor mode operation. */ MCOUNT_LABEL(user) MCOUNT_LABEL(btrap) #define TRAP(a) pushl $(a) ; jmp alltraps IDTVEC(div) pushl $0; TRAP(T_DIVIDE) IDTVEC(dbg) pushl $0; TRAP(T_TRCTRAP) IDTVEC(nmi) pushl $0; TRAP(T_NMI) IDTVEC(bpt) pushl $0; TRAP(T_BPTFLT) IDTVEC(dtrace_ret) pushl $0; TRAP(T_DTRACE_RET) IDTVEC(ofl) pushl $0; TRAP(T_OFLOW) IDTVEC(bnd) pushl $0; TRAP(T_BOUND) #ifndef KDTRACE_HOOKS IDTVEC(ill) pushl $0; TRAP(T_PRIVINFLT) #endif IDTVEC(dna) pushl $0; TRAP(T_DNA) IDTVEC(fpusegm) pushl $0; TRAP(T_FPOPFLT) IDTVEC(tss) TRAP(T_TSSFLT) IDTVEC(missing) TRAP(T_SEGNPFLT) IDTVEC(stk) TRAP(T_STKFLT) IDTVEC(prot) TRAP(T_PROTFLT) IDTVEC(page) TRAP(T_PAGEFLT) IDTVEC(mchk) pushl $0; TRAP(T_MCHK) IDTVEC(rsvd_pti) IDTVEC(rsvd) pushl $0; TRAP(T_RESERVED) IDTVEC(fpu) pushl $0; TRAP(T_ARITHTRAP) IDTVEC(align) TRAP(T_ALIGNFLT) IDTVEC(xmm) pushl $0; TRAP(T_XMMFLT) /* * All traps except ones for syscalls jump to alltraps. If * interrupts were enabled when the trap occurred, then interrupts * are enabled now if the trap was through a trap gate, else * disabled if the trap was through an interrupt gate. Note that * int0x80_syscall is a trap gate. Interrupt gates are used by * page faults, non-maskable interrupts, debug and breakpoint * exceptions. */ SUPERALIGN_TEXT .globl alltraps .type alltraps,@function alltraps: pushal pushl $0 movw %ds,(%esp) pushl $0 movw %es,(%esp) pushl $0 movw %fs,(%esp) alltraps_with_regs_pushed: SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) calltrap: pushl %esp call trap add $4, %esp /* * Return via doreti to handle ASTs. */ MEXITCOUNT jmp doreti /* * Privileged instruction fault. */ #ifdef KDTRACE_HOOKS SUPERALIGN_TEXT IDTVEC(ill) /* * Check if a DTrace hook is registered. The default (data) segment * cannot be used for this since %ds is not known good until we * verify that the entry was from kernel mode. */ cmpl $0,%ss:dtrace_invop_jump_addr je norm_ill /* * Check if this is a user fault. If so, just handle it as a normal * trap. */ cmpl $GSEL_KPL, 4(%esp) /* Check the code segment */ jne norm_ill testl $PSL_VM, 8(%esp) /* and vm86 mode. */ jnz norm_ill /* * This is a kernel instruction fault that might have been caused * by a DTrace provider. */ pushal cld /* * Set our jump address for the jump back in the event that * the exception wasn't caused by DTrace at all. */ movl $norm_ill, dtrace_invop_calltrap_addr /* Jump to the code hooked in by DTrace. */ jmpl *dtrace_invop_jump_addr /* * Process the instruction fault in the normal way. */ norm_ill: pushl $0 TRAP(T_PRIVINFLT) #endif /* * Call gate entry for syscalls (lcall 7,0). * This is used by FreeBSD 1.x a.out executables and "old" NetBSD executables. * * The intersegment call has been set up to specify one dummy parameter. * This leaves a place to put eflags so that the call frame can be * converted to a trap frame. Note that the eflags is (semi-)bogusly * pushed into (what will be) tf_err and then copied later into the * final spot. It has to be done this way because esp can't be just * temporarily altered for the pushfl - an interrupt might come in * and clobber the saved cs/eip. */ SUPERALIGN_TEXT IDTVEC(lcall_syscall) pushfl /* save eflags */ popl 8(%esp) /* shuffle into tf_eflags */ pushl $7 /* sizeof "lcall 7,0" */ pushl $0 /* tf_trapno */ pushal pushl $0 movw %ds,(%esp) pushl $0 movw %es,(%esp) pushl $0 movw %fs,(%esp) SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) pushl %esp call syscall add $4, %esp MEXITCOUNT jmp doreti /* * Trap gate entry for syscalls (int 0x80). * This is used by FreeBSD ELF executables, "new" NetBSD executables, and all * Linux executables. * * Even though the name says 'int0x80', this is actually a trap gate, not an * interrupt gate. Thus interrupts are enabled on entry just as they are for * a normal syscall. */ SUPERALIGN_TEXT IDTVEC(int0x80_syscall) pushl $2 /* sizeof "int 0x80" */ pushl $0 /* tf_trapno */ pushal pushl $0 movw %ds,(%esp) pushl $0 movw %es,(%esp) pushl $0 movw %fs,(%esp) SET_KERNEL_SREGS cld FAKE_MCOUNT(TF_EIP(%esp)) pushl %esp call syscall add $4, %esp MEXITCOUNT jmp doreti ENTRY(fork_trampoline) pushl %esp /* trapframe pointer */ pushl %ebx /* arg1 */ pushl %esi /* function */ call fork_exit addl $12,%esp /* cut from syscall */ /* * Return via doreti to handle ASTs. */ MEXITCOUNT jmp doreti /* * To efficiently implement classification of trap and interrupt handlers * for profiling, there must be only trap handlers between the labels btrap * and bintr, and only interrupt handlers between the labels bintr and * eintr. This is implemented (partly) by including files that contain * some of the handlers. Before including the files, set up a normal asm * environment so that the included files doen't need to know that they are * included. */ .data .p2align 4 .text SUPERALIGN_TEXT MCOUNT_LABEL(bintr) #ifdef DEV_ATPIC #include #endif #if defined(DEV_APIC) && defined(DEV_ATPIC) .data .p2align 4 .text SUPERALIGN_TEXT #endif #ifdef DEV_APIC #include #endif .data .p2align 4 .text SUPERALIGN_TEXT #include .text MCOUNT_LABEL(eintr) /* * void doreti(struct trapframe) * * Handle return from interrupts, traps and syscalls. */ .text SUPERALIGN_TEXT .type doreti,@function .globl doreti doreti: FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ doreti_next: /* * Check if ASTs can be handled now. ASTs cannot be safely * processed when returning from an NMI. */ cmpb $T_NMI,TF_TRAPNO(%esp) #ifdef HWPMC_HOOKS je doreti_nmi #else je doreti_exit #endif /* * PSL_VM must be checked first since segment registers only * have an RPL in non-VM86 mode. * ASTs can not be handled now if we are in a vm86 call. */ testl $PSL_VM,TF_EFLAGS(%esp) jz doreti_notvm86 movl PCPU(CURPCB),%ecx testl $PCB_VM86CALL,PCB_FLAGS(%ecx) jz doreti_ast jmp doreti_exit doreti_notvm86: testb $SEL_RPL_MASK,TF_CS(%esp) /* are we returning to user mode? */ jz doreti_exit /* can't handle ASTs now if not */ doreti_ast: /* * Check for ASTs atomically with returning. Disabling CPU * interrupts provides sufficient locking even in the SMP case, * since we will be informed of any new ASTs by an IPI. */ cli movl PCPU(CURTHREAD),%eax testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%eax) je doreti_exit sti pushl %esp /* pass a pointer to the trapframe */ call ast add $4,%esp jmp doreti_ast /* * doreti_exit: pop registers, iret. * * The segment register pop is a special case, since it may * fault if (for example) a sigreturn specifies bad segment * registers. The fault is handled in trap.c. */ doreti_exit: MEXITCOUNT .globl doreti_popl_fs doreti_popl_fs: popl %fs .globl doreti_popl_es doreti_popl_es: popl %es .globl doreti_popl_ds doreti_popl_ds: popl %ds popal addl $8,%esp .globl doreti_iret doreti_iret: iret /* * doreti_iret_fault and friends. Alternative return code for * the case where we get a fault in the doreti_exit code * above. trap() (i386/i386/trap.c) catches this specific * case, and continues in the corresponding place in the code * below. * * If the fault occured during return to usermode, we recreate * the trap frame and call trap() to send a signal. Otherwise * the kernel was tricked into fault by attempt to restore invalid * usermode segment selectors on return from nested fault or * interrupt, where interrupted kernel entry code not yet loaded * kernel selectors. In the latter case, emulate iret and zero * the invalid selector. */ ALIGN_TEXT .globl doreti_iret_fault doreti_iret_fault: subl $8,%esp pushal pushl $0 movw %ds,(%esp) .globl doreti_popl_ds_fault doreti_popl_ds_fault: testb $SEL_RPL_MASK,TF_CS-TF_DS(%esp) jz doreti_popl_ds_kfault pushl $0 movw %es,(%esp) .globl doreti_popl_es_fault doreti_popl_es_fault: testb $SEL_RPL_MASK,TF_CS-TF_ES(%esp) jz doreti_popl_es_kfault pushl $0 movw %fs,(%esp) .globl doreti_popl_fs_fault doreti_popl_fs_fault: testb $SEL_RPL_MASK,TF_CS-TF_FS(%esp) jz doreti_popl_fs_kfault sti movl $0,TF_ERR(%esp) /* XXX should be the error code */ movl $T_PROTFLT,TF_TRAPNO(%esp) jmp alltraps_with_regs_pushed doreti_popl_ds_kfault: movl $0,(%esp) jmp doreti_popl_ds doreti_popl_es_kfault: movl $0,(%esp) jmp doreti_popl_es doreti_popl_fs_kfault: movl $0,(%esp) jmp doreti_popl_fs #ifdef HWPMC_HOOKS doreti_nmi: /* * Since we are returning from an NMI, check if the current trap * was from user mode and if so whether the current thread * needs a user call chain capture. */ testb $SEL_RPL_MASK,TF_CS(%esp) jz doreti_exit movl PCPU(CURTHREAD),%eax /* curthread present? */ orl %eax,%eax jz doreti_exit testl $TDP_CALLCHAIN,TD_PFLAGS(%eax) /* flagged for capture? */ jz doreti_exit /* * Take the processor out of NMI mode by executing a fake "iret". */ pushfl pushl %cs pushl $outofnmi iret outofnmi: /* * Call the callchain capture hook after turning interrupts back on. */ movl pmc_hook,%ecx orl %ecx,%ecx jz doreti_exit pushl %esp /* frame pointer */ pushl $PMC_FN_USER_CALLCHAIN /* command */ movl PCPU(CURTHREAD),%eax pushl %eax /* curthread */ sti call *%ecx addl $12,%esp jmp doreti_ast ENTRY(end_exceptions) #endif Index: head/sys/i386/i386/locore.s =================================================================== --- head/sys/i386/i386/locore.s (revision 331253) +++ head/sys/i386/i386/locore.s (revision 331254) @@ -1,788 +1,788 @@ /*- * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 * $FreeBSD$ * * originally from: locore.s, by William F. Jolitz * * Substantially rewritten by David Greenman, Rod Grimes, * Bruce Evans, Wolfgang Solfrank, Poul-Henning Kamp * and many others. */ #include "opt_bootp.h" #include "opt_compat.h" #include "opt_nfsroot.h" #include "opt_pmap.h" #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" /* * XXX * * Note: This version greatly munged to avoid various assembler errors * that may be fixed in newer versions of gas. Perhaps newer versions * will have more pleasant appearance. */ /* * PTmap is recursive pagemap at top of virtual address space. * Within PTmap, the page directory can be found (third indirection). */ .globl PTmap,PTD,PTDpde .set PTmap,(PTDPTDI << PDRSHIFT) .set PTD,PTmap + (PTDPTDI * PAGE_SIZE) .set PTDpde,PTD + (PTDPTDI * PDESIZE) /* * Compiled KERNBASE location and the kernel load address */ .globl kernbase .set kernbase,KERNBASE .globl kernload .set kernload,KERNLOAD /* * Globals */ .data ALIGN_DATA /* just to be sure */ .space 0x2000 /* space for tmpstk - temporary stack */ tmpstk: .globl bootinfo bootinfo: .space BOOTINFO_SIZE /* bootinfo that we can handle */ .globl KERNend KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ .globl IdlePTD IdlePTD: .long 0 /* phys addr of kernel PTD */ #if defined(PAE) || defined(PAE_TABLES) .globl IdlePDPT IdlePDPT: .long 0 /* phys addr of kernel PDPT */ #endif .globl KPTmap KPTmap: .long 0 /* address of kernel page tables */ .globl KPTphys KPTphys: .long 0 /* phys addr of kernel page tables */ .globl proc0kstack proc0kstack: .long 0 /* address of proc 0 kstack space */ p0kpa: .long 0 /* phys addr of proc0's STACK */ vm86phystk: .long 0 /* PA of vm86/bios stack */ .globl vm86paddr, vm86pa vm86paddr: .long 0 /* address of vm86 region */ vm86pa: .long 0 /* phys addr of vm86 region */ /********************************************************************** * * Some handy macros * */ #define R(foo) ((foo)-KERNBASE) #define ALLOCPAGES(foo) \ movl R(physfree), %esi ; \ movl $((foo)*PAGE_SIZE), %eax ; \ addl %esi, %eax ; \ movl %eax, R(physfree) ; \ movl %esi, %edi ; \ movl $((foo)*PAGE_SIZE),%ecx ; \ xorl %eax,%eax ; \ cld ; \ rep ; \ stosb /* * fillkpt * eax = page frame address * ebx = index into page table * ecx = how many pages to map * base = base address of page dir/table * prot = protection bits */ #define fillkpt(base, prot) \ shll $PTESHIFT,%ebx ; \ addl base,%ebx ; \ orl $PG_V,%eax ; \ orl prot,%eax ; \ 1: movl %eax,(%ebx) ; \ addl $PAGE_SIZE,%eax ; /* increment physical address */ \ addl $PTESIZE,%ebx ; /* next pte */ \ loop 1b /* * fillkptphys(prot) * eax = physical address * ecx = how many pages to map * prot = protection bits */ #define fillkptphys(prot) \ movl %eax, %ebx ; \ shrl $PAGE_SHIFT, %ebx ; \ fillkpt(R(KPTphys), prot) .text /********************************************************************** * * This is where the bootblocks start us, set the ball rolling... * */ NON_GPROF_ENTRY(btext) /* Tell the bios to warmboot next time */ movw $0x1234,0x472 /* Set up a real frame in case the double return in newboot is executed. */ pushl %ebp movl %esp, %ebp /* Don't trust what the BIOS gives for eflags. */ pushl $PSL_KERNEL popfl /* * Don't trust what the BIOS gives for %fs and %gs. Trust the bootstrap * to set %cs, %ds, %es and %ss. */ mov %ds, %ax mov %ax, %fs mov %ax, %gs /* * Clear the bss. Not all boot programs do it, and it is our job anyway. * * XXX we don't check that there is memory for our bss and page tables * before using it. * * Note: we must be careful to not overwrite an active gdt or idt. They * inactive from now until we switch to new ones, since we don't load any * more segment registers or permit interrupts until after the switch. */ movl $R(end),%ecx movl $R(edata),%edi subl %edi,%ecx xorl %eax,%eax cld rep stosb call recover_bootinfo /* Get onto a stack that we can trust. */ /* * XXX this step is delayed in case recover_bootinfo needs to return via * the old stack, but it need not be, since recover_bootinfo actually * returns via the old frame. */ movl $R(tmpstk),%esp call identify_cpu call create_pagetables /* * If the CPU has support for VME, turn it on. */ testl $CPUID_VME, R(cpu_feature) jz 1f movl %cr4, %eax orl $CR4_VME, %eax movl %eax, %cr4 1: /* Now enable paging */ #if defined(PAE) || defined(PAE_TABLES) movl R(IdlePDPT), %eax movl %eax, %cr3 movl %cr4, %edx orl $CR4_PAE, %edx movl %edx, %cr4 #else movl R(IdlePTD), %eax movl %eax,%cr3 /* load ptd addr into mmu */ #endif movl %cr0,%edx /* get control word */ orl $CR0_PE|CR0_PG,%edx /* enable paging */ movl %edx,%cr0 /* and let's page NOW! */ pushl $begin /* jump to high virtualized address */ ret begin: /* * Now running relocated at KERNBASE where the system is linked to run. * * Remove the lowest part of the double mapping of low memory to get * some null pointer checks. */ movl $0,PTD movl %eax,%cr3 /* invalidate TLB */ /* set up bootstrap stack */ movl proc0kstack,%eax /* location of in-kernel stack */ /* * Only use bottom page for init386(). init386() calculates the * PCB + FPU save area size and returns the true top of stack. */ leal PAGE_SIZE(%eax),%esp xorl %ebp,%ebp /* mark end of frames */ pushl physfree /* value of first for init386(first) */ call init386 /* wire 386 chip for unix operation */ /* * Clean up the stack in a way that db_numargs() understands, so * that backtraces in ddb don't underrun the stack. Traps for * inaccessible memory are more fatal than usual this early. */ addl $4,%esp /* Switch to true top of stack. */ movl %eax,%esp call mi_startup /* autoconfiguration, mountroot etc */ /* NOTREACHED */ addl $0,%esp /* for db_numargs() again */ /********************************************************************** * * Recover the bootinfo passed to us from the boot program * */ recover_bootinfo: /* * This code is called in different ways depending on what loaded * and started the kernel. This is used to detect how we get the * arguments from the other code and what we do with them. * * Old disk boot blocks: * (*btext)(howto, bootdev, cyloffset, esym); * [return address == 0, and can NOT be returned to] * [cyloffset was not supported by the FreeBSD boot code * and always passed in as 0] * [esym is also known as total in the boot code, and * was never properly supported by the FreeBSD boot code] * * Old diskless netboot code: * (*btext)(0,0,0,0,&nfsdiskless,0,0,0); * [return address != 0, and can NOT be returned to] * If we are being booted by this code it will NOT work, * so we are just going to halt if we find this case. * * New uniform boot code: * (*btext)(howto, bootdev, 0, 0, 0, &bootinfo) * [return address != 0, and can be returned to] * * There may seem to be a lot of wasted arguments in here, but * that is so the newer boot code can still load very old kernels * and old boot code can load new kernels. */ /* * The old style disk boot blocks fake a frame on the stack and * did an lret to get here. The frame on the stack has a return * address of 0. */ cmpl $0,4(%ebp) je olddiskboot /* * We have some form of return address, so this is either the * old diskless netboot code, or the new uniform code. That can * be detected by looking at the 5th argument, if it is 0 * we are being booted by the new uniform boot code. */ cmpl $0,24(%ebp) je newboot /* * Seems we have been loaded by the old diskless boot code, we * don't stand a chance of running as the diskless structure * changed considerably between the two, so just halt. */ hlt /* * We have been loaded by the new uniform boot code. * Let's check the bootinfo version, and if we do not understand * it we return to the loader with a status of 1 to indicate this error */ newboot: movl 28(%ebp),%ebx /* &bootinfo.version */ movl BI_VERSION(%ebx),%eax cmpl $1,%eax /* We only understand version 1 */ je 1f movl $1,%eax /* Return status */ leave /* * XXX this returns to our caller's caller (as is required) since * we didn't set up a frame and our caller did. */ ret 1: /* * If we have a kernelname copy it in */ movl BI_KERNELNAME(%ebx),%esi cmpl $0,%esi je 2f /* No kernelname */ movl $MAXPATHLEN,%ecx /* Brute force!!! */ movl $R(kernelname),%edi cmpb $'/',(%esi) /* Make sure it starts with a slash */ je 1f movb $'/',(%edi) incl %edi decl %ecx 1: cld rep movsb 2: /* * Determine the size of the boot loader's copy of the bootinfo * struct. This is impossible to do properly because old versions * of the struct don't contain a size field and there are 2 old * versions with the same version number. */ movl $BI_ENDCOMMON,%ecx /* prepare for sizeless version */ testl $RB_BOOTINFO,8(%ebp) /* bi_size (and bootinfo) valid? */ je got_bi_size /* no, sizeless version */ movl BI_SIZE(%ebx),%ecx got_bi_size: /* * Copy the common part of the bootinfo struct */ movl %ebx,%esi movl $R(bootinfo),%edi cmpl $BOOTINFO_SIZE,%ecx jbe got_common_bi_size movl $BOOTINFO_SIZE,%ecx got_common_bi_size: cld rep movsb #ifdef NFS_ROOT #ifndef BOOTP_NFSV3 /* * If we have a nfs_diskless structure copy it in */ movl BI_NFS_DISKLESS(%ebx),%esi cmpl $0,%esi je olddiskboot movl $R(nfs_diskless),%edi movl $NFSDISKLESS_SIZE,%ecx cld rep movsb movl $R(nfs_diskless_valid),%edi movl $1,(%edi) #endif #endif /* * The old style disk boot. * (*btext)(howto, bootdev, cyloffset, esym); * Note that the newer boot code just falls into here to pick * up howto and bootdev, cyloffset and esym are no longer used */ olddiskboot: movl 8(%ebp),%eax movl %eax,R(boothowto) movl 12(%ebp),%eax movl %eax,R(bootdev) ret /********************************************************************** * * Identify the CPU and initialize anything special about it * */ identify_cpu: /* Try to toggle alignment check flag; does not exist on 386. */ pushfl popl %eax movl %eax,%ecx orl $PSL_AC,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_AC,%eax pushl %ecx popfl testl %eax,%eax jnz try486 /* NexGen CPU does not have aligment check flag. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jz trynexgen popfl movl $CPU_386,R(cpu) jmp 3f trynexgen: popfl movl $CPU_NX586,R(cpu) movl $0x4778654e,R(cpu_vendor) # store vendor string movl $0x72446e65,R(cpu_vendor+4) movl $0x6e657669,R(cpu_vendor+8) movl $0,R(cpu_vendor+12) jmp 3f try486: /* Try to toggle identification flag; does not exist on early 486s. */ pushfl popl %eax movl %eax,%ecx xorl $PSL_ID,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_ID,%eax pushl %ecx popfl testl %eax,%eax jnz trycpuid movl $CPU_486,R(cpu) /* * Check Cyrix CPU * Cyrix CPUs do not change the undefined flags following * execution of the divide instruction which divides 5 by 2. * * Note: CPUID is enabled on M2, so it passes another way. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jnc trycyrix popfl jmp 3f /* You may use Intel CPU. */ trycyrix: popfl /* * IBM Bluelighting CPU also doesn't change the undefined flags. * Because IBM doesn't disclose the information for Bluelighting * CPU, we couldn't distinguish it from Cyrix's (including IBM * brand of Cyrix CPUs). */ movl $0x69727943,R(cpu_vendor) # store vendor string movl $0x736e4978,R(cpu_vendor+4) movl $0x64616574,R(cpu_vendor+8) jmp 3f trycpuid: /* Use the `cpuid' instruction. */ xorl %eax,%eax cpuid # cpuid 0 movl %eax,R(cpu_high) # highest capability movl %ebx,R(cpu_vendor) # store vendor string movl %edx,R(cpu_vendor+4) movl %ecx,R(cpu_vendor+8) movb $0,R(cpu_vendor+12) movl $1,%eax cpuid # cpuid 1 movl %eax,R(cpu_id) # store cpu_id movl %ebx,R(cpu_procinfo) # store cpu_procinfo movl %edx,R(cpu_feature) # store cpu_feature movl %ecx,R(cpu_feature2) # store cpu_feature2 rorl $8,%eax # extract family type andl $15,%eax cmpl $5,%eax jae 1f /* less than Pentium; must be 486 */ movl $CPU_486,R(cpu) jmp 3f 1: /* a Pentium? */ cmpl $5,%eax jne 2f movl $CPU_586,R(cpu) jmp 3f 2: /* Greater than Pentium...call it a Pentium Pro */ movl $CPU_686,R(cpu) 3: ret /********************************************************************** * * Create the first page directory and its page tables. * */ create_pagetables: /* Find end of kernel image (rounded up to a page boundary). */ movl $R(_end),%esi /* Include symbols, if any. */ movl R(bootinfo+BI_ESYMTAB),%edi testl %edi,%edi je over_symalloc movl %edi,%esi movl $KERNBASE,%edi addl %edi,R(bootinfo+BI_SYMTAB) addl %edi,R(bootinfo+BI_ESYMTAB) over_symalloc: /* If we are told where the end of the kernel space is, believe it. */ movl R(bootinfo+BI_KERNEND),%edi testl %edi,%edi je no_kernend movl %edi,%esi no_kernend: addl $PDRMASK,%esi /* Play conservative for now, and */ andl $~PDRMASK,%esi /* ... round up to PDR boundary */ movl %esi,R(KERNend) /* save end of kernel */ movl %esi,R(physfree) /* next free page is at end of kernel */ /* Allocate Kernel Page Tables */ ALLOCPAGES(NKPT) movl %esi,R(KPTphys) addl $(KERNBASE-(KPTDI<<(PDRSHIFT-PAGE_SHIFT+PTESHIFT))),%esi movl %esi,R(KPTmap) /* Allocate Page Table Directory */ #if defined(PAE) || defined(PAE_TABLES) /* XXX only need 32 bytes (easier for now) */ ALLOCPAGES(1) movl %esi,R(IdlePDPT) #endif ALLOCPAGES(NPGPTD) movl %esi,R(IdlePTD) /* Allocate KSTACK */ ALLOCPAGES(TD0_KSTACK_PAGES) movl %esi,R(p0kpa) addl $KERNBASE, %esi movl %esi, R(proc0kstack) ALLOCPAGES(1) /* vm86/bios stack */ movl %esi,R(vm86phystk) ALLOCPAGES(3) /* pgtable + ext + IOPAGES */ movl %esi,R(vm86pa) addl $KERNBASE, %esi movl %esi, R(vm86paddr) /* * Enable PSE and PGE. */ #ifndef DISABLE_PSE testl $CPUID_PSE, R(cpu_feature) jz 1f movl $PG_PS, R(pseflag) movl %cr4, %eax orl $CR4_PSE, %eax movl %eax, %cr4 1: #endif #ifndef DISABLE_PG_G testl $CPUID_PGE, R(cpu_feature) jz 2f movl $PG_G, R(pgeflag) movl %cr4, %eax orl $CR4_PGE, %eax movl %eax, %cr4 2: #endif /* * Initialize page table pages mapping physical address zero through the * (physical) end of the kernel. Many of these pages must be reserved, * and we reserve them all and map them linearly for convenience. We do * this even if we've enabled PSE above; we'll just switch the corresponding * kernel PDEs before we turn on paging. * * XXX: We waste some pages here in the PSE case! * * This and all other page table entries allow read and write access for * various reasons. Kernel mappings never have any access restrictions. */ xorl %eax, %eax movl R(KERNend),%ecx shrl $PAGE_SHIFT,%ecx fillkptphys($PG_RW) /* Map page table pages. */ movl R(KPTphys),%eax movl $NKPT,%ecx fillkptphys($PG_RW) /* Map page directory. */ #if defined(PAE) || defined(PAE_TABLES) movl R(IdlePDPT), %eax movl $1, %ecx fillkptphys($PG_RW) #endif movl R(IdlePTD), %eax movl $NPGPTD, %ecx fillkptphys($PG_RW) /* Map proc0's KSTACK in the physical way ... */ movl R(p0kpa), %eax movl $(TD0_KSTACK_PAGES), %ecx fillkptphys($PG_RW) /* Map ISA hole */ movl $ISA_HOLE_START, %eax movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkptphys($PG_RW) /* Map space for the vm86 region */ movl R(vm86phystk), %eax movl $4, %ecx fillkptphys($PG_RW) /* Map page 0 into the vm86 page table */ movl $0, %eax movl $0, %ebx movl $1, %ecx fillkpt(R(vm86pa), $PG_RW|PG_U) /* ...likewise for the ISA hole */ movl $ISA_HOLE_START, %eax movl $ISA_HOLE_START>>PAGE_SHIFT, %ebx movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkpt(R(vm86pa), $PG_RW|PG_U) /* * Create an identity mapping for low physical memory, including the kernel. * This is only used to map the 2 instructions for jumping to 'begin' in * locore (we map everything to avoid having to determine where these * instructions are). ACPI resume will transiently restore the first PDE in * this mapping (and depend on this PDE's page table created here not being * destroyed). See pmap_bootstrap() for more details. * * Note: There are errata concerning large pages and physical address zero, * so a PG_PS mapping should not be used for PDE 0. Our double mapping * avoids this automatically by not using PG_PS for PDE #KPDI so that PAT * bits can be set at the page level for i/o pages below 1 MB. */ movl R(KPTphys), %eax xorl %ebx, %ebx movl $NKPT, %ecx fillkpt(R(IdlePTD), $PG_RW) /* * Install PDEs for PTs covering enough kva to bootstrap. Then for the PSE * case, replace the PDEs whose coverage is strictly within the kernel * (between KERNLOAD (rounded up) and KERNend) by large-page PDEs. */ movl R(KPTphys), %eax movl $KPTDI, %ebx movl $NKPT, %ecx fillkpt(R(IdlePTD), $PG_RW) cmpl $0,R(pseflag) je done_pde movl R(KERNend), %ecx movl $(KERNLOAD + PDRMASK) & ~PDRMASK, %eax subl %eax, %ecx shrl $PDRSHIFT, %ecx movl $KPTDI + ((KERNLOAD + PDRMASK) >> PDRSHIFT), %ebx shll $PDESHIFT, %ebx addl R(IdlePTD), %ebx orl $(PG_V|PG_RW|PG_PS), %eax 1: movl %eax, (%ebx) addl $(1 << PDRSHIFT), %eax addl $PDESIZE, %ebx loop 1b done_pde: /* install a pde recursively mapping page directory as a page table */ movl R(IdlePTD), %eax movl $PTDPTDI, %ebx movl $NPGPTD,%ecx fillkpt(R(IdlePTD), $PG_RW) #if defined(PAE) || defined(PAE_TABLES) movl R(IdlePTD), %eax xorl %ebx, %ebx movl $NPGPTD, %ecx fillkpt(R(IdlePDPT), $0x0) #endif ret #ifdef XENHVM /* Xen Hypercall page */ .text .p2align PAGE_SHIFT, 0x90 /* Hypercall_page needs to be PAGE aligned */ NON_GPROF_ENTRY(hypercall_page) .skip 0x1000, 0x90 /* Fill with "nop"s */ #endif Index: head/sys/i386/i386/mpboot.s =================================================================== --- head/sys/i386/i386/mpboot.s (revision 331253) +++ head/sys/i386/i386/mpboot.s (revision 331254) @@ -1,279 +1,279 @@ /*- * Copyright (c) 1995 Jack F. Vogel * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * mpboot.s: FreeBSD machine support for the Intel MP Spec * multiprocessor systems. * * $FreeBSD$ */ #include "opt_pmap.h" #include /* miscellaneous asm macros */ #include #include -#include "assym.s" +#include "assym.inc" #define R(x) ((x)-KERNBASE) /* * this code MUST be enabled here and in mp_machdep.c * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define CHECKPOINT(A,D) \ movb $(A),%al ; \ outb %al,$CMOS_REG ; \ movb $(D),%al ; \ outb %al,$CMOS_DATA #else #define CHECKPOINT(A,D) #endif /* CHECK_POINTS */ /* * the APs enter here from their trampoline code (bootMP, below) */ .p2align 4 NON_GPROF_ENTRY(MPentry) CHECKPOINT(0x36, 3) /* * Enable features on this processor. We don't support SMP on * CPUs older than a Pentium, so we know that we can use the cpuid * instruction. */ movl $1,%eax cpuid /* Retrieve features */ movl %cr4,%eax #ifndef DISABLE_PSE testl $CPUID_PSE,%edx jz 1f orl $CR4_PSE,%eax /* Enable PSE */ 1: #endif #ifndef DISABLE_PG_G testl $CPUID_PGE,%edx jz 1f orl $CR4_PGE,%eax /* Enable PGE */ 1: #endif testl $CPUID_VME,%edx jz 1f orl $CR4_VME,%eax /* Enable VME */ 1: movl %eax,%cr4 /* Now enable paging mode */ #if defined(PAE) || defined(PAE_TABLES) movl R(IdlePDPT), %eax movl %eax, %cr3 movl %cr4, %eax orl $CR4_PAE, %eax movl %eax, %cr4 #else movl R(IdlePTD), %eax movl %eax,%cr3 #endif movl %cr0,%eax orl $CR0_PE|CR0_PG,%eax /* enable paging */ movl %eax,%cr0 /* let the games begin! */ movl bootSTK,%esp /* boot stack end loc. */ pushl $mp_begin /* jump to high mem */ ret /* * Wait for the booting CPU to signal startup */ mp_begin: /* now running relocated at KERNBASE */ CHECKPOINT(0x37, 4) call init_secondary /* load i386 tables */ /* * This is the embedded trampoline or bootstrap that is * copied into 'real-mode' low memory, it is where the * secondary processor "wakes up". When it is executed * the processor will eventually jump into the routine * MPentry, which resides in normal kernel text above * 1Meg. -jackv */ .data ALIGN_DATA /* just to be sure */ BOOTMP1: NON_GPROF_ENTRY(bootMP) .code16 cli CHECKPOINT(0x34, 1) /* First guarantee a 'clean slate' */ xorl %eax, %eax movl %eax, %ebx movl %eax, %ecx movl %eax, %edx movl %eax, %esi movl %eax, %edi /* set up data segments */ mov %cs, %ax mov %ax, %ds mov %ax, %es mov %ax, %fs mov %ax, %gs mov %ax, %ss mov $(boot_stk-bootMP), %esp /* Now load the global descriptor table */ lgdt MP_GDTptr-bootMP /* Enable protected mode */ movl %cr0, %eax orl $CR0_PE, %eax movl %eax, %cr0 /* * make intrasegment jump to flush the processor pipeline and * reload CS register */ pushl $0x18 pushl $(protmode-bootMP) lretl .code32 protmode: CHECKPOINT(0x35, 2) /* * we are NOW running for the first time with %eip * having the full physical address, BUT we still * are using a segment descriptor with the origin * not matching the booting kernel. * * SO NOW... for the BIG Jump into kernel's segment * and physical text above 1 Meg. */ mov $0x10, %ebx movw %bx, %ds movw %bx, %es movw %bx, %fs movw %bx, %gs movw %bx, %ss .globl bigJump bigJump: /* this will be modified by mpInstallTramp() */ ljmp $0x08, $0 /* far jmp to MPentry() */ dead: hlt /* We should never get here */ jmp dead /* * MP boot strap Global Descriptor Table */ .p2align 4 .globl MP_GDT .globl bootCodeSeg .globl bootDataSeg MP_GDT: nulldesc: /* offset = 0x0 */ .word 0x0 .word 0x0 .byte 0x0 .byte 0x0 .byte 0x0 .byte 0x0 kernelcode: /* offset = 0x08 */ .word 0xffff /* segment limit 0..15 */ .word 0x0000 /* segment base 0..15 */ .byte 0x0 /* segment base 16..23; set for 0K */ .byte 0x9f /* flags; Type */ .byte 0xcf /* flags; Limit */ .byte 0x0 /* segment base 24..32 */ kerneldata: /* offset = 0x10 */ .word 0xffff /* segment limit 0..15 */ .word 0x0000 /* segment base 0..15 */ .byte 0x0 /* segment base 16..23; set for 0k */ .byte 0x93 /* flags; Type */ .byte 0xcf /* flags; Limit */ .byte 0x0 /* segment base 24..32 */ bootcode: /* offset = 0x18 */ .word 0xffff /* segment limit 0..15 */ bootCodeSeg: /* this will be modified by mpInstallTramp() */ .word 0x0000 /* segment base 0..15 */ .byte 0x00 /* segment base 16...23; set for 0x000xx000 */ .byte 0x9e /* flags; Type */ .byte 0xcf /* flags; Limit */ .byte 0x0 /*segment base 24..32 */ bootdata: /* offset = 0x20 */ .word 0xffff bootDataSeg: /* this will be modified by mpInstallTramp() */ .word 0x0000 /* segment base 0..15 */ .byte 0x00 /* segment base 16...23; set for 0x000xx000 */ .byte 0x92 .byte 0xcf .byte 0x0 /* * GDT pointer for the lgdt call */ .globl mp_gdtbase MP_GDTptr: mp_gdtlimit: .word 0x0028 mp_gdtbase: /* this will be modified by mpInstallTramp() */ .long 0 .space 0x100 /* space for boot_stk - 1st temporary stack */ boot_stk: BOOTMP2: .globl bootMP_size bootMP_size: .long BOOTMP2 - BOOTMP1 Index: head/sys/i386/i386/sigtramp.s =================================================================== --- head/sys/i386/i386/sigtramp.s (revision 331253) +++ head/sys/i386/i386/sigtramp.s (revision 331254) @@ -1,118 +1,118 @@ /*- * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 * $FreeBSD$ * * originally from: locore.s, by William F. Jolitz * * Substantially rewritten by David Greenman, Rod Grimes, * Bruce Evans, Wolfgang Solfrank, Poul-Henning Kamp * and many others. */ #include "opt_compat.h" #include #include #include -#include "assym.s" +#include "assym.inc" /* * Signal trampoline, copied to top of user stack */ NON_GPROF_ENTRY(sigcode) calll *SIGF_HANDLER(%esp) leal SIGF_UC(%esp),%eax /* get ucontext */ pushl %eax testl $PSL_VM,UC_EFLAGS(%eax) jne 1f mov UC_GS(%eax),%gs /* restore %gs */ 1: movl $SYS_sigreturn,%eax pushl %eax /* junk to fake return addr. */ int $0x80 /* enter kernel with args */ /* on stack */ 1: jmp 1b #ifdef COMPAT_FREEBSD4 ALIGN_TEXT freebsd4_sigcode: calll *SIGF_HANDLER(%esp) leal SIGF_UC4(%esp),%eax /* get ucontext */ pushl %eax testl $PSL_VM,UC4_EFLAGS(%eax) jne 1f mov UC4_GS(%eax),%gs /* restore %gs */ 1: movl $344,%eax /* 4.x SYS_sigreturn */ pushl %eax /* junk to fake return addr. */ int $0x80 /* enter kernel with args */ /* on stack */ 1: jmp 1b #endif #ifdef COMPAT_43 ALIGN_TEXT osigcode: call *SIGF_HANDLER(%esp) /* call signal handler */ lea SIGF_SC(%esp),%eax /* get sigcontext */ pushl %eax testl $PSL_VM,SC_PS(%eax) jne 9f mov SC_GS(%eax),%gs /* restore %gs */ 9: movl $103,%eax /* 3.x SYS_sigreturn */ pushl %eax /* junk to fake return addr. */ int $0x80 /* enter kernel with args */ 0: jmp 0b #endif /* COMPAT_43 */ ALIGN_TEXT esigcode: .data .globl szsigcode szsigcode: .long esigcode-sigcode #ifdef COMPAT_FREEBSD4 .globl szfreebsd4_sigcode szfreebsd4_sigcode: .long esigcode-freebsd4_sigcode #endif #ifdef COMPAT_43 .globl szosigcode szosigcode: .long esigcode-osigcode #endif Index: head/sys/i386/i386/support.s =================================================================== --- head/sys/i386/i386/support.s (revision 331253) +++ head/sys/i386/i386/support.s (revision 331254) @@ -1,837 +1,837 @@ /*- * Copyright (c) 1993 The Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include -#include "assym.s" +#include "assym.inc" #define IDXSHIFT 10 .text /* * bcopy family * void bzero(void *buf, u_int len) */ ENTRY(bzero) pushl %edi movl 8(%esp),%edi movl 12(%esp),%ecx xorl %eax,%eax shrl $2,%ecx rep stosl movl 12(%esp),%ecx andl $3,%ecx rep stosb popl %edi ret END(bzero) ENTRY(sse2_pagezero) pushl %ebx movl 8(%esp),%ecx movl %ecx,%eax addl $4096,%eax xor %ebx,%ebx jmp 1f /* * The loop takes 14 bytes. Ensure that it doesn't cross a 16-byte * cache line. */ .p2align 4,0x90 1: movnti %ebx,(%ecx) movnti %ebx,4(%ecx) addl $8,%ecx cmpl %ecx,%eax jne 1b sfence popl %ebx ret END(sse2_pagezero) ENTRY(i686_pagezero) pushl %edi pushl %ebx movl 12(%esp),%edi movl $1024,%ecx ALIGN_TEXT 1: xorl %eax,%eax repe scasl jnz 2f popl %ebx popl %edi ret ALIGN_TEXT 2: incl %ecx subl $4,%edi movl %ecx,%edx cmpl $16,%ecx jge 3f movl %edi,%ebx andl $0x3f,%ebx shrl %ebx shrl %ebx movl $16,%ecx subl %ebx,%ecx 3: subl %ecx,%edx rep stosl movl %edx,%ecx testl %edx,%edx jnz 1b popl %ebx popl %edi ret END(i686_pagezero) /* fillw(pat, base, cnt) */ ENTRY(fillw) pushl %edi movl 8(%esp),%eax movl 12(%esp),%edi movl 16(%esp),%ecx rep stosw popl %edi ret END(fillw) ENTRY(bcopyb) pushl %esi pushl %edi movl 12(%esp),%esi movl 16(%esp),%edi movl 20(%esp),%ecx movl %edi,%eax subl %esi,%eax cmpl %ecx,%eax /* overlapping && src < dst? */ jb 1f rep movsb popl %edi popl %esi ret ALIGN_TEXT 1: addl %ecx,%edi /* copy backwards. */ addl %ecx,%esi decl %edi decl %esi std rep movsb popl %edi popl %esi cld ret END(bcopyb) /* * bcopy(src, dst, cnt) * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 */ ENTRY(bcopy) pushl %ebp movl %esp,%ebp pushl %esi pushl %edi movl 8(%ebp),%esi movl 12(%ebp),%edi movl 16(%ebp),%ecx movl %edi,%eax subl %esi,%eax cmpl %ecx,%eax /* overlapping && src < dst? */ jb 1f shrl $2,%ecx /* copy by 32-bit words */ rep movsl movl 16(%ebp),%ecx andl $3,%ecx /* any bytes left? */ rep movsb popl %edi popl %esi popl %ebp ret ALIGN_TEXT 1: addl %ecx,%edi /* copy backwards */ addl %ecx,%esi decl %edi decl %esi andl $3,%ecx /* any fractional bytes? */ std rep movsb movl 16(%ebp),%ecx /* copy remainder by 32-bit words */ shrl $2,%ecx subl $3,%esi subl $3,%edi rep movsl popl %edi popl %esi cld popl %ebp ret END(bcopy) /* * Note: memcpy does not support overlapping copies */ ENTRY(memcpy) pushl %edi pushl %esi movl 12(%esp),%edi movl 16(%esp),%esi movl 20(%esp),%ecx movl %edi,%eax shrl $2,%ecx /* copy by 32-bit words */ rep movsl movl 20(%esp),%ecx andl $3,%ecx /* any bytes left? */ rep movsb popl %esi popl %edi ret END(memcpy) /*****************************************************************************/ /* copyout and fubyte family */ /*****************************************************************************/ /* * Access user memory from inside the kernel. These routines and possibly * the math- and DOS emulators should be the only places that do this. * * We have to access the memory with user's permissions, so use a segment * selector with RPL 3. For writes to user space we have to additionally * check the PTE for write permission, because the 386 does not check * write permissions when we are executing with EPL 0. The 486 does check * this if the WP bit is set in CR0, so we can use a simpler version here. * * These routines set curpcb->pcb_onfault for the time they execute. When a * protection violation occurs inside the functions, the trap handler * returns to *curpcb->pcb_onfault instead of the function. */ /* * copyout(from_kernel, to_user, len) - MP SAFE */ ENTRY(copyout) movl PCPU(CURPCB),%eax movl $copyout_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi pushl %ebx movl 16(%esp),%esi movl 20(%esp),%edi movl 24(%esp),%ebx testl %ebx,%ebx /* anything to do? */ jz done_copyout /* * Check explicitly for non-user addresses. This check is essential * because it prevents usermode from writing into the kernel. We do * not verify anywhere else that the user did not specify a rogue * address. */ /* * First, prevent address wrapping. */ movl %edi,%eax addl %ebx,%eax jc copyout_fault /* * XXX STOP USING VM_MAXUSER_ADDRESS. * It is an end address, not a max, so every time it is used correctly it * looks like there is an off by one error, and of course it caused an off * by one error in several places. */ cmpl $VM_MAXUSER_ADDRESS,%eax ja copyout_fault /* bcopy(%esi, %edi, %ebx) */ movl %ebx,%ecx shrl $2,%ecx rep movsl movb %bl,%cl andb $3,%cl rep movsb done_copyout: popl %ebx popl %edi popl %esi xorl %eax,%eax movl PCPU(CURPCB),%edx movl %eax,PCB_ONFAULT(%edx) ret END(copyout) ALIGN_TEXT copyout_fault: popl %ebx popl %edi popl %esi movl PCPU(CURPCB),%edx movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret /* * copyin(from_user, to_kernel, len) - MP SAFE */ ENTRY(copyin) movl PCPU(CURPCB),%eax movl $copyin_fault,PCB_ONFAULT(%eax) pushl %esi pushl %edi movl 12(%esp),%esi /* caddr_t from */ movl 16(%esp),%edi /* caddr_t to */ movl 20(%esp),%ecx /* size_t len */ /* * make sure address is valid */ movl %esi,%edx addl %ecx,%edx jc copyin_fault cmpl $VM_MAXUSER_ADDRESS,%edx ja copyin_fault movb %cl,%al shrl $2,%ecx /* copy longword-wise */ rep movsl movb %al,%cl andb $3,%cl /* copy remaining bytes */ rep movsb popl %edi popl %esi xorl %eax,%eax movl PCPU(CURPCB),%edx movl %eax,PCB_ONFAULT(%edx) ret END(copyin) ALIGN_TEXT copyin_fault: popl %edi popl %esi movl PCPU(CURPCB),%edx movl $0,PCB_ONFAULT(%edx) movl $EFAULT,%eax ret /* * casueword. Compare and set user word. Returns -1 on fault, * 0 on non-faulting access. The current value is in *oldp. */ ALTENTRY(casueword32) ENTRY(casueword) movl PCPU(CURPCB),%ecx movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx /* dst */ movl 8(%esp),%eax /* old */ movl 16(%esp),%ecx /* new */ cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address is valid */ ja fusufault #ifdef SMP lock #endif cmpxchgl %ecx,(%edx) /* Compare and set. */ /* * The old value is in %eax. If the store succeeded it will be the * value we expected (old) from before the store, otherwise it will * be the current value. */ movl PCPU(CURPCB),%ecx movl $0,PCB_ONFAULT(%ecx) movl 12(%esp),%edx /* oldp */ movl %eax,(%edx) xorl %eax,%eax ret END(casueword32) END(casueword) /* * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user * memory. */ ALTENTRY(fueword32) ENTRY(fueword) movl PCPU(CURPCB),%ecx movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx /* from */ cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address is valid */ ja fusufault movl (%edx),%eax movl $0,PCB_ONFAULT(%ecx) movl 8(%esp),%edx movl %eax,(%edx) xorl %eax,%eax ret END(fueword32) END(fueword) /* * fuswintr() and suswintr() are specialized variants of fuword16() and * suword16(), respectively. They are called from the profiling code, * potentially at interrupt time. If they fail, that's okay; good things * will happen later. They always fail for now, until the trap code is * able to deal with this. */ ALTENTRY(suswintr) ENTRY(fuswintr) movl $-1,%eax ret END(suswintr) END(fuswintr) ENTRY(fuword16) movl PCPU(CURPCB),%ecx movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-2,%edx ja fusufault movzwl (%edx),%eax movl $0,PCB_ONFAULT(%ecx) ret END(fuword16) ENTRY(fubyte) movl PCPU(CURPCB),%ecx movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-1,%edx ja fusufault movzbl (%edx),%eax movl $0,PCB_ONFAULT(%ecx) ret END(fubyte) ALIGN_TEXT fusufault: movl PCPU(CURPCB),%ecx xorl %eax,%eax movl %eax,PCB_ONFAULT(%ecx) decl %eax ret /* * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory. * All these functions are MPSAFE. */ ALTENTRY(suword32) ENTRY(suword) movl PCPU(CURPCB),%ecx movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-4,%edx /* verify address validity */ ja fusufault movl 8(%esp),%eax movl %eax,(%edx) xorl %eax,%eax movl PCPU(CURPCB),%ecx movl %eax,PCB_ONFAULT(%ecx) ret END(suword32) END(suword) ENTRY(suword16) movl PCPU(CURPCB),%ecx movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-2,%edx /* verify address validity */ ja fusufault movw 8(%esp),%ax movw %ax,(%edx) xorl %eax,%eax movl PCPU(CURPCB),%ecx /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret END(suword16) ENTRY(subyte) movl PCPU(CURPCB),%ecx movl $fusufault,PCB_ONFAULT(%ecx) movl 4(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-1,%edx /* verify address validity */ ja fusufault movb 8(%esp),%al movb %al,(%edx) xorl %eax,%eax movl PCPU(CURPCB),%ecx /* restore trashed register */ movl %eax,PCB_ONFAULT(%ecx) ret END(subyte) /* * copyinstr(from, to, maxlen, int *lencopied) - MP SAFE * * copy a string from 'from' to 'to', stop when a 0 character is reached. * return ENAMETOOLONG if string is longer than maxlen, and * EFAULT on protection violations. If lencopied is non-zero, * return the actual length in *lencopied. */ ENTRY(copyinstr) pushl %esi pushl %edi movl PCPU(CURPCB),%ecx movl $cpystrflt,PCB_ONFAULT(%ecx) movl 12(%esp),%esi /* %esi = from */ movl 16(%esp),%edi /* %edi = to */ movl 20(%esp),%edx /* %edx = maxlen */ movl $VM_MAXUSER_ADDRESS,%eax /* make sure 'from' is within bounds */ subl %esi,%eax jbe cpystrflt /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */ cmpl %edx,%eax jae 1f movl %eax,%edx movl %eax,20(%esp) 1: incl %edx 2: decl %edx jz 3f lodsb stosb orb %al,%al jnz 2b /* Success -- 0 byte reached */ decl %edx xorl %eax,%eax jmp cpystrflt_x 3: /* edx is zero - return ENAMETOOLONG or EFAULT */ cmpl $VM_MAXUSER_ADDRESS,%esi jae cpystrflt 4: movl $ENAMETOOLONG,%eax jmp cpystrflt_x cpystrflt: movl $EFAULT,%eax cpystrflt_x: /* set *lencopied and return %eax */ movl PCPU(CURPCB),%ecx movl $0,PCB_ONFAULT(%ecx) movl 20(%esp),%ecx subl %edx,%ecx movl 24(%esp),%edx testl %edx,%edx jz 1f movl %ecx,(%edx) 1: popl %edi popl %esi ret END(copyinstr) /* * copystr(from, to, maxlen, int *lencopied) - MP SAFE */ ENTRY(copystr) pushl %esi pushl %edi movl 12(%esp),%esi /* %esi = from */ movl 16(%esp),%edi /* %edi = to */ movl 20(%esp),%edx /* %edx = maxlen */ incl %edx 1: decl %edx jz 4f lodsb stosb orb %al,%al jnz 1b /* Success -- 0 byte reached */ decl %edx xorl %eax,%eax jmp 6f 4: /* edx is zero -- return ENAMETOOLONG */ movl $ENAMETOOLONG,%eax 6: /* set *lencopied and return %eax */ movl 20(%esp),%ecx subl %edx,%ecx movl 24(%esp),%edx testl %edx,%edx jz 7f movl %ecx,(%edx) 7: popl %edi popl %esi ret END(copystr) ENTRY(bcmp) pushl %edi pushl %esi movl 12(%esp),%edi movl 16(%esp),%esi movl 20(%esp),%edx movl %edx,%ecx shrl $2,%ecx repe cmpsl jne 1f movl %edx,%ecx andl $3,%ecx repe cmpsb 1: setne %al movsbl %al,%eax popl %esi popl %edi ret END(bcmp) /* * Handling of special 386 registers and descriptor tables etc */ /* void lgdt(struct region_descriptor *rdp); */ ENTRY(lgdt) /* reload the descriptor table */ movl 4(%esp),%eax lgdt (%eax) /* flush the prefetch q */ jmp 1f nop 1: /* reload "stale" selectors */ movl $KDSEL,%eax movl %eax,%ds movl %eax,%es movl %eax,%gs movl %eax,%ss movl $KPSEL,%eax movl %eax,%fs /* reload code selector by turning return into intersegmental return */ movl (%esp),%eax pushl %eax movl $KCSEL,4(%esp) MEXITCOUNT lret END(lgdt) /* ssdtosd(*ssdp,*sdp) */ ENTRY(ssdtosd) pushl %ebx movl 8(%esp),%ecx movl 8(%ecx),%ebx shll $16,%ebx movl (%ecx),%edx roll $16,%edx movb %dh,%bl movb %dl,%bh rorl $8,%ebx movl 4(%ecx),%eax movw %ax,%dx andl $0xf0000,%eax orl %eax,%ebx movl 12(%esp),%ecx movl %edx,(%ecx) movl %ebx,4(%ecx) popl %ebx ret END(ssdtosd) /* void reset_dbregs() */ ENTRY(reset_dbregs) movl $0,%eax movl %eax,%dr7 /* disable all breakpoints first */ movl %eax,%dr0 movl %eax,%dr1 movl %eax,%dr2 movl %eax,%dr3 movl %eax,%dr6 ret END(reset_dbregs) /*****************************************************************************/ /* setjump, longjump */ /*****************************************************************************/ ENTRY(setjmp) movl 4(%esp),%eax movl %ebx,(%eax) /* save ebx */ movl %esp,4(%eax) /* save esp */ movl %ebp,8(%eax) /* save ebp */ movl %esi,12(%eax) /* save esi */ movl %edi,16(%eax) /* save edi */ movl (%esp),%edx /* get rta */ movl %edx,20(%eax) /* save eip */ xorl %eax,%eax /* return(0); */ ret END(setjmp) ENTRY(longjmp) movl 4(%esp),%eax movl (%eax),%ebx /* restore ebx */ movl 4(%eax),%esp /* restore esp */ movl 8(%eax),%ebp /* restore ebp */ movl 12(%eax),%esi /* restore esi */ movl 16(%eax),%edi /* restore edi */ movl 20(%eax),%edx /* get rta */ movl %edx,(%esp) /* put in return frame */ xorl %eax,%eax /* return(1); */ incl %eax ret END(longjmp) /* * Support for reading MSRs in the safe manner. (Instead of panic on #gp, * return an error.) */ ENTRY(rdmsr_safe) /* int rdmsr_safe(u_int msr, uint64_t *data) */ movl PCPU(CURPCB),%ecx movl $msr_onfault,PCB_ONFAULT(%ecx) movl 4(%esp),%ecx rdmsr movl 8(%esp),%ecx movl %eax,(%ecx) movl %edx,4(%ecx) xorl %eax,%eax movl PCPU(CURPCB),%ecx movl %eax,PCB_ONFAULT(%ecx) ret /* * Support for writing MSRs in the safe manner. (Instead of panic on #gp, * return an error.) */ ENTRY(wrmsr_safe) /* int wrmsr_safe(u_int msr, uint64_t data) */ movl PCPU(CURPCB),%ecx movl $msr_onfault,PCB_ONFAULT(%ecx) movl 4(%esp),%ecx movl 8(%esp),%eax movl 12(%esp),%edx wrmsr xorl %eax,%eax movl PCPU(CURPCB),%ecx movl %eax,PCB_ONFAULT(%ecx) ret /* * MSR operations fault handler */ ALIGN_TEXT msr_onfault: movl PCPU(CURPCB),%ecx movl $0,PCB_ONFAULT(%ecx) movl $EFAULT,%eax ret ENTRY(handle_ibrs_entry) ret END(handle_ibrs_entry) ENTRY(handle_ibrs_exit) ret END(handle_ibrs_exit) Index: head/sys/i386/i386/swtch.s =================================================================== --- head/sys/i386/i386/swtch.s (revision 331253) +++ head/sys/i386/i386/swtch.s (revision 331254) @@ -1,471 +1,471 @@ /*- * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_sched.h" #include -#include "assym.s" +#include "assym.inc" #if defined(SMP) && defined(SCHED_ULE) #define SETOP xchgl #define BLOCK_SPIN(reg) \ movl $blocked_lock,%eax ; \ 100: ; \ lock ; \ cmpxchgl %eax,TD_LOCK(reg) ; \ jne 101f ; \ pause ; \ jmp 100b ; \ 101: #else #define SETOP movl #define BLOCK_SPIN(reg) #endif /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ .text /* * cpu_throw() * * This is the second half of cpu_switch(). It is used when the current * thread is either a dummy or slated to die, and we no longer care * about its state. This is only a slight optimization and is probably * not worth it anymore. Note that we need to clear the pm_active bits so * we do need the old proc if it still exists. * 0(%esp) = ret * 4(%esp) = oldtd * 8(%esp) = newtd */ ENTRY(cpu_throw) movl PCPU(CPUID), %esi movl 4(%esp),%ecx /* Old thread */ testl %ecx,%ecx /* no thread? */ jz 1f /* release bit from old pm_active */ movl PCPU(CURPMAP), %ebx #ifdef SMP lock #endif btrl %esi, PM_ACTIVE(%ebx) /* clear old */ 1: movl 8(%esp),%ecx /* New thread */ movl TD_PCB(%ecx),%edx movl PCB_CR3(%edx),%eax movl %eax,%cr3 /* set bit in new pm_active */ movl TD_PROC(%ecx),%eax movl P_VMSPACE(%eax), %ebx addl $VM_PMAP, %ebx movl %ebx, PCPU(CURPMAP) #ifdef SMP lock #endif btsl %esi, PM_ACTIVE(%ebx) /* set new */ jmp sw1 END(cpu_throw) /* * cpu_switch(old, new) * * Save the current thread state, then select the next thread to run * and load its state. * 0(%esp) = ret * 4(%esp) = oldtd * 8(%esp) = newtd * 12(%esp) = newlock */ ENTRY(cpu_switch) /* Switch to new thread. First, save context. */ movl 4(%esp),%ecx #ifdef INVARIANTS testl %ecx,%ecx /* no thread? */ jz badsw2 /* no, panic */ #endif movl TD_PCB(%ecx),%edx movl (%esp),%eax /* Hardware registers */ movl %eax,PCB_EIP(%edx) movl %ebx,PCB_EBX(%edx) movl %esp,PCB_ESP(%edx) movl %ebp,PCB_EBP(%edx) movl %esi,PCB_ESI(%edx) movl %edi,PCB_EDI(%edx) mov %gs,PCB_GS(%edx) /* Test if debug registers should be saved. */ testl $PCB_DBREGS,PCB_FLAGS(%edx) jz 1f /* no, skip over */ movl %dr7,%eax /* yes, do the save */ movl %eax,PCB_DR7(%edx) andl $0x0000fc00, %eax /* disable all watchpoints */ movl %eax,%dr7 movl %dr6,%eax movl %eax,PCB_DR6(%edx) movl %dr3,%eax movl %eax,PCB_DR3(%edx) movl %dr2,%eax movl %eax,PCB_DR2(%edx) movl %dr1,%eax movl %eax,PCB_DR1(%edx) movl %dr0,%eax movl %eax,PCB_DR0(%edx) 1: /* have we used fp, and need a save? */ cmpl %ecx,PCPU(FPCURTHREAD) jne 1f pushl PCB_SAVEFPU(%edx) /* h/w bugs make saving complicated */ call npxsave /* do it in a big C function */ popl %eax 1: /* Save is done. Now fire up new thread. Leave old vmspace. */ movl 4(%esp),%edi movl 8(%esp),%ecx /* New thread */ movl 12(%esp),%esi /* New lock */ #ifdef INVARIANTS testl %ecx,%ecx /* no thread? */ jz badsw3 /* no, panic */ #endif movl TD_PCB(%ecx),%edx /* switch address space */ movl PCB_CR3(%edx),%eax movl %cr3,%ebx /* The same address space? */ cmpl %ebx,%eax je sw0 movl %eax,%cr3 /* new address space */ movl %esi,%eax movl PCPU(CPUID),%esi SETOP %eax,TD_LOCK(%edi) /* Switchout td_lock */ /* Release bit from old pmap->pm_active */ movl PCPU(CURPMAP), %ebx #ifdef SMP lock #endif btrl %esi, PM_ACTIVE(%ebx) /* clear old */ /* Set bit in new pmap->pm_active */ movl TD_PROC(%ecx),%eax /* newproc */ movl P_VMSPACE(%eax), %ebx addl $VM_PMAP, %ebx movl %ebx, PCPU(CURPMAP) #ifdef SMP lock #endif btsl %esi, PM_ACTIVE(%ebx) /* set new */ jmp sw1 sw0: SETOP %esi,TD_LOCK(%edi) /* Switchout td_lock */ sw1: BLOCK_SPIN(%ecx) /* * At this point, we've switched address spaces and are ready * to load up the rest of the next context. */ cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f /* If not, use the default */ movl $1, PCPU(PRIVATE_TSS) /* mark use of private tss */ movl PCB_EXT(%edx), %edi /* new tss descriptor */ jmp 2f /* Load it up */ 1: /* * Use the common default TSS instead of our own. * Set our stack pointer into the TSS, it's set to just * below the PCB. In C, common_tss.tss_esp0 = &pcb - 16; */ leal -16(%edx), %ebx /* leave space for vm86 */ movl %ebx, PCPU(COMMON_TSS) + TSS_ESP0 /* * Test this CPU's bit in the bitmap to see if this * CPU was using a private TSS. */ cmpl $0, PCPU(PRIVATE_TSS) /* Already using the common? */ je 3f /* if so, skip reloading */ movl $0, PCPU(PRIVATE_TSS) PCPU_ADDR(COMMON_TSSD, %edi) 2: /* Move correct tss descriptor into GDT slot, then reload tr. */ movl PCPU(TSS_GDT), %ebx /* entry in GDT */ movl 0(%edi), %eax movl 4(%edi), %esi movl %eax, 0(%ebx) movl %esi, 4(%ebx) movl $GPROC0_SEL*8, %esi /* GSEL(GPROC0_SEL, SEL_KPL) */ ltr %si 3: /* Copy the %fs and %gs selectors into this pcpu gdt */ leal PCB_FSD(%edx), %esi movl PCPU(FSGS_GDT), %edi movl 0(%esi), %eax /* %fs selector */ movl 4(%esi), %ebx movl %eax, 0(%edi) movl %ebx, 4(%edi) movl 8(%esi), %eax /* %gs selector, comes straight after */ movl 12(%esi), %ebx movl %eax, 8(%edi) movl %ebx, 12(%edi) /* Restore context. */ movl PCB_EBX(%edx),%ebx movl PCB_ESP(%edx),%esp movl PCB_EBP(%edx),%ebp movl PCB_ESI(%edx),%esi movl PCB_EDI(%edx),%edi movl PCB_EIP(%edx),%eax movl %eax,(%esp) movl %edx, PCPU(CURPCB) movl %ecx, PCPU(CURTHREAD) /* into next thread */ /* * Determine the LDT to use and load it if is the default one and * that is not the current one. */ movl TD_PROC(%ecx),%eax cmpl $0,P_MD+MD_LDT(%eax) jnz 1f movl _default_ldt,%eax cmpl PCPU(CURRENTLDT),%eax je 2f lldt _default_ldt movl %eax,PCPU(CURRENTLDT) jmp 2f 1: /* Load the LDT when it is not the default one. */ pushl %edx /* Preserve pointer to pcb. */ addl $P_MD,%eax /* Pointer to mdproc is arg. */ pushl %eax /* * Holding dt_lock prevents context switches, so dt_lock cannot * be held now and set_user_ldt() will not deadlock acquiring it. */ call set_user_ldt addl $4,%esp popl %edx 2: /* This must be done after loading the user LDT. */ .globl cpu_switch_load_gs cpu_switch_load_gs: mov PCB_GS(%edx),%gs /* Test if debug registers should be restored. */ testl $PCB_DBREGS,PCB_FLAGS(%edx) jz 1f /* * Restore debug registers. The special code for dr7 is to * preserve the current values of its reserved bits. */ movl PCB_DR6(%edx),%eax movl %eax,%dr6 movl PCB_DR3(%edx),%eax movl %eax,%dr3 movl PCB_DR2(%edx),%eax movl %eax,%dr2 movl PCB_DR1(%edx),%eax movl %eax,%dr1 movl PCB_DR0(%edx),%eax movl %eax,%dr0 movl %dr7,%eax andl $0x0000fc00,%eax movl PCB_DR7(%edx),%ecx andl $~0x0000fc00,%ecx orl %ecx,%eax movl %eax,%dr7 1: ret #ifdef INVARIANTS badsw1: pushal pushl $sw0_1 call panic sw0_1: .asciz "cpu_throw: no newthread supplied" badsw2: pushal pushl $sw0_2 call panic sw0_2: .asciz "cpu_switch: no curthread supplied" badsw3: pushal pushl $sw0_3 call panic sw0_3: .asciz "cpu_switch: no newthread supplied" #endif END(cpu_switch) /* * savectx(pcb) * Update pcb, saving current processor state. */ ENTRY(savectx) /* Fetch PCB. */ movl 4(%esp),%ecx /* Save caller's return address. Child won't execute this routine. */ movl (%esp),%eax movl %eax,PCB_EIP(%ecx) movl %cr3,%eax movl %eax,PCB_CR3(%ecx) movl %ebx,PCB_EBX(%ecx) movl %esp,PCB_ESP(%ecx) movl %ebp,PCB_EBP(%ecx) movl %esi,PCB_ESI(%ecx) movl %edi,PCB_EDI(%ecx) mov %gs,PCB_GS(%ecx) movl %cr0,%eax movl %eax,PCB_CR0(%ecx) movl %cr2,%eax movl %eax,PCB_CR2(%ecx) movl %cr4,%eax movl %eax,PCB_CR4(%ecx) movl %dr0,%eax movl %eax,PCB_DR0(%ecx) movl %dr1,%eax movl %eax,PCB_DR1(%ecx) movl %dr2,%eax movl %eax,PCB_DR2(%ecx) movl %dr3,%eax movl %eax,PCB_DR3(%ecx) movl %dr6,%eax movl %eax,PCB_DR6(%ecx) movl %dr7,%eax movl %eax,PCB_DR7(%ecx) mov %ds,PCB_DS(%ecx) mov %es,PCB_ES(%ecx) mov %fs,PCB_FS(%ecx) mov %ss,PCB_SS(%ecx) sgdt PCB_GDT(%ecx) sidt PCB_IDT(%ecx) sldt PCB_LDT(%ecx) str PCB_TR(%ecx) movl $1,%eax ret END(savectx) /* * resumectx(pcb) __fastcall * Resuming processor state from pcb. */ ENTRY(resumectx) /* Restore GDT. */ lgdt PCB_GDT(%ecx) /* Restore segment registers */ movzwl PCB_DS(%ecx),%eax mov %ax,%ds movzwl PCB_ES(%ecx),%eax mov %ax,%es movzwl PCB_FS(%ecx),%eax mov %ax,%fs movzwl PCB_GS(%ecx),%eax movw %ax,%gs movzwl PCB_SS(%ecx),%eax mov %ax,%ss /* Restore CR2, CR4, CR3 and CR0 */ movl PCB_CR2(%ecx),%eax movl %eax,%cr2 movl PCB_CR4(%ecx),%eax movl %eax,%cr4 movl PCB_CR3(%ecx),%eax movl %eax,%cr3 movl PCB_CR0(%ecx),%eax movl %eax,%cr0 jmp 1f 1: /* Restore descriptor tables */ lidt PCB_IDT(%ecx) lldt PCB_LDT(%ecx) #define SDT_SYS386TSS 9 #define SDT_SYS386BSY 11 /* Clear "task busy" bit and reload TR */ movl PCPU(TSS_GDT),%eax andb $(~SDT_SYS386BSY | SDT_SYS386TSS),5(%eax) movzwl PCB_TR(%ecx),%eax ltr %ax #undef SDT_SYS386TSS #undef SDT_SYS386BSY /* Restore debug registers */ movl PCB_DR0(%ecx),%eax movl %eax,%dr0 movl PCB_DR1(%ecx),%eax movl %eax,%dr1 movl PCB_DR2(%ecx),%eax movl %eax,%dr2 movl PCB_DR3(%ecx),%eax movl %eax,%dr3 movl PCB_DR6(%ecx),%eax movl %eax,%dr6 movl PCB_DR7(%ecx),%eax movl %eax,%dr7 /* Restore other registers */ movl PCB_EDI(%ecx),%edi movl PCB_ESI(%ecx),%esi movl PCB_EBP(%ecx),%ebp movl PCB_ESP(%ecx),%esp movl PCB_EBX(%ecx),%ebx /* reload code selector by turning return into intersegmental return */ pushl PCB_EIP(%ecx) movl $KCSEL,4(%esp) xorl %eax,%eax lret END(resumectx) Index: head/sys/i386/i386/vm86bios.s =================================================================== --- head/sys/i386/i386/vm86bios.s (revision 331253) +++ head/sys/i386/i386/vm86bios.s (revision 331254) @@ -1,173 +1,173 @@ /*- * Copyright (c) 1998 Jonathan Lemon * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include /* miscellaneous asm macros */ #include -#include "assym.s" +#include "assym.inc" #define SCR_NEWPTD PCB_ESI /* readability macros */ #define SCR_VMFRAME PCB_EBP /* see vm86.c for explanation */ #define SCR_STACK PCB_ESP #define SCR_PGTABLE PCB_EBX #define SCR_ARGFRAME PCB_EIP #define SCR_TSS0 PCB_VM86 #define SCR_TSS1 (PCB_VM86+4) .data ALIGN_DATA .globl vm86pcb vm86pcb: .long 0 .text /* * vm86_bioscall(struct trapframe_vm86 *vm86) */ ENTRY(vm86_bioscall) movl vm86pcb,%edx /* scratch data area */ movl 4(%esp),%eax movl %eax,SCR_ARGFRAME(%edx) /* save argument pointer */ pushl %ebx pushl %ebp pushl %esi pushl %edi pushl %gs pushfl cli movl PCPU(CURTHREAD),%ecx cmpl %ecx,PCPU(FPCURTHREAD) /* do we need to save fp? */ jne 1f pushl %edx movl TD_PCB(%ecx),%ecx pushl PCB_SAVEFPU(%ecx) call npxsave addl $4,%esp popl %edx /* recover our pcb */ 1: popfl movl SCR_VMFRAME(%edx),%ebx /* target frame location */ movl %ebx,%edi /* destination */ movl SCR_ARGFRAME(%edx),%esi /* source (set on entry) */ movl $VM86_FRAMESIZE/4,%ecx /* sizeof(struct vm86frame)/4 */ cld rep movsl /* copy frame to new stack */ movl PCPU(CURPCB),%eax pushl %eax /* save curpcb */ movl %edx,PCPU(CURPCB) /* set curpcb to vm86pcb */ movl PCPU(TSS_GDT),%ebx /* entry in GDT */ movl 0(%ebx),%eax movl %eax,SCR_TSS0(%edx) /* save first word */ movl 4(%ebx),%eax andl $~0x200, %eax /* flip 386BSY -> 386TSS */ movl %eax,SCR_TSS1(%edx) /* save second word */ movl PCB_EXT(%edx),%edi /* vm86 tssd entry */ movl 0(%edi),%eax movl %eax,0(%ebx) movl 4(%edi),%eax movl %eax,4(%ebx) movl $GPROC0_SEL*8,%esi /* GSEL(entry, SEL_KPL) */ ltr %si movl %cr3,%eax pushl %eax /* save address space */ movl IdlePTD,%ecx movl %ecx,%ebx addl $KERNBASE,%ebx /* va of Idle PTD */ movl 0(%ebx),%eax pushl %eax /* old ptde != 0 when booting */ pushl %ebx /* keep for reuse */ movl %esp,SCR_STACK(%edx) /* save current stack location */ movl SCR_NEWPTD(%edx),%eax /* mapping for vm86 page table */ movl %eax,0(%ebx) /* ... install as PTD entry 0 */ #if defined(PAE) || defined(PAE_TABLES) movl IdlePDPT,%ecx #endif movl %ecx,%cr3 /* new page tables */ movl SCR_VMFRAME(%edx),%esp /* switch to new stack */ pushl %esp call vm86_prepcall /* finish setup */ add $4, %esp /* * Return via doreti */ MEXITCOUNT jmp doreti /* * vm86_biosret(struct trapframe_vm86 *vm86) */ ENTRY(vm86_biosret) movl vm86pcb,%edx /* data area */ movl 4(%esp),%esi /* source */ movl SCR_ARGFRAME(%edx),%edi /* destination */ movl $VM86_FRAMESIZE/4,%ecx /* size */ cld rep movsl /* copy frame to original frame */ movl SCR_STACK(%edx),%esp /* back to old stack */ popl %ebx /* saved va of Idle PTD */ popl %eax movl %eax,0(%ebx) /* restore old pte */ popl %eax movl %eax,%cr3 /* install old page table */ movl PCPU(TSS_GDT),%ebx /* entry in GDT */ movl SCR_TSS0(%edx),%eax movl %eax,0(%ebx) /* restore first word */ movl SCR_TSS1(%edx),%eax movl %eax,4(%ebx) /* restore second word */ movl $GPROC0_SEL*8,%esi /* GSEL(entry, SEL_KPL) */ ltr %si popl PCPU(CURPCB) /* restore curpcb/curproc */ movl SCR_ARGFRAME(%edx),%edx /* original stack frame */ movl TF_TRAPNO(%edx),%eax /* return (trapno) */ popl %gs popl %edi popl %esi popl %ebp popl %ebx ret /* back to our normal program */ Index: head/sys/i386/linux/linux_locore.s =================================================================== --- head/sys/i386/linux/linux_locore.s (revision 331253) +++ head/sys/i386/linux/linux_locore.s (revision 331254) @@ -1,149 +1,149 @@ /* $FreeBSD$ */ #include "linux_assym.h" /* system definitions */ #include /* miscellaneous asm macros */ #include /* system call numbers */ -#include "assym.s" +#include "assym.inc" /* * To avoid excess stack frame the signal trampoline code emulates * the 'call' instruction. */ NON_GPROF_ENTRY(linux_sigcode) movl %esp, %ebx /* preserve sigframe */ call .getip0 .getip0: popl %eax add $.startsigcode-.getip0, %eax /* ret address */ push %eax jmp *LINUX_SIGF_HANDLER(%ebx) .startsigcode: popl %eax /* gcc unwind code need this */ movl $LINUX_SYS_linux_sigreturn,%eax /* linux_sigreturn() */ int $0x80 /* enter kernel with args */ .endsigcode: 0: jmp 0b NON_GPROF_ENTRY(linux_rt_sigcode) leal LINUX_RT_SIGF_UC(%esp),%ebx /* linux ucp */ leal LINUX_RT_SIGF_SC(%ebx),%ecx /* linux sigcontext */ movl %esp, %edi call .getip1 .getip1: popl %eax add $.startrtsigcode-.getip1, %eax /* ret address */ push %eax jmp *LINUX_RT_SIGF_HANDLER(%edi) .startrtsigcode: movl $LINUX_SYS_linux_rt_sigreturn,%eax /* linux_rt_sigreturn() */ int $0x80 /* enter kernel with args */ .endrtsigcode: 0: jmp 0b NON_GPROF_ENTRY(linux_vsyscall) .startvsyscall: int $0x80 ret .endvsyscall: #if 0 .section .note.Linux, "a",@note .long 2f - 1f /* namesz */ .balign 4 .long 4f - 3f /* descsz */ .long 0 1: .asciz "Linux" 2: .balign 4 3: .long LINUX_VERSION_CODE 4: .balign 4 .previous #endif #define do_cfa_expr(offset) \ .byte 0x0f; /* DW_CFA_def_cfa_expression */ \ .uleb128 11f-10f; /* length */ \ 10: .byte 0x74; /* DW_OP_breg4 */ \ .sleb128 offset; /* offset */ \ .byte 0x06; /* DW_OP_deref */ \ 11: /* CIE */ .section .eh_frame,"a",@progbits .LSTARTFRAMEDLSI1: .long .LENDCIEDLSI1-.LSTARTCIEDLSI1 .LSTARTCIEDLSI1: .long 0 /* CIE ID */ .byte 1 /* Version number */ .string "zRS" /* NULL-terminated * augmentation string */ .uleb128 1 /* Code alignment factor */ .sleb128 -4 /* Data alignment factor */ .byte 8 /* Return address * register column */ .uleb128 1 /* Augmentation value length */ .byte 0x1b /* DW_EH_PE_pcrel|DW_EH_PE_sdata4. */ .byte 0 /* DW_CFA_nop */ .align 4 .LENDCIEDLSI1: /* FDE */ .long .LENDFDEDLSI1-.LSTARTFDEDLSI1 /* Length FDE */ .LSTARTFDEDLSI1: .long .LSTARTFDEDLSI1-.LSTARTFRAMEDLSI1 /* CIE pointer */ .long .startsigcode-. /* PC-relative start address */ .long .endsigcode-.startsigcode .uleb128 0 /* Augmentation */ do_cfa_expr(LINUX_SIGF_SC-8) .align 4 .LENDFDEDLSI1: .long .LENDFDEDLSI2-.LSTARTFDEDLSI2 /* Length FDE */ .LSTARTFDEDLSI2: .long .LSTARTFDEDLSI2-.LSTARTFRAMEDLSI1 /* CIE pointer */ .long .startrtsigcode-. /* PC-relative start address */ .long .endrtsigcode-.startrtsigcode .uleb128 0 /* Augmentation */ do_cfa_expr(LINUX_RT_SIGF_SC-4+LINUX_SC_ESP) .align 4 .LENDFDEDLSI2: .previous .section .eh_frame,"a",@progbits .LSTARTFRAMEDLSI2: .long .LENDCIEDLSI2-.LSTARTCIEDLSI2 .LSTARTCIEDLSI2: .long 0 /* CIE ID */ .byte 1 /* Version number */ .string "zR" /* NULL-terminated * augmentation string */ .uleb128 1 /* Code alignment factor */ .sleb128 -4 /* Data alignment factor */ .byte 8 /* Return address register column */ .uleb128 1 /* Augmentation value length */ .byte 0x1b /* DW_EH_PE_pcrel|DW_EH_PE_sdata4. */ .byte 0x0c /* DW_CFA_def_cfa */ .uleb128 4 .uleb128 4 .byte 0x88 /* DW_CFA_offset, column 0x8 */ .uleb128 1 .align 4 .LENDCIEDLSI2: .long .LENDFDEDLSI3-.LSTARTFDEDLSI3 /* Length FDE */ .LSTARTFDEDLSI3: .long .LSTARTFDEDLSI3-.LSTARTFRAMEDLSI2 /* CIE pointer */ .long .startvsyscall-. /* PC-relative start address */ .long .endvsyscall-.startvsyscall .uleb128 0 .align 4 .LENDFDEDLSI3: .previous Index: head/sys/i386/linux/linux_support.s =================================================================== --- head/sys/i386/linux/linux_support.s (revision 331253) +++ head/sys/i386/linux/linux_support.s (revision 331254) @@ -1,126 +1,126 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2006,2007 Konstantin Belousov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "linux_assym.h" /* system definitions */ #include /* miscellaneous asm macros */ -#include "assym.s" +#include "assym.inc" futex_fault_decx: movl PCPU(CURPCB),%ecx futex_fault: movl $0,PCB_ONFAULT(%ecx) movl $-EFAULT,%eax ret ENTRY(futex_xchgl) movl PCPU(CURPCB),%ecx movl $futex_fault,PCB_ONFAULT(%ecx) movl 4(%esp),%eax movl 8(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-4,%edx ja futex_fault xchgl %eax,(%edx) movl 12(%esp),%edx movl %eax,(%edx) xorl %eax,%eax movl %eax,PCB_ONFAULT(%ecx) ret ENTRY(futex_addl) movl PCPU(CURPCB),%ecx movl $futex_fault,PCB_ONFAULT(%ecx) movl 4(%esp),%eax movl 8(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-4,%edx ja futex_fault #ifdef SMP lock #endif xaddl %eax,(%edx) movl 12(%esp),%edx movl %eax,(%edx) xorl %eax,%eax movl %eax,PCB_ONFAULT(%ecx) ret ENTRY(futex_orl) movl PCPU(CURPCB),%ecx movl $futex_fault_decx,PCB_ONFAULT(%ecx) movl 8(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-4,%edx ja futex_fault movl (%edx),%eax 1: movl %eax,%ecx orl 4(%esp),%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%edx) jnz 1b futex_tail: movl 12(%esp),%edx movl %eax,(%edx) xorl %eax,%eax movl PCPU(CURPCB),%ecx movl %eax,PCB_ONFAULT(%ecx) ret ENTRY(futex_andl) movl PCPU(CURPCB),%ecx movl $futex_fault_decx,PCB_ONFAULT(%ecx) movl 8(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-4,%edx ja futex_fault movl (%edx),%eax 1: movl %eax,%ecx andl 4(%esp),%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%edx) jnz 1b jmp futex_tail ENTRY(futex_xorl) movl PCPU(CURPCB),%ecx movl $futex_fault_decx,PCB_ONFAULT(%ecx) movl 8(%esp),%edx cmpl $VM_MAXUSER_ADDRESS-4,%edx ja futex_fault movl (%edx),%eax 1: movl %eax,%ecx xorl 4(%esp),%ecx #ifdef SMP lock #endif cmpxchgl %ecx,(%edx) jnz 1b jmp futex_tail Index: head/sys/mips/cavium/octeon_cop2.S =================================================================== --- head/sys/mips/cavium/octeon_cop2.S (revision 331253) +++ head/sys/mips/cavium/octeon_cop2.S (revision 331254) @@ -1,225 +1,225 @@ /*- * Copyright (c) 2011 Oleksandr Tymoshenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include -#include "assym.s" +#include "assym.inc" .set noreorder #define SAVE_COP2_REGISTER(reg) \ dmfc2 t1, reg; sd t1, reg##_OFFSET(a0) #define RESTORE_COP2_REGISTER(reg) \ ld t1, reg##_OFFSET(a0); dmtc2 t1, reg##_SET LEAF(octeon_cop2_save) /* Get CvmCtl register */ dmfc0 t0, $9, 7 /* CRC state */ SAVE_COP2_REGISTER(COP2_CRC_IV) SAVE_COP2_REGISTER(COP2_CRC_LENGTH) SAVE_COP2_REGISTER(COP2_CRC_POLY) /* if CvmCtl[NODFA_CP2] -> save_nodfa */ bbit1 t0, 28, save_nodfa nop /* LLM state */ SAVE_COP2_REGISTER(COP2_LLM_DAT0) SAVE_COP2_REGISTER(COP2_LLM_DAT1) save_nodfa: /* crypto stuff is irrelevant if CvmCtl[NOCRYPTO] */ bbit1 t0, 26, save_done nop SAVE_COP2_REGISTER(COP2_3DES_IV) SAVE_COP2_REGISTER(COP2_3DES_KEY0) SAVE_COP2_REGISTER(COP2_3DES_KEY1) SAVE_COP2_REGISTER(COP2_3DES_KEY2) SAVE_COP2_REGISTER(COP2_3DES_RESULT) SAVE_COP2_REGISTER(COP2_AES_INP0) SAVE_COP2_REGISTER(COP2_AES_IV0) SAVE_COP2_REGISTER(COP2_AES_IV1) SAVE_COP2_REGISTER(COP2_AES_KEY0) SAVE_COP2_REGISTER(COP2_AES_KEY1) SAVE_COP2_REGISTER(COP2_AES_KEY2) SAVE_COP2_REGISTER(COP2_AES_KEY3) SAVE_COP2_REGISTER(COP2_AES_KEYLEN) SAVE_COP2_REGISTER(COP2_AES_RESULT0) SAVE_COP2_REGISTER(COP2_AES_RESULT1) dmfc0 t0, $15 li t1, 0x000d0000 /* Octeon Pass1 */ beq t0, t1, save_pass1 nop SAVE_COP2_REGISTER(COP2_HSH_DATW0) SAVE_COP2_REGISTER(COP2_HSH_DATW1) SAVE_COP2_REGISTER(COP2_HSH_DATW2) SAVE_COP2_REGISTER(COP2_HSH_DATW3) SAVE_COP2_REGISTER(COP2_HSH_DATW4) SAVE_COP2_REGISTER(COP2_HSH_DATW5) SAVE_COP2_REGISTER(COP2_HSH_DATW6) SAVE_COP2_REGISTER(COP2_HSH_DATW7) SAVE_COP2_REGISTER(COP2_HSH_DATW8) SAVE_COP2_REGISTER(COP2_HSH_DATW9) SAVE_COP2_REGISTER(COP2_HSH_DATW10) SAVE_COP2_REGISTER(COP2_HSH_DATW11) SAVE_COP2_REGISTER(COP2_HSH_DATW12) SAVE_COP2_REGISTER(COP2_HSH_DATW13) SAVE_COP2_REGISTER(COP2_HSH_DATW14) SAVE_COP2_REGISTER(COP2_HSH_IVW0) SAVE_COP2_REGISTER(COP2_HSH_IVW1) SAVE_COP2_REGISTER(COP2_HSH_IVW2) SAVE_COP2_REGISTER(COP2_HSH_IVW3) SAVE_COP2_REGISTER(COP2_HSH_IVW4) SAVE_COP2_REGISTER(COP2_HSH_IVW5) SAVE_COP2_REGISTER(COP2_HSH_IVW6) SAVE_COP2_REGISTER(COP2_HSH_IVW7) SAVE_COP2_REGISTER(COP2_GFM_MULT0) SAVE_COP2_REGISTER(COP2_GFM_MULT1) SAVE_COP2_REGISTER(COP2_GFM_POLY) SAVE_COP2_REGISTER(COP2_GFM_RESULT0) SAVE_COP2_REGISTER(COP2_GFM_RESULT1) jr ra nop save_pass1: SAVE_COP2_REGISTER(COP2_HSH_DATW0_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW1_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW2_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW3_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW4_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW5_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW6_PASS1) SAVE_COP2_REGISTER(COP2_HSH_IVW0_PASS1) SAVE_COP2_REGISTER(COP2_HSH_IVW1_PASS1) SAVE_COP2_REGISTER(COP2_HSH_IVW2_PASS1) save_done: jr ra nop END(octeon_cop2_save) LEAF(octeon_cop2_restore) /* Get CvmCtl register */ dmfc0 t0, $9, 7 /* CRC state */ RESTORE_COP2_REGISTER(COP2_CRC_IV) RESTORE_COP2_REGISTER(COP2_CRC_LENGTH) RESTORE_COP2_REGISTER(COP2_CRC_POLY) /* if CvmCtl[NODFA_CP2] -> save_nodfa */ bbit1 t0, 28, restore_nodfa nop /* LLM state */ RESTORE_COP2_REGISTER(COP2_LLM_DAT0) RESTORE_COP2_REGISTER(COP2_LLM_DAT1) restore_nodfa: /* crypto stuff is irrelevant if CvmCtl[NOCRYPTO] */ bbit1 t0, 26, restore_done nop RESTORE_COP2_REGISTER(COP2_3DES_IV) RESTORE_COP2_REGISTER(COP2_3DES_KEY0) RESTORE_COP2_REGISTER(COP2_3DES_KEY1) RESTORE_COP2_REGISTER(COP2_3DES_KEY2) RESTORE_COP2_REGISTER(COP2_3DES_RESULT) RESTORE_COP2_REGISTER(COP2_AES_INP0) RESTORE_COP2_REGISTER(COP2_AES_IV0) RESTORE_COP2_REGISTER(COP2_AES_IV1) RESTORE_COP2_REGISTER(COP2_AES_KEY0) RESTORE_COP2_REGISTER(COP2_AES_KEY1) RESTORE_COP2_REGISTER(COP2_AES_KEY2) RESTORE_COP2_REGISTER(COP2_AES_KEY3) RESTORE_COP2_REGISTER(COP2_AES_KEYLEN) RESTORE_COP2_REGISTER(COP2_AES_RESULT0) RESTORE_COP2_REGISTER(COP2_AES_RESULT1) dmfc0 t0, $15 li t1, 0x000d0000 /* Octeon Pass1 */ beq t0, t1, restore_pass1 nop RESTORE_COP2_REGISTER(COP2_HSH_DATW0) RESTORE_COP2_REGISTER(COP2_HSH_DATW1) RESTORE_COP2_REGISTER(COP2_HSH_DATW2) RESTORE_COP2_REGISTER(COP2_HSH_DATW3) RESTORE_COP2_REGISTER(COP2_HSH_DATW4) RESTORE_COP2_REGISTER(COP2_HSH_DATW5) RESTORE_COP2_REGISTER(COP2_HSH_DATW6) RESTORE_COP2_REGISTER(COP2_HSH_DATW7) RESTORE_COP2_REGISTER(COP2_HSH_DATW8) RESTORE_COP2_REGISTER(COP2_HSH_DATW9) RESTORE_COP2_REGISTER(COP2_HSH_DATW10) RESTORE_COP2_REGISTER(COP2_HSH_DATW11) RESTORE_COP2_REGISTER(COP2_HSH_DATW12) RESTORE_COP2_REGISTER(COP2_HSH_DATW13) RESTORE_COP2_REGISTER(COP2_HSH_DATW14) RESTORE_COP2_REGISTER(COP2_HSH_IVW0) RESTORE_COP2_REGISTER(COP2_HSH_IVW1) RESTORE_COP2_REGISTER(COP2_HSH_IVW2) RESTORE_COP2_REGISTER(COP2_HSH_IVW3) RESTORE_COP2_REGISTER(COP2_HSH_IVW4) RESTORE_COP2_REGISTER(COP2_HSH_IVW5) RESTORE_COP2_REGISTER(COP2_HSH_IVW6) RESTORE_COP2_REGISTER(COP2_HSH_IVW7) RESTORE_COP2_REGISTER(COP2_GFM_MULT0) RESTORE_COP2_REGISTER(COP2_GFM_MULT1) RESTORE_COP2_REGISTER(COP2_GFM_POLY) RESTORE_COP2_REGISTER(COP2_GFM_RESULT0) RESTORE_COP2_REGISTER(COP2_GFM_RESULT1) jr ra nop restore_pass1: RESTORE_COP2_REGISTER(COP2_HSH_DATW0_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW1_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW2_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW3_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW4_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW5_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW6_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_IVW0_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_IVW1_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_IVW2_PASS1) restore_done: jr ra nop END(octeon_cop2_restore) Index: head/sys/mips/ingenic/jz4780_mpboot.S =================================================================== --- head/sys/mips/ingenic/jz4780_mpboot.S (revision 331253) +++ head/sys/mips/ingenic/jz4780_mpboot.S (revision 331254) @@ -1,45 +1,45 @@ /*- * Copyright (c) 2015 Alexander Kabaev * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include -#include "assym.s" +#include "assym.inc" .text .set noat .set noreorder .section .text.mpentry_jz4780 .balign 0x10000 /* * JZ4870 has stricter alignment requirement for * CPU entry point. Enforce it in CPU-specific * file. */ GLOBAL(jz4780_mpentry) j mpentry nop Index: head/sys/mips/mips/exception.S =================================================================== --- head/sys/mips/mips/exception.S (revision 331253) +++ head/sys/mips/mips/exception.S (revision 331254) @@ -1,1280 +1,1280 @@ /* $OpenBSD: locore.S,v 1.18 1998/09/15 10:58:53 pefo Exp $ */ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Digital Equipment Corporation and Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Copyright (C) 1989 Digital Equipment Corporation. * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby granted, * provided that the above copyright notice appears in all copies. * Digital Equipment Corporation makes no representations about the * suitability of this software for any purpose. It is provided "as is" * without express or implied warranty. * * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/loMem.s, * v 1.1 89/07/11 17:55:04 nelson Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/machAsm.s, * v 9.2 90/01/29 18:00:39 shirriff Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/vm/ds3100.md/vmPmaxAsm.s, * v 1.1 89/07/10 14:27:41 nelson Exp SPRITE (DECWRL) * from: @(#)locore.s 8.5 (Berkeley) 1/4/94 * JNPR: exception.S,v 1.5 2007/01/08 04:58:37 katta * $FreeBSD$ */ /* * Contains code that is the first executed at boot time plus * assembly language support routines. */ #include "opt_ddb.h" #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" .set noreorder # Noreorder is default style! #ifdef KDTRACE_HOOKS .data .globl dtrace_invop_calltrap_addr .align 4 .type dtrace_invop_calltrap_addr, @object .size dtrace_invop_calltrap_addr, 8 dtrace_invop_calltrap_addr: .word 0 .word 0 .text #endif /* * Reasonable limit */ #define INTRCNT_COUNT 256 /* *---------------------------------------------------------------------------- * * MipsTLBMiss -- * * Vector code for the TLB-miss exception vector 0x80000000. * * This code is copied to the TLB exception vector address to * which the CPU jumps in response to an exception or a TLB miss. * NOTE: This code must be position independent!!! * * */ VECTOR(MipsTLBMiss, unknown) .set push .set noat j MipsDoTLBMiss MFC0 k0, MIPS_COP_0_BAD_VADDR # get the fault address .set pop VECTOR_END(MipsTLBMiss) /* *---------------------------------------------------------------------------- * * MipsDoTLBMiss -- * * This is the real TLB Miss Handler code. * 'segbase' points to the base of the segment table for user processes. * * Don't check for invalid pte's here. We load them as well and * let the processor trap to load the correct value after service. *---------------------------------------------------------------------------- */ .set push .set noat MipsDoTLBMiss: bltz k0, 1f #02: k0<0 -> 1f (kernel fault) PTR_SRL k0, k0, SEGSHIFT - PTRSHIFT #03: k0=seg offset (almost) GET_CPU_PCPU(k1) PTR_L k1, PC_SEGBASE(k1) beqz k1, 2f #05: make sure segbase is not null andi k0, k0, PDEPTRMASK #06: k0=seg offset PTR_ADDU k1, k0, k1 #07: k1=seg entry address PTR_L k1, 0(k1) #08: k1=seg entry MFC0 k0, MIPS_COP_0_BAD_VADDR #09: k0=bad address (again) beq k1, zero, 2f #0a: ==0 -- no page table #ifdef __mips_n64 PTR_SRL k0, PDRSHIFT - PTRSHIFT # k0=VPN andi k0, k0, PDEPTRMASK # k0=pde offset PTR_ADDU k1, k0, k1 # k1=pde entry address PTR_L k1, 0(k1) # k1=pde entry MFC0 k0, MIPS_COP_0_BAD_VADDR # k0=bad address (again) beq k1, zero, 2f # ==0 -- no page table #endif PTR_SRL k0, PAGE_SHIFT - PTESHIFT #0b: k0=VPN (aka va>>10) andi k0, k0, PTE2MASK #0c: k0=page tab offset PTR_ADDU k1, k1, k0 #0d: k1=pte address PTE_L k0, 0(k1) #0e: k0=lo0 pte PTE_L k1, PTESIZE(k1) #0f: k1=lo0 pte CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 #12: lo0 is loaded COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 #15: lo1 is loaded COP0_SYNC tlbwr #1a: write to tlb HAZARD_DELAY eret #1f: retUrn from exception 1: j MipsTLBMissException #20: kernel exception nop #21: branch delay slot 2: j SlowFault #22: no page table present nop #23: branch delay slot .set pop /* * This code is copied to the general exception vector address to * handle all execptions except RESET and TLBMiss. * NOTE: This code must be position independent!!! */ VECTOR(MipsException, unknown) /* * Find out what mode we came from and jump to the proper handler. * * Note: at turned off here because we cannot trash the at register * in this exception code. Only k0 and k1 may be modified before * we save registers. This is true of all functions called through * the pointer magic: Mips{User,Kern}Intr, Mips{User,Kern}GenException * and MipsTLBInvalidException */ .set noat mfc0 k0, MIPS_COP_0_STATUS # Get the status register mfc0 k1, MIPS_COP_0_CAUSE # Get the cause register value. and k0, k0, MIPS_SR_KSU_USER # test for user mode # sneaky but the bits are # with us........ sll k0, k0, 3 # shift user bit for cause index and k1, k1, MIPS_CR_EXC_CODE # Mask out the cause bits. or k1, k1, k0 # change index to user table #if defined(__mips_n64) PTR_SLL k1, k1, 1 # shift to get 8-byte offset #endif 1: PTR_LA k0, _C_LABEL(machExceptionTable) # get base of the jump table PTR_ADDU k0, k0, k1 # Get the address of the # function entry. Note that # the cause is already # shifted left by 2 bits so # we dont have to shift. PTR_L k0, 0(k0) # Get the function address nop j k0 # Jump to the function. nop .set at VECTOR_END(MipsException) /* * We couldn't find a TLB entry. * Find out what mode we came from and call the appropriate handler. */ SlowFault: .set noat mfc0 k0, MIPS_COP_0_STATUS nop and k0, k0, MIPS_SR_KSU_USER bne k0, zero, _C_LABEL(MipsUserGenException) nop .set at /* * Fall though ... */ /*---------------------------------------------------------------------------- * * MipsKernGenException -- * * Handle an exception from kernel mode. * * Results: * None. * * Side effects: * None. * *---------------------------------------------------------------------------- */ #define SAVE_REG(reg, offs, base) \ REG_S reg, CALLFRAME_SIZ + (SZREG * offs) (base) #if defined(CPU_CNMIPS) #define CLEAR_STATUS \ mfc0 a0, MIPS_COP_0_STATUS ;\ li a2, (MIPS_SR_KX | MIPS_SR_SX | MIPS_SR_UX) ; \ or a0, a0, a2 ; \ li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | MIPS_SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, MIPS_COP_0_STATUS ; \ ITLBNOPFIX #elif defined(CPU_RMI) || defined(CPU_NLM) #define CLEAR_STATUS \ mfc0 a0, MIPS_COP_0_STATUS ;\ li a2, (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_COP_2_BIT) ; \ or a0, a0, a2 ; \ li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | MIPS_SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, MIPS_COP_0_STATUS ; \ ITLBNOPFIX #else #define CLEAR_STATUS \ mfc0 a0, MIPS_COP_0_STATUS ;\ li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | MIPS_SR_KSU_USER) ; \ and a0, a0, a2 ; \ mtc0 a0, MIPS_COP_0_STATUS ; \ ITLBNOPFIX #endif /* * Save CPU and CP0 register state. * * This is straightforward except for saving the exception program * counter. The ddb backtrace code looks for the first instruction * matching the form "sw ra, (off)sp" to figure out the address of the * calling function. So we must make sure that we save the exception * PC by staging it through 'ra' as opposed to any other register. */ #define SAVE_CPU \ SAVE_REG(AT, AST, sp) ;\ .set at ; \ SAVE_REG(v0, V0, sp) ;\ SAVE_REG(v1, V1, sp) ;\ SAVE_REG(a0, A0, sp) ;\ SAVE_REG(a1, A1, sp) ;\ SAVE_REG(a2, A2, sp) ;\ SAVE_REG(a3, A3, sp) ;\ SAVE_REG(t0, T0, sp) ;\ SAVE_REG(t1, T1, sp) ;\ SAVE_REG(t2, T2, sp) ;\ SAVE_REG(t3, T3, sp) ;\ SAVE_REG(ta0, TA0, sp) ;\ SAVE_REG(ta1, TA1, sp) ;\ SAVE_REG(ta2, TA2, sp) ;\ SAVE_REG(ta3, TA3, sp) ;\ SAVE_REG(t8, T8, sp) ;\ SAVE_REG(t9, T9, sp) ;\ SAVE_REG(gp, GP, sp) ;\ SAVE_REG(s0, S0, sp) ;\ SAVE_REG(s1, S1, sp) ;\ SAVE_REG(s2, S2, sp) ;\ SAVE_REG(s3, S3, sp) ;\ SAVE_REG(s4, S4, sp) ;\ SAVE_REG(s5, S5, sp) ;\ SAVE_REG(s6, S6, sp) ;\ SAVE_REG(s7, S7, sp) ;\ SAVE_REG(s8, S8, sp) ;\ mflo v0 ;\ mfhi v1 ;\ mfc0 a0, MIPS_COP_0_STATUS ;\ mfc0 a1, MIPS_COP_0_CAUSE ;\ MFC0 a2, MIPS_COP_0_BAD_VADDR;\ MFC0 a3, MIPS_COP_0_EXC_PC ;\ SAVE_REG(v0, MULLO, sp) ;\ SAVE_REG(v1, MULHI, sp) ;\ SAVE_REG(a0, SR, sp) ;\ SAVE_REG(a1, CAUSE, sp) ;\ SAVE_REG(a2, BADVADDR, sp) ;\ move t0, ra ;\ move ra, a3 ;\ SAVE_REG(ra, PC, sp) ;\ move ra, t0 ;\ SAVE_REG(ra, RA, sp) ;\ PTR_ADDU v0, sp, KERN_EXC_FRAME_SIZE ;\ SAVE_REG(v0, SP, sp) ;\ CLEAR_STATUS ;\ PTR_ADDU a0, sp, CALLFRAME_SIZ ;\ ITLBNOPFIX #define RESTORE_REG(reg, offs, base) \ REG_L reg, CALLFRAME_SIZ + (SZREG * offs) (base) #define RESTORE_CPU \ CLEAR_STATUS ;\ RESTORE_REG(k0, SR, sp) ;\ RESTORE_REG(t0, MULLO, sp) ;\ RESTORE_REG(t1, MULHI, sp) ;\ mtlo t0 ;\ mthi t1 ;\ MTC0 v0, MIPS_COP_0_EXC_PC ;\ .set noat ;\ RESTORE_REG(AT, AST, sp) ;\ RESTORE_REG(v0, V0, sp) ;\ RESTORE_REG(v1, V1, sp) ;\ RESTORE_REG(a0, A0, sp) ;\ RESTORE_REG(a1, A1, sp) ;\ RESTORE_REG(a2, A2, sp) ;\ RESTORE_REG(a3, A3, sp) ;\ RESTORE_REG(t0, T0, sp) ;\ RESTORE_REG(t1, T1, sp) ;\ RESTORE_REG(t2, T2, sp) ;\ RESTORE_REG(t3, T3, sp) ;\ RESTORE_REG(ta0, TA0, sp) ;\ RESTORE_REG(ta1, TA1, sp) ;\ RESTORE_REG(ta2, TA2, sp) ;\ RESTORE_REG(ta3, TA3, sp) ;\ RESTORE_REG(t8, T8, sp) ;\ RESTORE_REG(t9, T9, sp) ;\ RESTORE_REG(s0, S0, sp) ;\ RESTORE_REG(s1, S1, sp) ;\ RESTORE_REG(s2, S2, sp) ;\ RESTORE_REG(s3, S3, sp) ;\ RESTORE_REG(s4, S4, sp) ;\ RESTORE_REG(s5, S5, sp) ;\ RESTORE_REG(s6, S6, sp) ;\ RESTORE_REG(s7, S7, sp) ;\ RESTORE_REG(s8, S8, sp) ;\ RESTORE_REG(gp, GP, sp) ;\ RESTORE_REG(ra, RA, sp) ;\ PTR_ADDU sp, sp, KERN_EXC_FRAME_SIZE;\ mtc0 k0, MIPS_COP_0_STATUS /* * The kernel exception stack contains 18 saved general registers, * the status register and the multiply lo and high registers. * In addition, we set this up for linkage conventions. */ #define KERN_REG_SIZE (NUMSAVEREGS * SZREG) #define KERN_EXC_FRAME_SIZE (CALLFRAME_SIZ + KERN_REG_SIZE + 16) NESTED_NOPROFILE(MipsKernGenException, KERN_EXC_FRAME_SIZE, ra) .set noat PTR_SUBU sp, sp, KERN_EXC_FRAME_SIZE .mask 0x80000000, (CALLFRAME_RA - KERN_EXC_FRAME_SIZE) /* * Save CPU state, building 'frame'. */ SAVE_CPU /* * Call the exception handler. a0 points at the saved frame. */ PTR_LA gp, _C_LABEL(_gp) PTR_LA k0, _C_LABEL(trap) jalr k0 REG_S a3, CALLFRAME_RA + KERN_REG_SIZE(sp) # for debugging /* * Update interrupt and CPU mask in saved status register * Some of interrupts could be disabled by * intr filters if interrupts are enabled later * in trap handler */ mfc0 a0, MIPS_COP_0_STATUS and a0, a0, (MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) RESTORE_REG(a1, SR, sp) and a1, a1, ~(MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) or a1, a1, a0 SAVE_REG(a1, SR, sp) RESTORE_CPU # v0 contains the return address. sync eret .set at END(MipsKernGenException) /*---------------------------------------------------------------------------- * * MipsUserGenException -- * * Handle an exception from user mode. * * Results: * None. * * Side effects: * None. * *---------------------------------------------------------------------------- */ NESTED_NOPROFILE(MipsUserGenException, CALLFRAME_SIZ, ra) .set noat .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) /* * Save all of the registers except for the kernel temporaries in u.u_pcb. */ GET_CPU_PCPU(k1) PTR_L k1, PC_CURPCB(k1) SAVE_U_PCB_REG(AT, AST, k1) .set at SAVE_U_PCB_REG(v0, V0, k1) SAVE_U_PCB_REG(v1, V1, k1) SAVE_U_PCB_REG(a0, A0, k1) mflo v0 SAVE_U_PCB_REG(a1, A1, k1) SAVE_U_PCB_REG(a2, A2, k1) SAVE_U_PCB_REG(a3, A3, k1) SAVE_U_PCB_REG(t0, T0, k1) mfhi v1 SAVE_U_PCB_REG(t1, T1, k1) SAVE_U_PCB_REG(t2, T2, k1) SAVE_U_PCB_REG(t3, T3, k1) SAVE_U_PCB_REG(ta0, TA0, k1) mfc0 a0, MIPS_COP_0_STATUS # First arg is the status reg. SAVE_U_PCB_REG(ta1, TA1, k1) SAVE_U_PCB_REG(ta2, TA2, k1) SAVE_U_PCB_REG(ta3, TA3, k1) SAVE_U_PCB_REG(s0, S0, k1) mfc0 a1, MIPS_COP_0_CAUSE # Second arg is the cause reg. SAVE_U_PCB_REG(s1, S1, k1) SAVE_U_PCB_REG(s2, S2, k1) SAVE_U_PCB_REG(s3, S3, k1) SAVE_U_PCB_REG(s4, S4, k1) MFC0 a2, MIPS_COP_0_BAD_VADDR # Third arg is the fault addr SAVE_U_PCB_REG(s5, S5, k1) SAVE_U_PCB_REG(s6, S6, k1) SAVE_U_PCB_REG(s7, S7, k1) SAVE_U_PCB_REG(t8, T8, k1) MFC0 a3, MIPS_COP_0_EXC_PC # Fourth arg is the pc. SAVE_U_PCB_REG(t9, T9, k1) SAVE_U_PCB_REG(gp, GP, k1) SAVE_U_PCB_REG(sp, SP, k1) SAVE_U_PCB_REG(s8, S8, k1) PTR_SUBU sp, k1, CALLFRAME_SIZ # switch to kernel SP SAVE_U_PCB_REG(ra, RA, k1) SAVE_U_PCB_REG(v0, MULLO, k1) SAVE_U_PCB_REG(v1, MULHI, k1) SAVE_U_PCB_REG(a0, SR, k1) SAVE_U_PCB_REG(a1, CAUSE, k1) SAVE_U_PCB_REG(a2, BADVADDR, k1) SAVE_U_PCB_REG(a3, PC, k1) REG_S a3, CALLFRAME_RA(sp) # for debugging PTR_LA gp, _C_LABEL(_gp) # switch to kernel GP # Turn off fpu and enter kernel mode and t0, a0, ~(MIPS_SR_COP_1_BIT | MIPS_SR_EXL | MIPS_SR_KSU_MASK | MIPS_SR_INT_IE) #if defined(CPU_CNMIPS) and t0, t0, ~(MIPS_SR_COP_2_BIT) or t0, t0, (MIPS_SR_KX | MIPS_SR_SX | MIPS_SR_UX | MIPS_SR_PX) #elif defined(CPU_RMI) || defined(CPU_NLM) or t0, t0, (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_COP_2_BIT) #endif mtc0 t0, MIPS_COP_0_STATUS PTR_ADDU a0, k1, U_PCB_REGS ITLBNOPFIX /* * Call the exception handler. */ PTR_LA k0, _C_LABEL(trap) jalr k0 nop /* * Restore user registers and return. * First disable interrupts and set exeption level. */ DO_AST CLEAR_STATUS /* * The use of k1 for storing the PCB pointer must be done only * after interrupts are disabled. Otherwise it will get overwritten * by the interrupt code. */ GET_CPU_PCPU(k1) PTR_L k1, PC_CURPCB(k1) /* * Update interrupt mask in saved status register * Some of interrupts could be enabled by ithread * scheduled by ast() */ mfc0 a0, MIPS_COP_0_STATUS and a0, a0, MIPS_SR_INT_MASK RESTORE_U_PCB_REG(a1, SR, k1) and a1, a1, ~MIPS_SR_INT_MASK or a1, a1, a0 SAVE_U_PCB_REG(a1, SR, k1) RESTORE_U_PCB_REG(t0, MULLO, k1) RESTORE_U_PCB_REG(t1, MULHI, k1) mtlo t0 mthi t1 RESTORE_U_PCB_REG(a0, PC, k1) RESTORE_U_PCB_REG(v0, V0, k1) MTC0 a0, MIPS_COP_0_EXC_PC # set return address RESTORE_U_PCB_REG(v1, V1, k1) RESTORE_U_PCB_REG(a0, A0, k1) RESTORE_U_PCB_REG(a1, A1, k1) RESTORE_U_PCB_REG(a2, A2, k1) RESTORE_U_PCB_REG(a3, A3, k1) RESTORE_U_PCB_REG(t0, T0, k1) RESTORE_U_PCB_REG(t1, T1, k1) RESTORE_U_PCB_REG(t2, T2, k1) RESTORE_U_PCB_REG(t3, T3, k1) RESTORE_U_PCB_REG(ta0, TA0, k1) RESTORE_U_PCB_REG(ta1, TA1, k1) RESTORE_U_PCB_REG(ta2, TA2, k1) RESTORE_U_PCB_REG(ta3, TA3, k1) RESTORE_U_PCB_REG(s0, S0, k1) RESTORE_U_PCB_REG(s1, S1, k1) RESTORE_U_PCB_REG(s2, S2, k1) RESTORE_U_PCB_REG(s3, S3, k1) RESTORE_U_PCB_REG(s4, S4, k1) RESTORE_U_PCB_REG(s5, S5, k1) RESTORE_U_PCB_REG(s6, S6, k1) RESTORE_U_PCB_REG(s7, S7, k1) RESTORE_U_PCB_REG(t8, T8, k1) RESTORE_U_PCB_REG(t9, T9, k1) RESTORE_U_PCB_REG(gp, GP, k1) RESTORE_U_PCB_REG(sp, SP, k1) RESTORE_U_PCB_REG(k0, SR, k1) RESTORE_U_PCB_REG(s8, S8, k1) RESTORE_U_PCB_REG(ra, RA, k1) .set noat RESTORE_U_PCB_REG(AT, AST, k1) mtc0 k0, MIPS_COP_0_STATUS # still exception level ITLBNOPFIX sync eret .set at END(MipsUserGenException) .set push .set noat NESTED(mips_wait, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) REG_S ra, CALLFRAME_RA(sp) # save RA mfc0 t0, MIPS_COP_0_STATUS xori t1, t0, MIPS_SR_INT_IE mtc0 t1, MIPS_COP_0_STATUS COP0_SYNC jal sched_runnable nop REG_L ra, CALLFRAME_RA(sp) mfc0 t0, MIPS_COP_0_STATUS ori t1, t0, MIPS_SR_INT_IE .align 4 GLOBAL(MipsWaitStart) # this is 16 byte aligned mtc0 t1, MIPS_COP_0_STATUS bnez v0, MipsWaitEnd nop #if defined(CPU_XBURST) && defined(SMP) nop #else wait #endif GLOBAL(MipsWaitEnd) # MipsWaitStart + 16 jr ra PTR_ADDU sp, sp, CALLFRAME_SIZ END(mips_wait) .set pop /*---------------------------------------------------------------------------- * * MipsKernIntr -- * * Handle an interrupt from kernel mode. * Interrupts use the standard kernel stack. * switch_exit sets up a kernel stack after exit so interrupts won't fail. * * Results: * None. * * Side effects: * None. * *---------------------------------------------------------------------------- */ NESTED_NOPROFILE(MipsKernIntr, KERN_EXC_FRAME_SIZE, ra) .set noat PTR_SUBU sp, sp, KERN_EXC_FRAME_SIZE .mask 0x80000000, (CALLFRAME_RA - KERN_EXC_FRAME_SIZE) /* * Check for getting interrupts just before wait */ MFC0 k0, MIPS_COP_0_EXC_PC ori k0, 0xf xori k0, 0xf # 16 byte align PTR_LA k1, MipsWaitStart bne k0, k1, 1f nop PTR_ADDU k1, 16 # skip over wait MTC0 k1, MIPS_COP_0_EXC_PC 1: /* * Save CPU state, building 'frame'. */ SAVE_CPU /* * Call the interrupt handler. a0 points at the saved frame. */ PTR_LA gp, _C_LABEL(_gp) #ifdef INTRNG PTR_LA k0, _C_LABEL(intr_irq_handler) #else PTR_LA k0, _C_LABEL(cpu_intr) #endif jalr k0 REG_S a3, CALLFRAME_RA + KERN_REG_SIZE(sp) # for debugging /* * Update interrupt and CPU mask in saved status register * Some of interrupts could be disabled by * intr filters if interrupts are enabled later * in trap handler */ mfc0 a0, MIPS_COP_0_STATUS and a0, a0, (MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) RESTORE_REG(a1, SR, sp) and a1, a1, ~(MIPS_SR_INT_MASK|MIPS_SR_COP_USABILITY) or a1, a1, a0 SAVE_REG(a1, SR, sp) REG_L v0, CALLFRAME_RA + KERN_REG_SIZE(sp) RESTORE_CPU # v0 contains the return address. sync eret .set at END(MipsKernIntr) /*---------------------------------------------------------------------------- * * MipsUserIntr -- * * Handle an interrupt from user mode. * Note: we save minimal state in the u.u_pcb struct and use the standard * kernel stack since there has to be a u page if we came from user mode. * If there is a pending software interrupt, then save the remaining state * and call softintr(). This is all because if we call switch() inside * interrupt(), not all the user registers have been saved in u.u_pcb. * * Results: * None. * * Side effects: * None. * *---------------------------------------------------------------------------- */ NESTED_NOPROFILE(MipsUserIntr, CALLFRAME_SIZ, ra) .set noat .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) /* * Save the relevant user registers into the u.u_pcb struct. * We don't need to save s0 - s8 because the compiler does it for us. */ GET_CPU_PCPU(k1) PTR_L k1, PC_CURPCB(k1) SAVE_U_PCB_REG(AT, AST, k1) .set at SAVE_U_PCB_REG(v0, V0, k1) SAVE_U_PCB_REG(v1, V1, k1) SAVE_U_PCB_REG(a0, A0, k1) SAVE_U_PCB_REG(a1, A1, k1) SAVE_U_PCB_REG(a2, A2, k1) SAVE_U_PCB_REG(a3, A3, k1) SAVE_U_PCB_REG(t0, T0, k1) SAVE_U_PCB_REG(t1, T1, k1) SAVE_U_PCB_REG(t2, T2, k1) SAVE_U_PCB_REG(t3, T3, k1) SAVE_U_PCB_REG(ta0, TA0, k1) SAVE_U_PCB_REG(ta1, TA1, k1) SAVE_U_PCB_REG(ta2, TA2, k1) SAVE_U_PCB_REG(ta3, TA3, k1) SAVE_U_PCB_REG(t8, T8, k1) SAVE_U_PCB_REG(t9, T9, k1) SAVE_U_PCB_REG(gp, GP, k1) SAVE_U_PCB_REG(sp, SP, k1) SAVE_U_PCB_REG(ra, RA, k1) /* * save remaining user state in u.u_pcb. */ SAVE_U_PCB_REG(s0, S0, k1) SAVE_U_PCB_REG(s1, S1, k1) SAVE_U_PCB_REG(s2, S2, k1) SAVE_U_PCB_REG(s3, S3, k1) SAVE_U_PCB_REG(s4, S4, k1) SAVE_U_PCB_REG(s5, S5, k1) SAVE_U_PCB_REG(s6, S6, k1) SAVE_U_PCB_REG(s7, S7, k1) SAVE_U_PCB_REG(s8, S8, k1) mflo v0 # get lo/hi late to avoid stall mfhi v1 mfc0 a0, MIPS_COP_0_STATUS mfc0 a1, MIPS_COP_0_CAUSE MFC0 a3, MIPS_COP_0_EXC_PC SAVE_U_PCB_REG(v0, MULLO, k1) SAVE_U_PCB_REG(v1, MULHI, k1) SAVE_U_PCB_REG(a0, SR, k1) SAVE_U_PCB_REG(a1, CAUSE, k1) SAVE_U_PCB_REG(a3, PC, k1) # PC in a3, note used later! PTR_SUBU sp, k1, CALLFRAME_SIZ # switch to kernel SP PTR_LA gp, _C_LABEL(_gp) # switch to kernel GP # Turn off fpu, disable interrupts, set kernel mode kernel mode, clear exception level. and t0, a0, ~(MIPS_SR_COP_1_BIT | MIPS_SR_EXL | MIPS_SR_INT_IE | MIPS_SR_KSU_MASK) #ifdef CPU_CNMIPS and t0, t0, ~(MIPS_SR_COP_2_BIT) or t0, t0, (MIPS_SR_KX | MIPS_SR_SX | MIPS_SR_UX | MIPS_SR_PX) #elif defined(CPU_RMI) || defined(CPU_NLM) or t0, t0, (MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_COP_2_BIT) #endif mtc0 t0, MIPS_COP_0_STATUS ITLBNOPFIX PTR_ADDU a0, k1, U_PCB_REGS /* * Call the interrupt handler. */ #ifdef INTRNG PTR_LA k0, _C_LABEL(intr_irq_handler) #else PTR_LA k0, _C_LABEL(cpu_intr) #endif jalr k0 REG_S a3, CALLFRAME_RA(sp) # for debugging /* * Enable interrupts before doing ast(). * * On SMP kernels the AST processing might trigger IPI to other processors. * If that processor is also doing AST processing with interrupts disabled * then we may deadlock. */ mfc0 a0, MIPS_COP_0_STATUS or a0, a0, MIPS_SR_INT_IE mtc0 a0, MIPS_COP_0_STATUS ITLBNOPFIX /* * DO_AST enabled interrupts */ DO_AST /* * Restore user registers and return. */ CLEAR_STATUS GET_CPU_PCPU(k1) PTR_L k1, PC_CURPCB(k1) /* * Update interrupt mask in saved status register * Some of interrupts could be disabled by * intr filters */ mfc0 a0, MIPS_COP_0_STATUS and a0, a0, MIPS_SR_INT_MASK RESTORE_U_PCB_REG(a1, SR, k1) and a1, a1, ~MIPS_SR_INT_MASK or a1, a1, a0 SAVE_U_PCB_REG(a1, SR, k1) RESTORE_U_PCB_REG(s0, S0, k1) RESTORE_U_PCB_REG(s1, S1, k1) RESTORE_U_PCB_REG(s2, S2, k1) RESTORE_U_PCB_REG(s3, S3, k1) RESTORE_U_PCB_REG(s4, S4, k1) RESTORE_U_PCB_REG(s5, S5, k1) RESTORE_U_PCB_REG(s6, S6, k1) RESTORE_U_PCB_REG(s7, S7, k1) RESTORE_U_PCB_REG(s8, S8, k1) RESTORE_U_PCB_REG(t0, MULLO, k1) RESTORE_U_PCB_REG(t1, MULHI, k1) RESTORE_U_PCB_REG(t2, PC, k1) mtlo t0 mthi t1 MTC0 t2, MIPS_COP_0_EXC_PC # set return address RESTORE_U_PCB_REG(v0, V0, k1) RESTORE_U_PCB_REG(v1, V1, k1) RESTORE_U_PCB_REG(a0, A0, k1) RESTORE_U_PCB_REG(a1, A1, k1) RESTORE_U_PCB_REG(a2, A2, k1) RESTORE_U_PCB_REG(a3, A3, k1) RESTORE_U_PCB_REG(t0, T0, k1) RESTORE_U_PCB_REG(t1, T1, k1) RESTORE_U_PCB_REG(t2, T2, k1) RESTORE_U_PCB_REG(t3, T3, k1) RESTORE_U_PCB_REG(ta0, TA0, k1) RESTORE_U_PCB_REG(ta1, TA1, k1) RESTORE_U_PCB_REG(ta2, TA2, k1) RESTORE_U_PCB_REG(ta3, TA3, k1) RESTORE_U_PCB_REG(t8, T8, k1) RESTORE_U_PCB_REG(t9, T9, k1) RESTORE_U_PCB_REG(gp, GP, k1) RESTORE_U_PCB_REG(k0, SR, k1) RESTORE_U_PCB_REG(sp, SP, k1) RESTORE_U_PCB_REG(ra, RA, k1) .set noat RESTORE_U_PCB_REG(AT, AST, k1) mtc0 k0, MIPS_COP_0_STATUS # SR with EXL set. ITLBNOPFIX sync eret .set at END(MipsUserIntr) LEAF_NOPROFILE(MipsTLBInvalidException) .set push .set noat .set noreorder MFC0 k0, MIPS_COP_0_BAD_VADDR PTR_LI k1, VM_MAXUSER_ADDRESS sltu k1, k0, k1 bnez k1, 1f nop /* Kernel address. */ lui k1, %hi(kernel_segmap) # k1=hi of segbase b 2f PTR_L k1, %lo(kernel_segmap)(k1) # k1=segment tab base 1: /* User address. */ GET_CPU_PCPU(k1) PTR_L k1, PC_SEGBASE(k1) 2: /* Validate page directory pointer. */ beqz k1, 3f nop PTR_SRL k0, SEGSHIFT - PTRSHIFT # k0=seg offset (almost) beq k1, zero, MipsKernGenException # ==0 -- no seg tab andi k0, k0, PDEPTRMASK #06: k0=seg offset PTR_ADDU k1, k0, k1 # k1=seg entry address PTR_L k1, 0(k1) # k1=seg entry /* Validate page table pointer. */ beqz k1, 3f nop #ifdef __mips_n64 MFC0 k0, MIPS_COP_0_BAD_VADDR PTR_SRL k0, PDRSHIFT - PTRSHIFT # k0=pde offset (almost) beq k1, zero, MipsKernGenException # ==0 -- no pde tab andi k0, k0, PDEPTRMASK # k0=pde offset PTR_ADDU k1, k0, k1 # k1=pde entry address PTR_L k1, 0(k1) # k1=pde entry /* Validate pde table pointer. */ beqz k1, 3f nop #endif MFC0 k0, MIPS_COP_0_BAD_VADDR # k0=bad address (again) PTR_SRL k0, PAGE_SHIFT - PTESHIFT # k0=VPN andi k0, k0, PTEMASK # k0=page tab offset PTR_ADDU k1, k1, k0 # k1=pte address PTE_L k0, 0(k1) # k0=this PTE /* Validate page table entry. */ andi k0, PTE_V beqz k0, 3f nop /* Check whether this is an even or odd entry. */ andi k0, k1, PTESIZE bnez k0, odd_page nop PTE_L k0, 0(k1) PTE_L k1, PTESIZE(k1) CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 COP0_SYNC b tlb_insert_entry nop odd_page: PTE_L k0, -PTESIZE(k1) PTE_L k1, 0(k1) CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 COP0_SYNC tlb_insert_entry: tlbp HAZARD_DELAY mfc0 k0, MIPS_COP_0_TLB_INDEX bltz k0, tlb_insert_random nop tlbwi eret ssnop tlb_insert_random: tlbwr eret ssnop 3: /* * Branch to the comprehensive exception processing. */ mfc0 k1, MIPS_COP_0_STATUS andi k1, k1, MIPS_SR_KSU_USER bnez k1, _C_LABEL(MipsUserGenException) nop /* * Check for kernel stack overflow. */ GET_CPU_PCPU(k1) PTR_L k0, PC_CURTHREAD(k1) PTR_L k0, TD_KSTACK(k0) sltu k0, k0, sp bnez k0, _C_LABEL(MipsKernGenException) nop /* * Kernel stack overflow. * * Move to a valid stack before we call panic. We use the boot stack * for this purpose. */ GET_CPU_PCPU(k1) lw k1, PC_CPUID(k1) sll k1, k1, PAGE_SHIFT + 1 PTR_LA k0, _C_LABEL(pcpu_space) PTR_ADDU k0, PAGE_SIZE * 2 PTR_ADDU k0, k0, k1 /* * Stash the original value of 'sp' so we can update trapframe later. * We assume that SAVE_CPU does not trash 'k1'. */ move k1, sp move sp, k0 PTR_SUBU sp, sp, KERN_EXC_FRAME_SIZE move k0, ra move ra, zero REG_S ra, CALLFRAME_RA(sp) /* stop the ddb backtrace right here */ REG_S zero, CALLFRAME_SP(sp) move ra, k0 SAVE_CPU /* * Now restore the value of 'sp' at the time of the tlb exception in * the trapframe. */ SAVE_REG(k1, SP, sp) /* * Squelch any more overflow checks by setting the stack base to 0. */ GET_CPU_PCPU(k1) PTR_L k0, PC_CURTHREAD(k1) PTR_S zero, TD_KSTACK(k0) move a1, a0 PANIC("kernel stack overflow - trapframe at %p") /* * This nop is necessary so that the 'ra' remains within the bounds * of this handler. Otherwise the ddb backtrace code will think that * the panic() was called from MipsTLBMissException. */ .globl MipsKStackOverflow MipsKStackOverflow: nop .set pop END(MipsTLBInvalidException) /*---------------------------------------------------------------------------- * * MipsTLBMissException -- * * Handle a TLB miss exception from kernel mode in kernel space. * The BaddVAddr, Context, and EntryHi registers contain the failed * virtual address. * * Results: * None. * * Side effects: * None. * *---------------------------------------------------------------------------- */ LEAF_NOPROFILE(MipsTLBMissException) .set noat MFC0 k0, MIPS_COP_0_BAD_VADDR # k0=bad address PTR_LI k1, VM_MAX_KERNEL_ADDRESS # check fault address against sltu k1, k1, k0 # upper bound of kernel_segmap bnez k1, MipsKernGenException # out of bound lui k1, %hi(kernel_segmap) # k1=hi of segbase PTR_SRL k0, SEGSHIFT - PTRSHIFT # k0=seg offset (almost) PTR_L k1, %lo(kernel_segmap)(k1) # k1=segment tab base beq k1, zero, MipsKernGenException # ==0 -- no seg tab andi k0, k0, PDEPTRMASK #06: k0=seg offset PTR_ADDU k1, k0, k1 # k1=seg entry address PTR_L k1, 0(k1) # k1=seg entry MFC0 k0, MIPS_COP_0_BAD_VADDR # k0=bad address (again) beq k1, zero, MipsKernGenException # ==0 -- no page table #ifdef __mips_n64 PTR_SRL k0, PDRSHIFT - PTRSHIFT # k0=VPN andi k0, k0, PDEPTRMASK # k0=pde offset PTR_ADDU k1, k0, k1 # k1=pde entry address PTR_L k1, 0(k1) # k1=pde entry MFC0 k0, MIPS_COP_0_BAD_VADDR # k0=bad address (again) beq k1, zero, MipsKernGenException # ==0 -- no page table #endif PTR_SRL k0, PAGE_SHIFT - PTESHIFT # k0=VPN andi k0, k0, PTE2MASK # k0=page tab offset PTR_ADDU k1, k1, k0 # k1=pte address PTE_L k0, 0(k1) # k0=lo0 pte PTE_L k1, PTESIZE(k1) # k1=lo1 pte CLEAR_PTE_SWBITS(k0) PTE_MTC0 k0, MIPS_COP_0_TLB_LO0 # lo0 is loaded COP0_SYNC CLEAR_PTE_SWBITS(k1) PTE_MTC0 k1, MIPS_COP_0_TLB_LO1 # lo1 is loaded COP0_SYNC tlbwr # write to tlb HAZARD_DELAY eret # return from exception .set at END(MipsTLBMissException) /*---------------------------------------------------------------------------- * * MipsFPTrap -- * * Handle a floating point Trap. * * MipsFPTrap(statusReg, causeReg, pc) * unsigned statusReg; * unsigned causeReg; * unsigned pc; * * Results: * None. * * Side effects: * None. * *---------------------------------------------------------------------------- */ NESTED(MipsFPTrap, CALLFRAME_SIZ, ra) .set push .set hardfloat PTR_SUBU sp, sp, CALLFRAME_SIZ mfc0 t0, MIPS_COP_0_STATUS HAZARD_DELAY REG_S ra, CALLFRAME_RA(sp) .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) #if defined(__mips_n32) || defined(__mips_n64) or t1, t0, MIPS_SR_COP_1_BIT | MIPS_SR_FR #else or t1, t0, MIPS_SR_COP_1_BIT #endif mtc0 t1, MIPS_COP_0_STATUS HAZARD_DELAY ITLBNOPFIX cfc1 t1, MIPS_FPU_CSR # stall til FP done cfc1 t1, MIPS_FPU_CSR # now get status nop sll t2, t1, (31 - 17) # unimplemented operation? bgez t2, 3f # no, normal trap nop /* * We got an unimplemented operation trap so * fetch the instruction, compute the next PC and emulate the instruction. */ bgez a1, 1f # Check the branch delay bit. nop /* * The instruction is in the branch delay slot so the branch will have to * be emulated to get the resulting PC. */ PTR_S a2, CALLFRAME_SIZ + 8(sp) GET_CPU_PCPU(a0) #mips64 unsafe? PTR_L a0, PC_CURPCB(a0) PTR_ADDU a0, a0, U_PCB_REGS # first arg is ptr to CPU registers move a1, a2 # second arg is instruction PC move a2, t1 # third arg is floating point CSR PTR_LA t3, _C_LABEL(MipsEmulateBranch) # compute PC after branch jalr t3 # compute PC after branch move a3, zero # fourth arg is FALSE /* * Now load the floating-point instruction in the branch delay slot * to be emulated. */ PTR_L a2, CALLFRAME_SIZ + 8(sp) # restore EXC pc b 2f lw a0, 4(a2) # a0 = coproc instruction /* * This is not in the branch delay slot so calculate the resulting * PC (epc + 4) into v0 and continue to MipsEmulateFP(). */ 1: lw a0, 0(a2) # a0 = coproc instruction #xxx mips64 unsafe? PTR_ADDU v0, a2, 4 # v0 = next pc 2: GET_CPU_PCPU(t2) PTR_L t2, PC_CURPCB(t2) SAVE_U_PCB_REG(v0, PC, t2) # save new pc /* * Check to see if the instruction to be emulated is a floating-point * instruction. */ srl a3, a0, MIPS_OPCODE_SHIFT beq a3, MIPS_OPCODE_C1, 4f # this should never fail nop /* * Send a floating point exception signal to the current process. */ 3: GET_CPU_PCPU(a0) PTR_L a0, PC_CURTHREAD(a0) # get current thread cfc1 a2, MIPS_FPU_CSR # code = FP execptions ctc1 zero, MIPS_FPU_CSR # Clear exceptions PTR_LA t3, _C_LABEL(trapsignal) jalr t3 li a1, SIGFPE b FPReturn nop /* * Finally, we can call MipsEmulateFP() where a0 is the instruction to emulate. */ 4: PTR_LA t3, _C_LABEL(MipsEmulateFP) jalr t3 nop /* * Turn off the floating point coprocessor and return. */ FPReturn: mfc0 t0, MIPS_COP_0_STATUS PTR_L ra, CALLFRAME_RA(sp) and t0, t0, ~MIPS_SR_COP_1_BIT mtc0 t0, MIPS_COP_0_STATUS ITLBNOPFIX j ra PTR_ADDU sp, sp, CALLFRAME_SIZ .set pop END(MipsFPTrap) #ifndef INTRNG /* * Interrupt counters for vmstat. */ .data .globl intrcnt .globl sintrcnt .globl intrnames .globl sintrnames intrnames: .space INTRCNT_COUNT * (MAXCOMLEN + 1) * 2 sintrnames: #ifdef __mips_n64 .quad INTRCNT_COUNT * (MAXCOMLEN + 1) * 2 #else .int INTRCNT_COUNT * (MAXCOMLEN + 1) * 2 #endif .align (_MIPS_SZLONG / 8) intrcnt: .space INTRCNT_COUNT * (_MIPS_SZLONG / 8) * 2 sintrcnt: #ifdef __mips_n64 .quad INTRCNT_COUNT * (_MIPS_SZLONG / 8) * 2 #else .int INTRCNT_COUNT * (_MIPS_SZLONG / 8) * 2 #endif #endif /* INTRNG */ /* * Vector to real handler in KSEG1. */ .text VECTOR(MipsCache, unknown) PTR_LA k0, _C_LABEL(MipsCacheException) li k1, MIPS_KSEG0_PHYS_MASK and k0, k1 PTR_LI k1, MIPS_KSEG1_START or k0, k1 j k0 nop VECTOR_END(MipsCache) .set at /* * Panic on cache errors. A lot more could be done to recover * from some types of errors but it is tricky. */ NESTED_NOPROFILE(MipsCacheException, KERN_EXC_FRAME_SIZE, ra) .set noat .mask 0x80000000, -4 PTR_LA k0, _C_LABEL(panic) # return to panic PTR_LA a0, 9f # panicstr MFC0 a1, MIPS_COP_0_ERROR_PC mfc0 a2, MIPS_COP_0_CACHE_ERR # 3rd arg cache error MTC0 k0, MIPS_COP_0_ERROR_PC # set return address mfc0 k0, MIPS_COP_0_STATUS # restore status li k1, MIPS_SR_DIAG_PE # ignore further errors or k0, k1 mtc0 k0, MIPS_COP_0_STATUS # restore status COP0_SYNC eret MSG("cache error @ EPC 0x%x CachErr 0x%x"); .set at END(MipsCacheException) Index: head/sys/mips/mips/fp.S =================================================================== --- head/sys/mips/mips/fp.S (revision 331253) +++ head/sys/mips/mips/fp.S (revision 331254) @@ -1,3610 +1,3610 @@ /* $OpenBSD: fp.S,v 1.2 1998/03/16 09:03:31 pefo Exp $ */ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)fp.s 8.1 (Berkeley) 6/10/93 * JNPR: fp.S,v 1.1 2006/08/07 05:38:57 katta * $FreeBSD$ */ /* * Standard header stuff. */ #include #include #include -#include "assym.s" +#include "assym.inc" #define SEXP_INF 0xff #define DEXP_INF 0x7ff #define SEXP_BIAS 127 #define DEXP_BIAS 1023 #define SEXP_MIN -126 #define DEXP_MIN -1022 #define SEXP_MAX 127 #define DEXP_MAX 1023 #define WEXP_MAX 30 /* maximum unbiased exponent for int */ #define WEXP_MIN -1 /* minimum unbiased exponent for int */ #define SFRAC_BITS 23 #define DFRAC_BITS 52 #define SIMPL_ONE 0x00800000 #define DIMPL_ONE 0x00100000 #define SLEAD_ZEROS 31 - 23 #define DLEAD_ZEROS 31 - 20 #define STICKYBIT 1 #define GUARDBIT 0x80000000 #define SSIGNAL_NAN 0x00400000 #define DSIGNAL_NAN 0x00080000 #define SQUIET_NAN 0x003fffff #define DQUIET_NAN0 0x0007ffff #define DQUIET_NAN1 0xffffffff #define INT_MIN 0x80000000 #define INT_MAX 0x7fffffff #define COND_UNORDERED 0x1 #define COND_EQUAL 0x2 #define COND_LESS 0x4 #define COND_SIGNAL 0x8 .set hardfloat /*---------------------------------------------------------------------------- * * MipsEmulateFP -- * * Emulate unimplemented floating point operations. * This routine should only be called by MipsFPInterrupt(). * * MipsEmulateFP(instr) * unsigned instr; * * Results: * None. * * Side effects: * Floating point registers are modified according to instruction. * *---------------------------------------------------------------------------- */ NESTED(MipsEmulateFP, CALLFRAME_SIZ, ra) subu sp, sp, CALLFRAME_SIZ sw ra, CALLFRAME_RA(sp) /* * Decode the FMT field (bits 24-21) and FUNCTION field (bits 5-0). */ srl v0, a0, 21 - 2 # get FMT field and v0, v0, 0xF << 2 # mask FMT field and v1, a0, 0x3F # mask FUNC field sll v1, v1, 5 # align for table lookup bgt v0, 4 << 2, ill # illegal format or v1, v1, v0 cfc1 a1, MIPS_FPU_CSR # get exception register lw a3, func_fmt_tbl(v1) # switch on FUNC & FMT and a1, a1, ~MIPS_FPU_EXCEPTION_UNIMPL # clear exception ctc1 a1, MIPS_FPU_CSR j a3 .rdata func_fmt_tbl: .word add_s # 0 .word add_d # 0 .word ill # 0 .word ill # 0 .word ill # 0 .word ill # 0 .word ill # 0 .word ill # 0 .word sub_s # 1 .word sub_d # 1 .word ill # 1 .word ill # 1 .word ill # 1 .word ill # 1 .word ill # 1 .word ill # 1 .word mul_s # 2 .word mul_d # 2 .word ill # 2 .word ill # 2 .word ill # 2 .word ill # 2 .word ill # 2 .word ill # 2 .word div_s # 3 .word div_d # 3 .word ill # 3 .word ill # 3 .word ill # 3 .word ill # 3 .word ill # 3 .word ill # 3 .word ill # 4 .word ill # 4 .word ill # 4 .word ill # 4 .word ill # 4 .word ill # 4 .word ill # 4 .word ill # 4 .word abs_s # 5 .word abs_d # 5 .word ill # 5 .word ill # 5 .word ill # 5 .word ill # 5 .word ill # 5 .word ill # 5 .word mov_s # 6 .word mov_d # 6 .word ill # 6 .word ill # 6 .word ill # 6 .word ill # 6 .word ill # 6 .word ill # 6 .word neg_s # 7 .word neg_d # 7 .word ill # 7 .word ill # 7 .word ill # 7 .word ill # 7 .word ill # 7 .word ill # 7 .word ill # 8 .word ill # 8 .word ill # 8 .word ill # 8 .word ill # 8 .word ill # 8 .word ill # 8 .word ill # 8 .word ill # 9 .word ill # 9 .word ill # 9 .word ill # 9 .word ill # 9 .word ill # 9 .word ill # 9 .word ill # 9 .word ill # 10 .word ill # 10 .word ill # 10 .word ill # 10 .word ill # 10 .word ill # 10 .word ill # 10 .word ill # 10 .word ill # 11 .word ill # 11 .word ill # 11 .word ill # 11 .word ill # 11 .word ill # 11 .word ill # 11 .word ill # 11 .word ill # 12 .word ill # 12 .word ill # 12 .word ill # 12 .word ill # 12 .word ill # 12 .word ill # 12 .word ill # 12 .word ill # 13 .word ill # 13 .word ill # 13 .word ill # 13 .word ill # 13 .word ill # 13 .word ill # 13 .word ill # 13 .word ill # 14 .word ill # 14 .word ill # 14 .word ill # 14 .word ill # 14 .word ill # 14 .word ill # 14 .word ill # 14 .word ill # 15 .word ill # 15 .word ill # 15 .word ill # 15 .word ill # 15 .word ill # 15 .word ill # 15 .word ill # 15 .word ill # 16 .word ill # 16 .word ill # 16 .word ill # 16 .word ill # 16 .word ill # 16 .word ill # 16 .word ill # 16 .word ill # 17 .word ill # 17 .word ill # 17 .word ill # 17 .word ill # 17 .word ill # 17 .word ill # 17 .word ill # 17 .word ill # 18 .word ill # 18 .word ill # 18 .word ill # 18 .word ill # 18 .word ill # 18 .word ill # 18 .word ill # 18 .word ill # 19 .word ill # 19 .word ill # 19 .word ill # 19 .word ill # 19 .word ill # 19 .word ill # 19 .word ill # 19 .word ill # 20 .word ill # 20 .word ill # 20 .word ill # 20 .word ill # 20 .word ill # 20 .word ill # 20 .word ill # 20 .word ill # 21 .word ill # 21 .word ill # 21 .word ill # 21 .word ill # 21 .word ill # 21 .word ill # 21 .word ill # 21 .word ill # 22 .word ill # 22 .word ill # 22 .word ill # 22 .word ill # 22 .word ill # 22 .word ill # 22 .word ill # 22 .word ill # 23 .word ill # 23 .word ill # 23 .word ill # 23 .word ill # 23 .word ill # 23 .word ill # 23 .word ill # 23 .word ill # 24 .word ill # 24 .word ill # 24 .word ill # 24 .word ill # 24 .word ill # 24 .word ill # 24 .word ill # 24 .word ill # 25 .word ill # 25 .word ill # 25 .word ill # 25 .word ill # 25 .word ill # 25 .word ill # 25 .word ill # 25 .word ill # 26 .word ill # 26 .word ill # 26 .word ill # 26 .word ill # 26 .word ill # 26 .word ill # 26 .word ill # 26 .word ill # 27 .word ill # 27 .word ill # 27 .word ill # 27 .word ill # 27 .word ill # 27 .word ill # 27 .word ill # 27 .word ill # 28 .word ill # 28 .word ill # 28 .word ill # 28 .word ill # 28 .word ill # 28 .word ill # 28 .word ill # 28 .word ill # 29 .word ill # 29 .word ill # 29 .word ill # 29 .word ill # 29 .word ill # 29 .word ill # 29 .word ill # 29 .word ill # 30 .word ill # 30 .word ill # 30 .word ill # 30 .word ill # 30 .word ill # 30 .word ill # 30 .word ill # 30 .word ill # 31 .word ill # 31 .word ill # 31 .word ill # 31 .word ill # 31 .word ill # 31 .word ill # 31 .word ill # 31 .word ill # 32 .word cvt_s_d # 32 .word ill # 32 .word ill # 32 .word cvt_s_w # 32 .word ill # 32 .word ill # 32 .word ill # 32 .word cvt_d_s # 33 .word ill # 33 .word ill # 33 .word ill # 33 .word cvt_d_w # 33 .word ill # 33 .word ill # 33 .word ill # 33 .word ill # 34 .word ill # 34 .word ill # 34 .word ill # 34 .word ill # 34 .word ill # 34 .word ill # 34 .word ill # 34 .word ill # 35 .word ill # 35 .word ill # 35 .word ill # 35 .word ill # 35 .word ill # 35 .word ill # 35 .word ill # 35 .word cvt_w_s # 36 .word cvt_w_d # 36 .word ill # 36 .word ill # 36 .word ill # 36 .word ill # 36 .word ill # 36 .word ill # 36 .word ill # 37 .word ill # 37 .word ill # 37 .word ill # 37 .word ill # 37 .word ill # 37 .word ill # 37 .word ill # 37 .word ill # 38 .word ill # 38 .word ill # 38 .word ill # 38 .word ill # 38 .word ill # 38 .word ill # 38 .word ill # 38 .word ill # 39 .word ill # 39 .word ill # 39 .word ill # 39 .word ill # 39 .word ill # 39 .word ill # 39 .word ill # 39 .word ill # 40 .word ill # 40 .word ill # 40 .word ill # 40 .word ill # 40 .word ill # 40 .word ill # 40 .word ill # 40 .word ill # 41 .word ill # 41 .word ill # 41 .word ill # 41 .word ill # 41 .word ill # 41 .word ill # 41 .word ill # 41 .word ill # 42 .word ill # 42 .word ill # 42 .word ill # 42 .word ill # 42 .word ill # 42 .word ill # 42 .word ill # 42 .word ill # 43 .word ill # 43 .word ill # 43 .word ill # 43 .word ill # 43 .word ill # 43 .word ill # 43 .word ill # 43 .word ill # 44 .word ill # 44 .word ill # 44 .word ill # 44 .word ill # 44 .word ill # 44 .word ill # 44 .word ill # 44 .word ill # 45 .word ill # 45 .word ill # 45 .word ill # 45 .word ill # 45 .word ill # 45 .word ill # 45 .word ill # 45 .word ill # 46 .word ill # 46 .word ill # 46 .word ill # 46 .word ill # 46 .word ill # 46 .word ill # 46 .word ill # 46 .word ill # 47 .word ill # 47 .word ill # 47 .word ill # 47 .word ill # 47 .word ill # 47 .word ill # 47 .word ill # 47 .word cmp_s # 48 .word cmp_d # 48 .word ill # 48 .word ill # 48 .word ill # 48 .word ill # 48 .word ill # 48 .word ill # 48 .word cmp_s # 49 .word cmp_d # 49 .word ill # 49 .word ill # 49 .word ill # 49 .word ill # 49 .word ill # 49 .word ill # 49 .word cmp_s # 50 .word cmp_d # 50 .word ill # 50 .word ill # 50 .word ill # 50 .word ill # 50 .word ill # 50 .word ill # 50 .word cmp_s # 51 .word cmp_d # 51 .word ill # 51 .word ill # 51 .word ill # 51 .word ill # 51 .word ill # 51 .word ill # 51 .word cmp_s # 52 .word cmp_d # 52 .word ill # 52 .word ill # 52 .word ill # 52 .word ill # 52 .word ill # 52 .word ill # 52 .word cmp_s # 53 .word cmp_d # 53 .word ill # 53 .word ill # 53 .word ill # 53 .word ill # 53 .word ill # 53 .word ill # 53 .word cmp_s # 54 .word cmp_d # 54 .word ill # 54 .word ill # 54 .word ill # 54 .word ill # 54 .word ill # 54 .word ill # 54 .word cmp_s # 55 .word cmp_d # 55 .word ill # 55 .word ill # 55 .word ill # 55 .word ill # 55 .word ill # 55 .word ill # 55 .word cmp_s # 56 .word cmp_d # 56 .word ill # 56 .word ill # 56 .word ill # 56 .word ill # 56 .word ill # 56 .word ill # 56 .word cmp_s # 57 .word cmp_d # 57 .word ill # 57 .word ill # 57 .word ill # 57 .word ill # 57 .word ill # 57 .word ill # 57 .word cmp_s # 58 .word cmp_d # 58 .word ill # 58 .word ill # 58 .word ill # 58 .word ill # 58 .word ill # 58 .word ill # 58 .word cmp_s # 59 .word cmp_d # 59 .word ill # 59 .word ill # 59 .word ill # 59 .word ill # 59 .word ill # 59 .word ill # 59 .word cmp_s # 60 .word cmp_d # 60 .word ill # 60 .word ill # 60 .word ill # 60 .word ill # 60 .word ill # 60 .word ill # 60 .word cmp_s # 61 .word cmp_d # 61 .word ill # 61 .word ill # 61 .word ill # 61 .word ill # 61 .word ill # 61 .word ill # 61 .word cmp_s # 62 .word cmp_d # 62 .word ill # 62 .word ill # 62 .word ill # 62 .word ill # 62 .word ill # 62 .word ill # 62 .word cmp_s # 63 .word cmp_d # 63 .word ill # 63 .word ill # 63 .word ill # 63 .word ill # 63 .word ill # 63 .word ill # 63 .text /* * Single precision subtract. */ sub_s: jal get_ft_fs_s xor ta0, ta0, 1 # negate FT sign bit b add_sub_s /* * Single precision add. */ add_s: jal get_ft_fs_s add_sub_s: bne t1, SEXP_INF, 1f # is FS an infinity? bne ta1, SEXP_INF, result_fs_s # if FT is not inf, result=FS bne t2, zero, result_fs_s # if FS is NAN, result is FS bne ta2, zero, result_ft_s # if FT is NAN, result is FT bne t0, ta0, invalid_s # both infinities same sign? b result_fs_s # result is in FS 1: beq ta1, SEXP_INF, result_ft_s # if FT is inf, result=FT bne t1, zero, 4f # is FS a denormalized num? beq t2, zero, 3f # is FS zero? bne ta1, zero, 2f # is FT a denormalized num? beq ta2, zero, result_fs_s # FT is zero, result=FS jal renorm_fs_s jal renorm_ft_s b 5f 2: jal renorm_fs_s subu ta1, ta1, SEXP_BIAS # unbias FT exponent or ta2, ta2, SIMPL_ONE # set implied one bit b 5f 3: bne ta1, zero, result_ft_s # if FT != 0, result=FT bne ta2, zero, result_ft_s and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode bne v0, MIPS_FPU_ROUND_RM, 1f # round to -infinity? or t0, t0, ta0 # compute result sign b result_fs_s 1: and t0, t0, ta0 # compute result sign b result_fs_s 4: bne ta1, zero, 2f # is FT a denormalized num? beq ta2, zero, result_fs_s # FT is zero, result=FS subu t1, t1, SEXP_BIAS # unbias FS exponent or t2, t2, SIMPL_ONE # set implied one bit jal renorm_ft_s b 5f 2: subu t1, t1, SEXP_BIAS # unbias FS exponent or t2, t2, SIMPL_ONE # set implied one bit subu ta1, ta1, SEXP_BIAS # unbias FT exponent or ta2, ta2, SIMPL_ONE # set implied one bit /* * Perform the addition. */ 5: move t8, zero # no shifted bits (sticky reg) beq t1, ta1, 4f # no shift needed subu v0, t1, ta1 # v0 = difference of exponents move v1, v0 # v1 = abs(difference) bge v0, zero, 1f negu v1 1: ble v1, SFRAC_BITS+2, 2f # is difference too great? li t8, STICKYBIT # set the sticky bit bge v0, zero, 1f # check which exp is larger move t1, ta1 # result exp is FTs move t2, zero # FSs fraction shifted is zero b 4f 1: move ta2, zero # FTs fraction shifted is zero b 4f 2: li t9, 32 # compute 32 - abs(exp diff) subu t9, t9, v1 bgt v0, zero, 3f # if FS > FT, shift FTs frac move t1, ta1 # FT > FS, result exp is FTs sll t8, t2, t9 # save bits shifted out srl t2, t2, v1 # shift FSs fraction b 4f 3: sll t8, ta2, t9 # save bits shifted out srl ta2, ta2, v1 # shift FTs fraction 4: bne t0, ta0, 1f # if signs differ, subtract addu t2, t2, ta2 # add fractions b norm_s 1: blt t2, ta2, 3f # subtract larger from smaller bne t2, ta2, 2f # if same, result=0 move t1, zero # result=0 move t2, zero and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode bne v0, MIPS_FPU_ROUND_RM, 1f # round to -infinity? or t0, t0, ta0 # compute result sign b result_fs_s 1: and t0, t0, ta0 # compute result sign b result_fs_s 2: sltu t9, zero, t8 # compute t2:zero - ta2:t8 subu t8, zero, t8 subu t2, t2, ta2 # subtract fractions subu t2, t2, t9 # subtract barrow b norm_s 3: move t0, ta0 # sign of result = FTs sltu t9, zero, t8 # compute ta2:zero - t2:t8 subu t8, zero, t8 subu t2, ta2, t2 # subtract fractions subu t2, t2, t9 # subtract barrow b norm_s /* * Double precision subtract. */ sub_d: jal get_ft_fs_d xor ta0, ta0, 1 # negate sign bit b add_sub_d /* * Double precision add. */ add_d: jal get_ft_fs_d add_sub_d: bne t1, DEXP_INF, 1f # is FS an infinity? bne ta1, DEXP_INF, result_fs_d # if FT is not inf, result=FS bne t2, zero, result_fs_d # if FS is NAN, result is FS bne t3, zero, result_fs_d bne ta2, zero, result_ft_d # if FT is NAN, result is FT bne ta3, zero, result_ft_d bne t0, ta0, invalid_d # both infinities same sign? b result_fs_d # result is in FS 1: beq ta1, DEXP_INF, result_ft_d # if FT is inf, result=FT bne t1, zero, 4f # is FS a denormalized num? bne t2, zero, 1f # is FS zero? beq t3, zero, 3f 1: bne ta1, zero, 2f # is FT a denormalized num? bne ta2, zero, 1f beq ta3, zero, result_fs_d # FT is zero, result=FS 1: jal renorm_fs_d jal renorm_ft_d b 5f 2: jal renorm_fs_d subu ta1, ta1, DEXP_BIAS # unbias FT exponent or ta2, ta2, DIMPL_ONE # set implied one bit b 5f 3: bne ta1, zero, result_ft_d # if FT != 0, result=FT bne ta2, zero, result_ft_d bne ta3, zero, result_ft_d and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode bne v0, MIPS_FPU_ROUND_RM, 1f # round to -infinity? or t0, t0, ta0 # compute result sign b result_fs_d 1: and t0, t0, ta0 # compute result sign b result_fs_d 4: bne ta1, zero, 2f # is FT a denormalized num? bne ta2, zero, 1f beq ta3, zero, result_fs_d # FT is zero, result=FS 1: subu t1, t1, DEXP_BIAS # unbias FS exponent or t2, t2, DIMPL_ONE # set implied one bit jal renorm_ft_d b 5f 2: subu t1, t1, DEXP_BIAS # unbias FS exponent or t2, t2, DIMPL_ONE # set implied one bit subu ta1, ta1, DEXP_BIAS # unbias FT exponent or ta2, ta2, DIMPL_ONE # set implied one bit /* * Perform the addition. */ 5: move t8, zero # no shifted bits (sticky reg) beq t1, ta1, 4f # no shift needed subu v0, t1, ta1 # v0 = difference of exponents move v1, v0 # v1 = abs(difference) bge v0, zero, 1f negu v1 1: ble v1, DFRAC_BITS+2, 2f # is difference too great? li t8, STICKYBIT # set the sticky bit bge v0, zero, 1f # check which exp is larger move t1, ta1 # result exp is FTs move t2, zero # FSs fraction shifted is zero move t3, zero b 4f 1: move ta2, zero # FTs fraction shifted is zero move ta3, zero b 4f 2: li t9, 32 bge v0, zero, 3f # if FS > FT, shift FTs frac move t1, ta1 # FT > FS, result exp is FTs blt v1, t9, 1f # shift right by < 32? subu v1, v1, t9 subu t9, t9, v1 sll t8, t2, t9 # save bits shifted out sltu t9, zero, t3 # dont lose any one bits or t8, t8, t9 # save sticky bit srl t3, t2, v1 # shift FSs fraction move t2, zero b 4f 1: subu t9, t9, v1 sll t8, t3, t9 # save bits shifted out srl t3, t3, v1 # shift FSs fraction sll t9, t2, t9 # save bits shifted out of t2 or t3, t3, t9 # and put into t3 srl t2, t2, v1 b 4f 3: blt v1, t9, 1f # shift right by < 32? subu v1, v1, t9 subu t9, t9, v1 sll t8, ta2, t9 # save bits shifted out srl ta3, ta2, v1 # shift FTs fraction move ta2, zero b 4f 1: subu t9, t9, v1 sll t8, ta3, t9 # save bits shifted out srl ta3, ta3, v1 # shift FTs fraction sll t9, ta2, t9 # save bits shifted out of t2 or ta3, ta3, t9 # and put into t3 srl ta2, ta2, v1 4: bne t0, ta0, 1f # if signs differ, subtract addu t3, t3, ta3 # add fractions sltu t9, t3, ta3 # compute carry addu t2, t2, ta2 # add fractions addu t2, t2, t9 # add carry b norm_d 1: blt t2, ta2, 3f # subtract larger from smaller bne t2, ta2, 2f bltu t3, ta3, 3f bne t3, ta3, 2f # if same, result=0 move t1, zero # result=0 move t2, zero move t3, zero and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode bne v0, MIPS_FPU_ROUND_RM, 1f # round to -infinity? or t0, t0, ta0 # compute result sign b result_fs_d 1: and t0, t0, ta0 # compute result sign b result_fs_d 2: beq t8, zero, 1f # compute t2:t3:zero - ta2:ta3:t8 subu t8, zero, t8 sltu v0, t3, 1 # compute barrow out subu t3, t3, 1 # subtract barrow subu t2, t2, v0 1: sltu v0, t3, ta3 subu t3, t3, ta3 # subtract fractions subu t2, t2, ta2 # subtract fractions subu t2, t2, v0 # subtract barrow b norm_d 3: move t0, ta0 # sign of result = FTs beq t8, zero, 1f # compute ta2:ta3:zero - t2:t3:t8 subu t8, zero, t8 sltu v0, ta3, 1 # compute barrow out subu ta3, ta3, 1 # subtract barrow subu ta2, ta2, v0 1: sltu v0, ta3, t3 subu t3, ta3, t3 # subtract fractions subu t2, ta2, t2 # subtract fractions subu t2, t2, v0 # subtract barrow b norm_d /* * Single precision multiply. */ mul_s: jal get_ft_fs_s xor t0, t0, ta0 # compute sign of result move ta0, t0 bne t1, SEXP_INF, 2f # is FS an infinity? bne t2, zero, result_fs_s # if FS is a NAN, result=FS bne ta1, SEXP_INF, 1f # FS is inf, is FT an infinity? bne ta2, zero, result_ft_s # if FT is a NAN, result=FT b result_fs_s # result is infinity 1: bne ta1, zero, result_fs_s # inf * zero? if no, result=FS bne ta2, zero, result_fs_s b invalid_s # infinity * zero is invalid 2: bne ta1, SEXP_INF, 1f # FS != inf, is FT an infinity? bne t1, zero, result_ft_s # zero * inf? if no, result=FT bne t2, zero, result_ft_s bne ta2, zero, result_ft_s # if FT is a NAN, result=FT b invalid_s # zero * infinity is invalid 1: bne t1, zero, 1f # is FS zero? beq t2, zero, result_fs_s # result is zero jal renorm_fs_s b 2f 1: subu t1, t1, SEXP_BIAS # unbias FS exponent or t2, t2, SIMPL_ONE # set implied one bit 2: bne ta1, zero, 1f # is FT zero? beq ta2, zero, result_ft_s # result is zero jal renorm_ft_s b 2f 1: subu ta1, ta1, SEXP_BIAS # unbias FT exponent or ta2, ta2, SIMPL_ONE # set implied one bit 2: addu t1, t1, ta1 # compute result exponent addu t1, t1, 9 # account for binary point multu t2, ta2 # multiply fractions mflo t8 mfhi t2 b norm_s /* * Double precision multiply. */ mul_d: jal get_ft_fs_d xor t0, t0, ta0 # compute sign of result move ta0, t0 bne t1, DEXP_INF, 2f # is FS an infinity? bne t2, zero, result_fs_d # if FS is a NAN, result=FS bne t3, zero, result_fs_d bne ta1, DEXP_INF, 1f # FS is inf, is FT an infinity? bne ta2, zero, result_ft_d # if FT is a NAN, result=FT bne ta3, zero, result_ft_d b result_fs_d # result is infinity 1: bne ta1, zero, result_fs_d # inf * zero? if no, result=FS bne ta2, zero, result_fs_d bne ta3, zero, result_fs_d b invalid_d # infinity * zero is invalid 2: bne ta1, DEXP_INF, 1f # FS != inf, is FT an infinity? bne t1, zero, result_ft_d # zero * inf? if no, result=FT bne t2, zero, result_ft_d # if FS is a NAN, result=FS bne t3, zero, result_ft_d bne ta2, zero, result_ft_d # if FT is a NAN, result=FT bne ta3, zero, result_ft_d b invalid_d # zero * infinity is invalid 1: bne t1, zero, 2f # is FS zero? bne t2, zero, 1f beq t3, zero, result_fs_d # result is zero 1: jal renorm_fs_d b 3f 2: subu t1, t1, DEXP_BIAS # unbias FS exponent or t2, t2, DIMPL_ONE # set implied one bit 3: bne ta1, zero, 2f # is FT zero? bne ta2, zero, 1f beq ta3, zero, result_ft_d # result is zero 1: jal renorm_ft_d b 3f 2: subu ta1, ta1, DEXP_BIAS # unbias FT exponent or ta2, ta2, DIMPL_ONE # set implied one bit 3: addu t1, t1, ta1 # compute result exponent addu t1, t1, 12 # ??? multu t3, ta3 # multiply fractions (low * low) move ta0, t2 # free up t2,t3 for result move ta1, t3 mflo a3 # save low order bits mfhi t8 not v0, t8 multu ta0, ta3 # multiply FS(high) * FT(low) mflo v1 mfhi t3 # init low result sltu v0, v0, v1 # compute carry addu t8, v1 multu ta1, ta2 # multiply FS(low) * FT(high) addu t3, t3, v0 # add carry not v0, t8 mflo v1 mfhi t2 sltu v0, v0, v1 addu t8, v1 multu ta0, ta2 # multiply FS(high) * FT(high) addu t3, v0 not v1, t3 sltu v1, v1, t2 addu t3, t2 not v0, t3 mfhi t2 addu t2, v1 mflo v1 sltu v0, v0, v1 addu t2, v0 addu t3, v1 sltu a3, zero, a3 # reduce t8,a3 to just t8 or t8, a3 b norm_d /* * Single precision divide. */ div_s: jal get_ft_fs_s xor t0, t0, ta0 # compute sign of result move ta0, t0 bne t1, SEXP_INF, 1f # is FS an infinity? bne t2, zero, result_fs_s # if FS is NAN, result is FS bne ta1, SEXP_INF, result_fs_s # is FT an infinity? bne ta2, zero, result_ft_s # if FT is NAN, result is FT b invalid_s # infinity/infinity is invalid 1: bne ta1, SEXP_INF, 1f # is FT an infinity? bne ta2, zero, result_ft_s # if FT is NAN, result is FT move t1, zero # x / infinity is zero move t2, zero b result_fs_s 1: bne t1, zero, 2f # is FS zero? bne t2, zero, 1f bne ta1, zero, result_fs_s # FS=zero, is FT zero? beq ta2, zero, invalid_s # 0 / 0 b result_fs_s # result = zero 1: jal renorm_fs_s b 3f 2: subu t1, t1, SEXP_BIAS # unbias FS exponent or t2, t2, SIMPL_ONE # set implied one bit 3: bne ta1, zero, 2f # is FT zero? bne ta2, zero, 1f or a1, a1, MIPS_FPU_EXCEPTION_DIV0 | MIPS_FPU_STICKY_DIV0 and v0, a1, MIPS_FPU_ENABLE_DIV0 # trap enabled? bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions li t1, SEXP_INF # result is infinity move t2, zero b result_fs_s 1: jal renorm_ft_s b 3f 2: subu ta1, ta1, SEXP_BIAS # unbias FT exponent or ta2, ta2, SIMPL_ONE # set implied one bit 3: subu t1, t1, ta1 # compute exponent subu t1, t1, 3 # compensate for result position li v0, SFRAC_BITS+3 # number of bits to divide move t8, t2 # init dividend move t2, zero # init result 1: bltu t8, ta2, 3f # is dividend >= divisor? 2: subu t8, t8, ta2 # subtract divisor from dividend or t2, t2, 1 # remember that we did bne t8, zero, 3f # if not done, continue sll t2, t2, v0 # shift result to final position b norm_s 3: sll t8, t8, 1 # shift dividend sll t2, t2, 1 # shift result subu v0, v0, 1 # are we done? bne v0, zero, 1b # no, continue b norm_s /* * Double precision divide. */ div_d: jal get_ft_fs_d xor t0, t0, ta0 # compute sign of result move ta0, t0 bne t1, DEXP_INF, 1f # is FS an infinity? bne t2, zero, result_fs_d # if FS is NAN, result is FS bne t3, zero, result_fs_d bne ta1, DEXP_INF, result_fs_d # is FT an infinity? bne ta2, zero, result_ft_d # if FT is NAN, result is FT bne ta3, zero, result_ft_d b invalid_d # infinity/infinity is invalid 1: bne ta1, DEXP_INF, 1f # is FT an infinity? bne ta2, zero, result_ft_d # if FT is NAN, result is FT bne ta3, zero, result_ft_d move t1, zero # x / infinity is zero move t2, zero move t3, zero b result_fs_d 1: bne t1, zero, 2f # is FS zero? bne t2, zero, 1f bne t3, zero, 1f bne ta1, zero, result_fs_d # FS=zero, is FT zero? bne ta2, zero, result_fs_d beq ta3, zero, invalid_d # 0 / 0 b result_fs_d # result = zero 1: jal renorm_fs_d b 3f 2: subu t1, t1, DEXP_BIAS # unbias FS exponent or t2, t2, DIMPL_ONE # set implied one bit 3: bne ta1, zero, 2f # is FT zero? bne ta2, zero, 1f bne ta3, zero, 1f or a1, a1, MIPS_FPU_EXCEPTION_DIV0 | MIPS_FPU_STICKY_DIV0 and v0, a1, MIPS_FPU_ENABLE_DIV0 # trap enabled? bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # Save exceptions li t1, DEXP_INF # result is infinity move t2, zero move t3, zero b result_fs_d 1: jal renorm_ft_d b 3f 2: subu ta1, ta1, DEXP_BIAS # unbias FT exponent or ta2, ta2, DIMPL_ONE # set implied one bit 3: subu t1, t1, ta1 # compute exponent subu t1, t1, 3 # compensate for result position li v0, DFRAC_BITS+3 # number of bits to divide move t8, t2 # init dividend move t9, t3 move t2, zero # init result move t3, zero 1: bltu t8, ta2, 3f # is dividend >= divisor? bne t8, ta2, 2f bltu t9, ta3, 3f 2: sltu v1, t9, ta3 # subtract divisor from dividend subu t9, t9, ta3 subu t8, t8, ta2 subu t8, t8, v1 or t3, t3, 1 # remember that we did bne t8, zero, 3f # if not done, continue bne t9, zero, 3f li v1, 32 # shift result to final position blt v0, v1, 2f # shift < 32 bits? subu v0, v0, v1 # shift by > 32 bits sll t2, t3, v0 # shift upper part move t3, zero b norm_d 2: subu v1, v1, v0 # shift by < 32 bits sll t2, t2, v0 # shift upper part srl t9, t3, v1 # save bits shifted out or t2, t2, t9 # and put into upper part sll t3, t3, v0 b norm_d 3: sll t8, t8, 1 # shift dividend srl v1, t9, 31 # save bit shifted out or t8, t8, v1 # and put into upper part sll t9, t9, 1 sll t2, t2, 1 # shift result srl v1, t3, 31 # save bit shifted out or t2, t2, v1 # and put into upper part sll t3, t3, 1 subu v0, v0, 1 # are we done? bne v0, zero, 1b # no, continue sltu v0, zero, t9 # be sure to save any one bits or t8, t8, v0 # from the lower remainder b norm_d /* * Single precision absolute value. */ abs_s: jal get_fs_s move t0, zero # set sign positive b result_fs_s /* * Double precision absolute value. */ abs_d: jal get_fs_d move t0, zero # set sign positive b result_fs_d /* * Single precision move. */ mov_s: jal get_fs_s b result_fs_s /* * Double precision move. */ mov_d: jal get_fs_d b result_fs_d /* * Single precision negate. */ neg_s: jal get_fs_s xor t0, t0, 1 # reverse sign b result_fs_s /* * Double precision negate. */ neg_d: jal get_fs_d xor t0, t0, 1 # reverse sign b result_fs_d /* * Convert double to single. */ cvt_s_d: jal get_fs_d bne t1, DEXP_INF, 1f # is FS an infinity? li t1, SEXP_INF # convert to single sll t2, t2, 3 # convert D fraction to S srl t8, t3, 32 - 3 or t2, t2, t8 b result_fs_s 1: bne t1, zero, 2f # is FS zero? bne t2, zero, 1f beq t3, zero, result_fs_s # result=0 1: jal renorm_fs_d subu t1, t1, 3 # correct exp for shift below b 3f 2: subu t1, t1, DEXP_BIAS # unbias exponent or t2, t2, DIMPL_ONE # add implied one bit 3: sll t2, t2, 3 # convert D fraction to S srl t8, t3, 32 - 3 or t2, t2, t8 sll t8, t3, 3 b norm_noshift_s /* * Convert integer to single. */ cvt_s_w: jal get_fs_int bne t2, zero, 1f # check for zero move t1, zero b result_fs_s /* * Find out how many leading zero bits are in t2 and put in t9. */ 1: move v0, t2 move t9, zero srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift t2 the correct number of bits. */ 1: subu t9, t9, SLEAD_ZEROS # dont count leading zeros li t1, 23 # init exponent subu t1, t1, t9 # compute exponent beq t9, zero, 1f li v0, 32 blt t9, zero, 2f # if shift < 0, shift right subu v0, v0, t9 sll t2, t2, t9 # shift left 1: add t1, t1, SEXP_BIAS # bias exponent and t2, t2, ~SIMPL_ONE # clear implied one bit b result_fs_s 2: negu t9 # shift right by t9 subu v0, v0, t9 sll t8, t2, v0 # save bits shifted out srl t2, t2, t9 b norm_noshift_s /* * Convert single to double. */ cvt_d_s: jal get_fs_s move t3, zero bne t1, SEXP_INF, 1f # is FS an infinity? li t1, DEXP_INF # convert to double b result_fs_d 1: bne t1, zero, 2f # is FS denormalized or zero? beq t2, zero, result_fs_d # is FS zero? jal renorm_fs_s move t8, zero b norm_d 2: addu t1, t1, DEXP_BIAS - SEXP_BIAS # bias exponent correctly sll t3, t2, 32 - 3 # convert S fraction to D srl t2, t2, 3 b result_fs_d /* * Convert integer to double. */ cvt_d_w: jal get_fs_int bne t2, zero, 1f # check for zero move t1, zero # result=0 move t3, zero b result_fs_d /* * Find out how many leading zero bits are in t2 and put in t9. */ 1: move v0, t2 move t9, zero srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift t2 the correct number of bits. */ 1: subu t9, t9, DLEAD_ZEROS # dont count leading zeros li t1, DEXP_BIAS + 20 # init exponent subu t1, t1, t9 # compute exponent beq t9, zero, 1f li v0, 32 blt t9, zero, 2f # if shift < 0, shift right subu v0, v0, t9 sll t2, t2, t9 # shift left 1: and t2, t2, ~DIMPL_ONE # clear implied one bit move t3, zero b result_fs_d 2: negu t9 # shift right by t9 subu v0, v0, t9 sll t3, t2, v0 srl t2, t2, t9 and t2, t2, ~DIMPL_ONE # clear implied one bit b result_fs_d /* * Convert single to integer. */ cvt_w_s: jal get_fs_s bne t1, SEXP_INF, 1f # is FS an infinity? bne t2, zero, invalid_w # invalid conversion 1: bne t1, zero, 1f # is FS zero? beq t2, zero, result_fs_w # result is zero move t2, zero # result is an inexact zero b inexact_w 1: subu t1, t1, SEXP_BIAS # unbias exponent or t2, t2, SIMPL_ONE # add implied one bit sll t3, t2, 32 - 3 # convert S fraction to D srl t2, t2, 3 b cvt_w /* * Convert double to integer. */ cvt_w_d: jal get_fs_d bne t1, DEXP_INF, 1f # is FS an infinity? bne t2, zero, invalid_w # invalid conversion bne t3, zero, invalid_w # invalid conversion 1: bne t1, zero, 2f # is FS zero? bne t2, zero, 1f beq t3, zero, result_fs_w # result is zero 1: move t2, zero # result is an inexact zero b inexact_w 2: subu t1, t1, DEXP_BIAS # unbias exponent or t2, t2, DIMPL_ONE # add implied one bit cvt_w: blt t1, WEXP_MIN, underflow_w # is exponent too small? li v0, WEXP_MAX+1 bgt t1, v0, overflow_w # is exponent too large? bne t1, v0, 1f # special check for INT_MIN beq t0, zero, overflow_w # if positive, overflow bne t2, DIMPL_ONE, overflow_w bne t3, zero, overflow_w li t2, INT_MIN # result is INT_MIN b result_fs_w 1: subu v0, t1, 20 # compute amount to shift beq v0, zero, 2f # is shift needed? li v1, 32 blt v0, zero, 1f # if shift < 0, shift right subu v1, v1, v0 # shift left sll t2, t2, v0 srl t9, t3, v1 # save bits shifted out of t3 or t2, t2, t9 # and put into t2 sll t3, t3, v0 # shift FSs fraction b 2f 1: negu v0 # shift right by v0 subu v1, v1, v0 sll t8, t3, v1 # save bits shifted out sltu t8, zero, t8 # dont lose any ones srl t3, t3, v0 # shift FSs fraction or t3, t3, t8 sll t9, t2, v1 # save bits shifted out of t2 or t3, t3, t9 # and put into t3 srl t2, t2, v0 /* * round result (t0 is sign, t2 is integer part, t3 is fractional part). */ 2: and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, 3f # round to nearest beq v0, MIPS_FPU_ROUND_RZ, 5f # round to zero (truncate) beq v0, MIPS_FPU_ROUND_RP, 1f # round to +infinity beq t0, zero, 5f # if sign is positive, truncate b 2f 1: bne t0, zero, 5f # if sign is negative, truncate 2: beq t3, zero, 5f # if no fraction bits, continue addu t2, t2, 1 # add rounding bit blt t2, zero, overflow_w # overflow? b 5f 3: li v0, GUARDBIT # load guard bit for rounding addu v0, v0, t3 # add remainder sltu v1, v0, t3 # compute carry out beq v1, zero, 4f # if no carry, continue addu t2, t2, 1 # add carry to result blt t2, zero, overflow_w # overflow? 4: bne v0, zero, 5f # if rounded remainder is zero and t2, t2, ~1 # clear LSB (round to nearest) 5: beq t0, zero, 1f # result positive? negu t2 # convert to negative integer 1: beq t3, zero, result_fs_w # is result exact? /* * Handle inexact exception. */ inexact_w: or a1, a1, MIPS_FPU_EXCEPTION_INEXACT | MIPS_FPU_STICKY_INEXACT and v0, a1, MIPS_FPU_ENABLE_INEXACT bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions b result_fs_w /* * Conversions to integer which overflow will trap (if enabled), * or generate an inexact trap (if enabled), * or generate an invalid exception. */ overflow_w: or a1, a1, MIPS_FPU_EXCEPTION_OVERFLOW | MIPS_FPU_STICKY_OVERFLOW and v0, a1, MIPS_FPU_ENABLE_OVERFLOW bne v0, zero, fpe_trap and v0, a1, MIPS_FPU_ENABLE_INEXACT bne v0, zero, inexact_w # inexact traps enabled? b invalid_w /* * Conversions to integer which underflow will trap (if enabled), * or generate an inexact trap (if enabled), * or generate an invalid exception. */ underflow_w: or a1, a1, MIPS_FPU_EXCEPTION_UNDERFLOW | MIPS_FPU_STICKY_UNDERFLOW and v0, a1, MIPS_FPU_ENABLE_UNDERFLOW bne v0, zero, fpe_trap and v0, a1, MIPS_FPU_ENABLE_INEXACT bne v0, zero, inexact_w # inexact traps enabled? b invalid_w /* * Compare single. */ cmp_s: jal get_cmp_s bne t1, SEXP_INF, 1f # is FS an infinity? bne t2, zero, unordered # FS is a NAN 1: bne ta1, SEXP_INF, 2f # is FT an infinity? bne ta2, zero, unordered # FT is a NAN 2: sll t1, t1, 23 # reassemble exp & frac or t1, t1, t2 sll ta1, ta1, 23 # reassemble exp & frac or ta1, ta1, ta2 beq t0, zero, 1f # is FS positive? negu t1 1: beq ta0, zero, 1f # is FT positive? negu ta1 1: li v0, COND_LESS blt t1, ta1, test_cond # is FS < FT? li v0, COND_EQUAL beq t1, ta1, test_cond # is FS == FT? move v0, zero # FS > FT b test_cond /* * Compare double. */ cmp_d: jal get_cmp_d bne t1, DEXP_INF, 1f # is FS an infinity? bne t2, zero, unordered bne t3, zero, unordered # FS is a NAN 1: bne ta1, DEXP_INF, 2f # is FT an infinity? bne ta2, zero, unordered bne ta3, zero, unordered # FT is a NAN 2: sll t1, t1, 20 # reassemble exp & frac or t1, t1, t2 sll ta1, ta1, 20 # reassemble exp & frac or ta1, ta1, ta2 beq t0, zero, 1f # is FS positive? not t3 # negate t1,t3 not t1 addu t3, t3, 1 seq v0, t3, zero # compute carry addu t1, t1, v0 1: beq ta0, zero, 1f # is FT positive? not ta3 # negate ta1,ta3 not ta1 addu ta3, ta3, 1 seq v0, ta3, zero # compute carry addu ta1, ta1, v0 1: li v0, COND_LESS blt t1, ta1, test_cond # is FS(MSW) < FT(MSW)? move v0, zero bne t1, ta1, test_cond # is FS(MSW) > FT(MSW)? li v0, COND_LESS bltu t3, ta3, test_cond # is FS(LSW) < FT(LSW)? li v0, COND_EQUAL beq t3, ta3, test_cond # is FS(LSW) == FT(LSW)? move v0, zero # FS > FT test_cond: and v0, v0, a0 # condition match instruction? set_cond: bne v0, zero, 1f and a1, a1, ~MIPS_FPU_COND_BIT # clear condition bit b 2f 1: or a1, a1, MIPS_FPU_COND_BIT # set condition bit 2: ctc1 a1, MIPS_FPU_CSR # save condition bit b done unordered: and v0, a0, COND_UNORDERED # this cmp match unordered? bne v0, zero, 1f and a1, a1, ~MIPS_FPU_COND_BIT # clear condition bit b 2f 1: or a1, a1, MIPS_FPU_COND_BIT # set condition bit 2: and v0, a0, COND_SIGNAL beq v0, zero, 1f # is this a signaling cmp? or a1, a1, MIPS_FPU_EXCEPTION_INVALID | MIPS_FPU_STICKY_INVALID and v0, a1, MIPS_FPU_ENABLE_INVALID bne v0, zero, fpe_trap 1: ctc1 a1, MIPS_FPU_CSR # save condition bit b done /* * Determine the amount to shift the fraction in order to restore the * normalized position. After that, round and handle exceptions. */ norm_s: move v0, t2 move t9, zero # t9 = num of leading zeros bne t2, zero, 1f move v0, t8 addu t9, 32 1: srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift t2,t8 the correct number of bits. */ 1: subu t9, t9, SLEAD_ZEROS # dont count leading zeros subu t1, t1, t9 # adjust the exponent beq t9, zero, norm_noshift_s li v1, 32 blt t9, zero, 1f # if shift < 0, shift right subu v1, v1, t9 sll t2, t2, t9 # shift t2,t8 left srl v0, t8, v1 # save bits shifted out or t2, t2, v0 sll t8, t8, t9 b norm_noshift_s 1: negu t9 # shift t2,t8 right by t9 subu v1, v1, t9 sll v0, t8, v1 # save bits shifted out sltu v0, zero, v0 # be sure to save any one bits srl t8, t8, t9 or t8, t8, v0 sll v0, t2, v1 # save bits shifted out or t8, t8, v0 srl t2, t2, t9 norm_noshift_s: move ta1, t1 # save unrounded exponent move ta2, t2 # save unrounded fraction and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, 3f # round to nearest beq v0, MIPS_FPU_ROUND_RZ, 5f # round to zero (truncate) beq v0, MIPS_FPU_ROUND_RP, 1f # round to +infinity beq t0, zero, 5f # if sign is positive, truncate b 2f 1: bne t0, zero, 5f # if sign is negative, truncate 2: beq t8, zero, 5f # if exact, continue addu t2, t2, 1 # add rounding bit bne t2, SIMPL_ONE<<1, 5f # need to adjust exponent? addu t1, t1, 1 # adjust exponent srl t2, t2, 1 # renormalize fraction b 5f 3: li v0, GUARDBIT # load guard bit for rounding addu v0, v0, t8 # add remainder sltu v1, v0, t8 # compute carry out beq v1, zero, 4f # if no carry, continue addu t2, t2, 1 # add carry to result bne t2, SIMPL_ONE<<1, 4f # need to adjust exponent? addu t1, t1, 1 # adjust exponent srl t2, t2, 1 # renormalize fraction 4: bne v0, zero, 5f # if rounded remainder is zero and t2, t2, ~1 # clear LSB (round to nearest) 5: bgt t1, SEXP_MAX, overflow_s # overflow? blt t1, SEXP_MIN, underflow_s # underflow? bne t8, zero, inexact_s # is result inexact? addu t1, t1, SEXP_BIAS # bias exponent and t2, t2, ~SIMPL_ONE # clear implied one bit b result_fs_s /* * Handle inexact exception. */ inexact_s: addu t1, t1, SEXP_BIAS # bias exponent and t2, t2, ~SIMPL_ONE # clear implied one bit inexact_nobias_s: jal set_fd_s # save result or a1, a1, MIPS_FPU_EXCEPTION_INEXACT | MIPS_FPU_STICKY_INEXACT and v0, a1, MIPS_FPU_ENABLE_INEXACT bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions b done /* * Overflow will trap (if enabled), * or generate an inexact trap (if enabled), * or generate an infinity. */ overflow_s: or a1, a1, MIPS_FPU_EXCEPTION_OVERFLOW | MIPS_FPU_STICKY_OVERFLOW and v0, a1, MIPS_FPU_ENABLE_OVERFLOW beq v0, zero, 1f subu t1, t1, 192 # bias exponent and t2, t2, ~SIMPL_ONE # clear implied one bit jal set_fd_s # save result b fpe_trap 1: and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, 3f # round to nearest beq v0, MIPS_FPU_ROUND_RZ, 1f # round to zero (truncate) beq v0, MIPS_FPU_ROUND_RP, 2f # round to +infinity bne t0, zero, 3f 1: li t1, SEXP_MAX # result is max finite li t2, 0x007fffff b inexact_s 2: bne t0, zero, 1b 3: li t1, SEXP_MAX + 1 # result is infinity move t2, zero b inexact_s /* * In this implementation, "tininess" is detected "after rounding" and * "loss of accuracy" is detected as "an inexact result". */ underflow_s: and v0, a1, MIPS_FPU_ENABLE_UNDERFLOW beq v0, zero, 1f /* * Underflow is enabled so compute the result and trap. */ addu t1, t1, 192 # bias exponent and t2, t2, ~SIMPL_ONE # clear implied one bit jal set_fd_s # save result or a1, a1, MIPS_FPU_EXCEPTION_UNDERFLOW | MIPS_FPU_STICKY_UNDERFLOW b fpe_trap /* * Underflow is not enabled so compute the result, * signal inexact result (if it is) and trap (if enabled). */ 1: move t1, ta1 # get unrounded exponent move t2, ta2 # get unrounded fraction li t9, SEXP_MIN # compute shift amount subu t9, t9, t1 # shift t2,t8 right by t9 blt t9, SFRAC_BITS+2, 3f # shift all the bits out? move t1, zero # result is inexact zero move t2, zero or a1, a1, MIPS_FPU_EXCEPTION_UNDERFLOW | MIPS_FPU_STICKY_UNDERFLOW /* * Now round the zero result. * Only need to worry about rounding to +- infinity when the sign matches. */ and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, inexact_nobias_s # round to nearest beq v0, MIPS_FPU_ROUND_RZ, inexact_nobias_s # round to zero beq v0, MIPS_FPU_ROUND_RP, 1f # round to +infinity beq t0, zero, inexact_nobias_s # if sign is positive, truncate b 2f 1: bne t0, zero, inexact_nobias_s # if sign is negative, truncate 2: addu t2, t2, 1 # add rounding bit b inexact_nobias_s 3: li v1, 32 subu v1, v1, t9 sltu v0, zero, t8 # be sure to save any one bits sll t8, t2, v1 # save bits shifted out or t8, t8, v0 # include sticky bits srl t2, t2, t9 /* * Now round the denormalized result. */ and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, 3f # round to nearest beq v0, MIPS_FPU_ROUND_RZ, 5f # round to zero (truncate) beq v0, MIPS_FPU_ROUND_RP, 1f # round to +infinity beq t0, zero, 5f # if sign is positive, truncate b 2f 1: bne t0, zero, 5f # if sign is negative, truncate 2: beq t8, zero, 5f # if exact, continue addu t2, t2, 1 # add rounding bit b 5f 3: li v0, GUARDBIT # load guard bit for rounding addu v0, v0, t8 # add remainder sltu v1, v0, t8 # compute carry out beq v1, zero, 4f # if no carry, continue addu t2, t2, 1 # add carry to result 4: bne v0, zero, 5f # if rounded remainder is zero and t2, t2, ~1 # clear LSB (round to nearest) 5: move t1, zero # denorm or zero exponent jal set_fd_s # save result beq t8, zero, done # check for exact result or a1, a1, MIPS_FPU_EXCEPTION_UNDERFLOW | MIPS_FPU_STICKY_UNDERFLOW or a1, a1, MIPS_FPU_EXCEPTION_INEXACT | MIPS_FPU_STICKY_INEXACT and v0, a1, MIPS_FPU_ENABLE_INEXACT bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions b done /* * Determine the amount to shift the fraction in order to restore the * normalized position. After that, round and handle exceptions. */ norm_d: move v0, t2 move t9, zero # t9 = num of leading zeros bne t2, zero, 1f move v0, t3 addu t9, 32 bne t3, zero, 1f move v0, t8 addu t9, 32 1: srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift t2,t3,t8 the correct number of bits. */ 1: subu t9, t9, DLEAD_ZEROS # dont count leading zeros subu t1, t1, t9 # adjust the exponent beq t9, zero, norm_noshift_d li v1, 32 blt t9, zero, 2f # if shift < 0, shift right blt t9, v1, 1f # shift by < 32? subu t9, t9, v1 # shift by >= 32 subu v1, v1, t9 sll t2, t3, t9 # shift left by t9 srl v0, t8, v1 # save bits shifted out or t2, t2, v0 sll t3, t8, t9 move t8, zero b norm_noshift_d 1: subu v1, v1, t9 sll t2, t2, t9 # shift left by t9 srl v0, t3, v1 # save bits shifted out or t2, t2, v0 sll t3, t3, t9 srl v0, t8, v1 # save bits shifted out or t3, t3, v0 sll t8, t8, t9 b norm_noshift_d 2: negu t9 # shift right by t9 subu v1, v1, t9 # (known to be < 32 bits) sll v0, t8, v1 # save bits shifted out sltu v0, zero, v0 # be sure to save any one bits srl t8, t8, t9 or t8, t8, v0 sll v0, t3, v1 # save bits shifted out or t8, t8, v0 srl t3, t3, t9 sll v0, t2, v1 # save bits shifted out or t3, t3, v0 srl t2, t2, t9 norm_noshift_d: move ta1, t1 # save unrounded exponent move ta2, t2 # save unrounded fraction (MS) move ta3, t3 # save unrounded fraction (LS) and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, 3f # round to nearest beq v0, MIPS_FPU_ROUND_RZ, 5f # round to zero (truncate) beq v0, MIPS_FPU_ROUND_RP, 1f # round to +infinity beq t0, zero, 5f # if sign is positive, truncate b 2f 1: bne t0, zero, 5f # if sign is negative, truncate 2: beq t8, zero, 5f # if exact, continue addu t3, t3, 1 # add rounding bit bne t3, zero, 5f # branch if no carry addu t2, t2, 1 # add carry bne t2, DIMPL_ONE<<1, 5f # need to adjust exponent? addu t1, t1, 1 # adjust exponent srl t2, t2, 1 # renormalize fraction b 5f 3: li v0, GUARDBIT # load guard bit for rounding addu v0, v0, t8 # add remainder sltu v1, v0, t8 # compute carry out beq v1, zero, 4f # branch if no carry addu t3, t3, 1 # add carry bne t3, zero, 4f # branch if no carry addu t2, t2, 1 # add carry to result bne t2, DIMPL_ONE<<1, 4f # need to adjust exponent? addu t1, t1, 1 # adjust exponent srl t2, t2, 1 # renormalize fraction 4: bne v0, zero, 5f # if rounded remainder is zero and t3, t3, ~1 # clear LSB (round to nearest) 5: bgt t1, DEXP_MAX, overflow_d # overflow? blt t1, DEXP_MIN, underflow_d # underflow? bne t8, zero, inexact_d # is result inexact? addu t1, t1, DEXP_BIAS # bias exponent and t2, t2, ~DIMPL_ONE # clear implied one bit b result_fs_d /* * Handle inexact exception. */ inexact_d: addu t1, t1, DEXP_BIAS # bias exponent and t2, t2, ~DIMPL_ONE # clear implied one bit inexact_nobias_d: jal set_fd_d # save result or a1, a1, MIPS_FPU_EXCEPTION_INEXACT | MIPS_FPU_STICKY_INEXACT and v0, a1, MIPS_FPU_ENABLE_INEXACT bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions b done /* * Overflow will trap (if enabled), * or generate an inexact trap (if enabled), * or generate an infinity. */ overflow_d: or a1, a1, MIPS_FPU_EXCEPTION_OVERFLOW | MIPS_FPU_STICKY_OVERFLOW and v0, a1, MIPS_FPU_ENABLE_OVERFLOW beq v0, zero, 1f subu t1, t1, 1536 # bias exponent and t2, t2, ~DIMPL_ONE # clear implied one bit jal set_fd_d # save result b fpe_trap 1: and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, 3f # round to nearest beq v0, MIPS_FPU_ROUND_RZ, 1f # round to zero (truncate) beq v0, MIPS_FPU_ROUND_RP, 2f # round to +infinity bne t0, zero, 3f 1: li t1, DEXP_MAX # result is max finite li t2, 0x000fffff li t3, 0xffffffff b inexact_d 2: bne t0, zero, 1b 3: li t1, DEXP_MAX + 1 # result is infinity move t2, zero move t3, zero b inexact_d /* * In this implementation, "tininess" is detected "after rounding" and * "loss of accuracy" is detected as "an inexact result". */ underflow_d: and v0, a1, MIPS_FPU_ENABLE_UNDERFLOW beq v0, zero, 1f /* * Underflow is enabled so compute the result and trap. */ addu t1, t1, 1536 # bias exponent and t2, t2, ~DIMPL_ONE # clear implied one bit jal set_fd_d # save result or a1, a1, MIPS_FPU_EXCEPTION_UNDERFLOW | MIPS_FPU_STICKY_UNDERFLOW b fpe_trap /* * Underflow is not enabled so compute the result, * signal inexact result (if it is) and trap (if enabled). */ 1: move t1, ta1 # get unrounded exponent move t2, ta2 # get unrounded fraction (MS) move t3, ta3 # get unrounded fraction (LS) li t9, DEXP_MIN # compute shift amount subu t9, t9, t1 # shift t2,t8 right by t9 blt t9, DFRAC_BITS+2, 3f # shift all the bits out? move t1, zero # result is inexact zero move t2, zero move t3, zero or a1, a1, MIPS_FPU_EXCEPTION_UNDERFLOW | MIPS_FPU_STICKY_UNDERFLOW /* * Now round the zero result. * Only need to worry about rounding to +- infinity when the sign matches. */ and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, inexact_nobias_d # round to nearest beq v0, MIPS_FPU_ROUND_RZ, inexact_nobias_d # round to zero beq v0, MIPS_FPU_ROUND_RP, 1f # round to +infinity beq t0, zero, inexact_nobias_d # if sign is positive, truncate b 2f 1: bne t0, zero, inexact_nobias_d # if sign is negative, truncate 2: addu t3, t3, 1 # add rounding bit b inexact_nobias_d 3: li v1, 32 blt t9, v1, 1f # shift by < 32? subu t9, t9, v1 # shift right by >= 32 subu v1, v1, t9 sltu v0, zero, t8 # be sure to save any one bits sll t8, t2, v1 # save bits shifted out or t8, t8, v0 # include sticky bits srl t3, t2, t9 move t2, zero b 2f 1: subu v1, v1, t9 # shift right by t9 sltu v0, zero, t8 # be sure to save any one bits sll t8, t3, v1 # save bits shifted out or t8, t8, v0 # include sticky bits srl t3, t3, t9 sll v0, t2, v1 # save bits shifted out or t3, t3, v0 srl t2, t2, t9 /* * Now round the denormalized result. */ 2: and v0, a1, MIPS_FPU_ROUNDING_BITS # get rounding mode beq v0, MIPS_FPU_ROUND_RN, 3f # round to nearest beq v0, MIPS_FPU_ROUND_RZ, 5f # round to zero (truncate) beq v0, MIPS_FPU_ROUND_RP, 1f # round to +infinity beq t0, zero, 5f # if sign is positive, truncate b 2f 1: bne t0, zero, 5f # if sign is negative, truncate 2: beq t8, zero, 5f # if exact, continue addu t3, t3, 1 # add rounding bit bne t3, zero, 5f # if no carry, continue addu t2, t2, 1 # add carry b 5f 3: li v0, GUARDBIT # load guard bit for rounding addu v0, v0, t8 # add remainder sltu v1, v0, t8 # compute carry out beq v1, zero, 4f # if no carry, continue addu t3, t3, 1 # add rounding bit bne t3, zero, 4f # if no carry, continue addu t2, t2, 1 # add carry 4: bne v0, zero, 5f # if rounded remainder is zero and t3, t3, ~1 # clear LSB (round to nearest) 5: move t1, zero # denorm or zero exponent jal set_fd_d # save result beq t8, zero, done # check for exact result or a1, a1, MIPS_FPU_EXCEPTION_UNDERFLOW | MIPS_FPU_STICKY_UNDERFLOW or a1, a1, MIPS_FPU_EXCEPTION_INEXACT | MIPS_FPU_STICKY_INEXACT and v0, a1, MIPS_FPU_ENABLE_INEXACT bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions b done /* * Signal an invalid operation if the trap is enabled; otherwise, * the result is a quiet NAN. */ invalid_s: # trap invalid operation or a1, a1, MIPS_FPU_EXCEPTION_INVALID | MIPS_FPU_STICKY_INVALID and v0, a1, MIPS_FPU_ENABLE_INVALID bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions move t0, zero # result is a quiet NAN li t1, SEXP_INF li t2, SQUIET_NAN jal set_fd_s # save result (in t0,t1,t2) b done /* * Signal an invalid operation if the trap is enabled; otherwise, * the result is a quiet NAN. */ invalid_d: # trap invalid operation or a1, a1, MIPS_FPU_EXCEPTION_INVALID | MIPS_FPU_STICKY_INVALID and v0, a1, MIPS_FPU_ENABLE_INVALID bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions move t0, zero # result is a quiet NAN li t1, DEXP_INF li t2, DQUIET_NAN0 li t3, DQUIET_NAN1 jal set_fd_d # save result (in t0,t1,t2,t3) b done /* * Signal an invalid operation if the trap is enabled; otherwise, * the result is INT_MAX or INT_MIN. */ invalid_w: # trap invalid operation or a1, a1, MIPS_FPU_EXCEPTION_INVALID | MIPS_FPU_STICKY_INVALID and v0, a1, MIPS_FPU_ENABLE_INVALID bne v0, zero, fpe_trap ctc1 a1, MIPS_FPU_CSR # save exceptions bne t0, zero, 1f li t2, INT_MAX # result is INT_MAX b result_fs_w 1: li t2, INT_MIN # result is INT_MIN b result_fs_w /* * Trap if the hardware should have handled this case. */ fpe_trap: move a2, a1 # code = FP CSR ctc1 a1, MIPS_FPU_CSR # save exceptions break 0 /* * Send an illegal instruction signal to the current process. */ ill: ctc1 a1, MIPS_FPU_CSR # save exceptions move a2, a0 # code = FP instruction break 0 result_ft_s: move t0, ta0 # result is FT move t1, ta1 move t2, ta2 result_fs_s: # result is FS jal set_fd_s # save result (in t0,t1,t2) b done result_fs_w: jal set_fd_word # save result (in t2) b done result_ft_d: move t0, ta0 # result is FT move t1, ta1 move t2, ta2 move t3, ta3 result_fs_d: # result is FS jal set_fd_d # save result (in t0,t1,t2,t3) done: lw ra, CALLFRAME_RA(sp) addu sp, sp, CALLFRAME_SIZ j ra END(MipsEmulateFP) /*---------------------------------------------------------------------------- * get_fs_int -- * * Read (integer) the FS register (bits 15-11). * This is an internal routine used by MipsEmulateFP only. * * Results: * t0 contains the sign * t2 contains the fraction * *---------------------------------------------------------------------------- */ LEAF(get_fs_int) srl a3, a0, 12 - 2 # get FS field (even regs only) and a3, a3, 0xF << 2 # mask FS field lw a3, get_fs_int_tbl(a3) # switch on register number j a3 .rdata get_fs_int_tbl: .word get_fs_int_f0 .word get_fs_int_f2 .word get_fs_int_f4 .word get_fs_int_f6 .word get_fs_int_f8 .word get_fs_int_f10 .word get_fs_int_f12 .word get_fs_int_f14 .word get_fs_int_f16 .word get_fs_int_f18 .word get_fs_int_f20 .word get_fs_int_f22 .word get_fs_int_f24 .word get_fs_int_f26 .word get_fs_int_f28 .word get_fs_int_f30 .text get_fs_int_f0: mfc1 t2, $f0 b get_fs_int_done get_fs_int_f2: mfc1 t2, $f2 b get_fs_int_done get_fs_int_f4: mfc1 t2, $f4 b get_fs_int_done get_fs_int_f6: mfc1 t2, $f6 b get_fs_int_done get_fs_int_f8: mfc1 t2, $f8 b get_fs_int_done get_fs_int_f10: mfc1 t2, $f10 b get_fs_int_done get_fs_int_f12: mfc1 t2, $f12 b get_fs_int_done get_fs_int_f14: mfc1 t2, $f14 b get_fs_int_done get_fs_int_f16: mfc1 t2, $f16 b get_fs_int_done get_fs_int_f18: mfc1 t2, $f18 b get_fs_int_done get_fs_int_f20: mfc1 t2, $f20 b get_fs_int_done get_fs_int_f22: mfc1 t2, $f22 b get_fs_int_done get_fs_int_f24: mfc1 t2, $f24 b get_fs_int_done get_fs_int_f26: mfc1 t2, $f26 b get_fs_int_done get_fs_int_f28: mfc1 t2, $f28 b get_fs_int_done get_fs_int_f30: mfc1 t2, $f30 get_fs_int_done: srl t0, t2, 31 # init the sign bit bge t2, zero, 1f negu t2 1: j ra END(get_fs_int) /*---------------------------------------------------------------------------- * get_ft_fs_s -- * * Read (single precision) the FT register (bits 20-16) and * the FS register (bits 15-11) and break up into fields. * This is an internal routine used by MipsEmulateFP only. * * Results: * t0 contains the FS sign * t1 contains the FS (biased) exponent * t2 contains the FS fraction * ta0 contains the FT sign * ta1 contains the FT (biased) exponent * ta2 contains the FT fraction * *---------------------------------------------------------------------------- */ LEAF(get_ft_fs_s) srl a3, a0, 17 - 2 # get FT field (even regs only) and a3, a3, 0xF << 2 # mask FT field lw a3, get_ft_s_tbl(a3) # switch on register number j a3 .rdata get_ft_s_tbl: .word get_ft_s_f0 .word get_ft_s_f2 .word get_ft_s_f4 .word get_ft_s_f6 .word get_ft_s_f8 .word get_ft_s_f10 .word get_ft_s_f12 .word get_ft_s_f14 .word get_ft_s_f16 .word get_ft_s_f18 .word get_ft_s_f20 .word get_ft_s_f22 .word get_ft_s_f24 .word get_ft_s_f26 .word get_ft_s_f28 .word get_ft_s_f30 .text get_ft_s_f0: mfc1 ta0, $f0 b get_ft_s_done get_ft_s_f2: mfc1 ta0, $f2 b get_ft_s_done get_ft_s_f4: mfc1 ta0, $f4 b get_ft_s_done get_ft_s_f6: mfc1 ta0, $f6 b get_ft_s_done get_ft_s_f8: mfc1 ta0, $f8 b get_ft_s_done get_ft_s_f10: mfc1 ta0, $f10 b get_ft_s_done get_ft_s_f12: mfc1 ta0, $f12 b get_ft_s_done get_ft_s_f14: mfc1 ta0, $f14 b get_ft_s_done get_ft_s_f16: mfc1 ta0, $f16 b get_ft_s_done get_ft_s_f18: mfc1 ta0, $f18 b get_ft_s_done get_ft_s_f20: mfc1 ta0, $f20 b get_ft_s_done get_ft_s_f22: mfc1 ta0, $f22 b get_ft_s_done get_ft_s_f24: mfc1 ta0, $f24 b get_ft_s_done get_ft_s_f26: mfc1 ta0, $f26 b get_ft_s_done get_ft_s_f28: mfc1 ta0, $f28 b get_ft_s_done get_ft_s_f30: mfc1 ta0, $f30 get_ft_s_done: srl ta1, ta0, 23 # get exponent and ta1, ta1, 0xFF and ta2, ta0, 0x7FFFFF # get fraction srl ta0, ta0, 31 # get sign bne ta1, SEXP_INF, 1f # is it a signaling NAN? and v0, ta2, SSIGNAL_NAN bne v0, zero, invalid_s 1: /* fall through to get FS */ /*---------------------------------------------------------------------------- * get_fs_s -- * * Read (single precision) the FS register (bits 15-11) and * break up into fields. * This is an internal routine used by MipsEmulateFP only. * * Results: * t0 contains the sign * t1 contains the (biased) exponent * t2 contains the fraction * *---------------------------------------------------------------------------- */ XLEAF(get_fs_s) srl a3, a0, 12 - 2 # get FS field (even regs only) and a3, a3, 0xF << 2 # mask FS field lw a3, get_fs_s_tbl(a3) # switch on register number j a3 .rdata get_fs_s_tbl: .word get_fs_s_f0 .word get_fs_s_f2 .word get_fs_s_f4 .word get_fs_s_f6 .word get_fs_s_f8 .word get_fs_s_f10 .word get_fs_s_f12 .word get_fs_s_f14 .word get_fs_s_f16 .word get_fs_s_f18 .word get_fs_s_f20 .word get_fs_s_f22 .word get_fs_s_f24 .word get_fs_s_f26 .word get_fs_s_f28 .word get_fs_s_f30 .text get_fs_s_f0: mfc1 t0, $f0 b get_fs_s_done get_fs_s_f2: mfc1 t0, $f2 b get_fs_s_done get_fs_s_f4: mfc1 t0, $f4 b get_fs_s_done get_fs_s_f6: mfc1 t0, $f6 b get_fs_s_done get_fs_s_f8: mfc1 t0, $f8 b get_fs_s_done get_fs_s_f10: mfc1 t0, $f10 b get_fs_s_done get_fs_s_f12: mfc1 t0, $f12 b get_fs_s_done get_fs_s_f14: mfc1 t0, $f14 b get_fs_s_done get_fs_s_f16: mfc1 t0, $f16 b get_fs_s_done get_fs_s_f18: mfc1 t0, $f18 b get_fs_s_done get_fs_s_f20: mfc1 t0, $f20 b get_fs_s_done get_fs_s_f22: mfc1 t0, $f22 b get_fs_s_done get_fs_s_f24: mfc1 t0, $f24 b get_fs_s_done get_fs_s_f26: mfc1 t0, $f26 b get_fs_s_done get_fs_s_f28: mfc1 t0, $f28 b get_fs_s_done get_fs_s_f30: mfc1 t0, $f30 get_fs_s_done: srl t1, t0, 23 # get exponent and t1, t1, 0xFF and t2, t0, 0x7FFFFF # get fraction srl t0, t0, 31 # get sign bne t1, SEXP_INF, 1f # is it a signaling NAN? and v0, t2, SSIGNAL_NAN bne v0, zero, invalid_s 1: j ra END(get_ft_fs_s) /*---------------------------------------------------------------------------- * get_ft_fs_d -- * * Read (double precision) the FT register (bits 20-16) and * the FS register (bits 15-11) and break up into fields. * This is an internal routine used by MipsEmulateFP only. * * Results: * t0 contains the FS sign * t1 contains the FS (biased) exponent * t2 contains the FS fraction * t3 contains the FS remaining fraction * ta0 contains the FT sign * ta1 contains the FT (biased) exponent * ta2 contains the FT fraction * ta3 contains the FT remaining fraction * *---------------------------------------------------------------------------- */ LEAF(get_ft_fs_d) srl a3, a0, 17 - 2 # get FT field (even regs only) and a3, a3, 0xF << 2 # mask FT field lw a3, get_ft_d_tbl(a3) # switch on register number j a3 .rdata get_ft_d_tbl: .word get_ft_d_f0 .word get_ft_d_f2 .word get_ft_d_f4 .word get_ft_d_f6 .word get_ft_d_f8 .word get_ft_d_f10 .word get_ft_d_f12 .word get_ft_d_f14 .word get_ft_d_f16 .word get_ft_d_f18 .word get_ft_d_f20 .word get_ft_d_f22 .word get_ft_d_f24 .word get_ft_d_f26 .word get_ft_d_f28 .word get_ft_d_f30 .text get_ft_d_f0: mfc1 ta3, $f0 mfc1 ta0, $f1 b get_ft_d_done get_ft_d_f2: mfc1 ta3, $f2 mfc1 ta0, $f3 b get_ft_d_done get_ft_d_f4: mfc1 ta3, $f4 mfc1 ta0, $f5 b get_ft_d_done get_ft_d_f6: mfc1 ta3, $f6 mfc1 ta0, $f7 b get_ft_d_done get_ft_d_f8: mfc1 ta3, $f8 mfc1 ta0, $f9 b get_ft_d_done get_ft_d_f10: mfc1 ta3, $f10 mfc1 ta0, $f11 b get_ft_d_done get_ft_d_f12: mfc1 ta3, $f12 mfc1 ta0, $f13 b get_ft_d_done get_ft_d_f14: mfc1 ta3, $f14 mfc1 ta0, $f15 b get_ft_d_done get_ft_d_f16: mfc1 ta3, $f16 mfc1 ta0, $f17 b get_ft_d_done get_ft_d_f18: mfc1 ta3, $f18 mfc1 ta0, $f19 b get_ft_d_done get_ft_d_f20: mfc1 ta3, $f20 mfc1 ta0, $f21 b get_ft_d_done get_ft_d_f22: mfc1 ta3, $f22 mfc1 ta0, $f23 b get_ft_d_done get_ft_d_f24: mfc1 ta3, $f24 mfc1 ta0, $f25 b get_ft_d_done get_ft_d_f26: mfc1 ta3, $f26 mfc1 ta0, $f27 b get_ft_d_done get_ft_d_f28: mfc1 ta3, $f28 mfc1 ta0, $f29 b get_ft_d_done get_ft_d_f30: mfc1 ta3, $f30 mfc1 ta0, $f31 get_ft_d_done: srl ta1, ta0, 20 # get exponent and ta1, ta1, 0x7FF and ta2, ta0, 0xFFFFF # get fraction srl ta0, ta0, 31 # get sign bne ta1, DEXP_INF, 1f # is it a signaling NAN? and v0, ta2, DSIGNAL_NAN bne v0, zero, invalid_d 1: /* fall through to get FS */ /*---------------------------------------------------------------------------- * get_fs_d -- * * Read (double precision) the FS register (bits 15-11) and * break up into fields. * This is an internal routine used by MipsEmulateFP only. * * Results: * t0 contains the sign * t1 contains the (biased) exponent * t2 contains the fraction * t3 contains the remaining fraction * *---------------------------------------------------------------------------- */ XLEAF(get_fs_d) srl a3, a0, 12 - 2 # get FS field (even regs only) and a3, a3, 0xF << 2 # mask FS field lw a3, get_fs_d_tbl(a3) # switch on register number j a3 .rdata get_fs_d_tbl: .word get_fs_d_f0 .word get_fs_d_f2 .word get_fs_d_f4 .word get_fs_d_f6 .word get_fs_d_f8 .word get_fs_d_f10 .word get_fs_d_f12 .word get_fs_d_f14 .word get_fs_d_f16 .word get_fs_d_f18 .word get_fs_d_f20 .word get_fs_d_f22 .word get_fs_d_f24 .word get_fs_d_f26 .word get_fs_d_f28 .word get_fs_d_f30 .text get_fs_d_f0: mfc1 t3, $f0 mfc1 t0, $f1 b get_fs_d_done get_fs_d_f2: mfc1 t3, $f2 mfc1 t0, $f3 b get_fs_d_done get_fs_d_f4: mfc1 t3, $f4 mfc1 t0, $f5 b get_fs_d_done get_fs_d_f6: mfc1 t3, $f6 mfc1 t0, $f7 b get_fs_d_done get_fs_d_f8: mfc1 t3, $f8 mfc1 t0, $f9 b get_fs_d_done get_fs_d_f10: mfc1 t3, $f10 mfc1 t0, $f11 b get_fs_d_done get_fs_d_f12: mfc1 t3, $f12 mfc1 t0, $f13 b get_fs_d_done get_fs_d_f14: mfc1 t3, $f14 mfc1 t0, $f15 b get_fs_d_done get_fs_d_f16: mfc1 t3, $f16 mfc1 t0, $f17 b get_fs_d_done get_fs_d_f18: mfc1 t3, $f18 mfc1 t0, $f19 b get_fs_d_done get_fs_d_f20: mfc1 t3, $f20 mfc1 t0, $f21 b get_fs_d_done get_fs_d_f22: mfc1 t3, $f22 mfc1 t0, $f23 b get_fs_d_done get_fs_d_f24: mfc1 t3, $f24 mfc1 t0, $f25 b get_fs_d_done get_fs_d_f26: mfc1 t3, $f26 mfc1 t0, $f27 b get_fs_d_done get_fs_d_f28: mfc1 t3, $f28 mfc1 t0, $f29 b get_fs_d_done get_fs_d_f30: mfc1 t3, $f30 mfc1 t0, $f31 get_fs_d_done: srl t1, t0, 20 # get exponent and t1, t1, 0x7FF and t2, t0, 0xFFFFF # get fraction srl t0, t0, 31 # get sign bne t1, DEXP_INF, 1f # is it a signaling NAN? and v0, t2, DSIGNAL_NAN bne v0, zero, invalid_d 1: j ra END(get_ft_fs_d) /*---------------------------------------------------------------------------- * get_cmp_s -- * * Read (single precision) the FS register (bits 15-11) and * the FT register (bits 20-16) and break up into fields. * This is an internal routine used by MipsEmulateFP only. * * Results: * t0 contains the sign * t1 contains the (biased) exponent * t2 contains the fraction * ta0 contains the sign * ta1 contains the (biased) exponent * ta2 contains the fraction * *---------------------------------------------------------------------------- */ LEAF(get_cmp_s) srl a3, a0, 12 - 2 # get FS field (even regs only) and a3, a3, 0xF << 2 # mask FS field lw a3, cmp_fs_s_tbl(a3) # switch on register number j a3 .rdata cmp_fs_s_tbl: .word cmp_fs_s_f0 .word cmp_fs_s_f2 .word cmp_fs_s_f4 .word cmp_fs_s_f6 .word cmp_fs_s_f8 .word cmp_fs_s_f10 .word cmp_fs_s_f12 .word cmp_fs_s_f14 .word cmp_fs_s_f16 .word cmp_fs_s_f18 .word cmp_fs_s_f20 .word cmp_fs_s_f22 .word cmp_fs_s_f24 .word cmp_fs_s_f26 .word cmp_fs_s_f28 .word cmp_fs_s_f30 .text cmp_fs_s_f0: mfc1 t0, $f0 b cmp_fs_s_done cmp_fs_s_f2: mfc1 t0, $f2 b cmp_fs_s_done cmp_fs_s_f4: mfc1 t0, $f4 b cmp_fs_s_done cmp_fs_s_f6: mfc1 t0, $f6 b cmp_fs_s_done cmp_fs_s_f8: mfc1 t0, $f8 b cmp_fs_s_done cmp_fs_s_f10: mfc1 t0, $f10 b cmp_fs_s_done cmp_fs_s_f12: mfc1 t0, $f12 b cmp_fs_s_done cmp_fs_s_f14: mfc1 t0, $f14 b cmp_fs_s_done cmp_fs_s_f16: mfc1 t0, $f16 b cmp_fs_s_done cmp_fs_s_f18: mfc1 t0, $f18 b cmp_fs_s_done cmp_fs_s_f20: mfc1 t0, $f20 b cmp_fs_s_done cmp_fs_s_f22: mfc1 t0, $f22 b cmp_fs_s_done cmp_fs_s_f24: mfc1 t0, $f24 b cmp_fs_s_done cmp_fs_s_f26: mfc1 t0, $f26 b cmp_fs_s_done cmp_fs_s_f28: mfc1 t0, $f28 b cmp_fs_s_done cmp_fs_s_f30: mfc1 t0, $f30 cmp_fs_s_done: srl t1, t0, 23 # get exponent and t1, t1, 0xFF and t2, t0, 0x7FFFFF # get fraction srl t0, t0, 31 # get sign srl a3, a0, 17 - 2 # get FT field (even regs only) and a3, a3, 0xF << 2 # mask FT field lw a3, cmp_ft_s_tbl(a3) # switch on register number j a3 .rdata cmp_ft_s_tbl: .word cmp_ft_s_f0 .word cmp_ft_s_f2 .word cmp_ft_s_f4 .word cmp_ft_s_f6 .word cmp_ft_s_f8 .word cmp_ft_s_f10 .word cmp_ft_s_f12 .word cmp_ft_s_f14 .word cmp_ft_s_f16 .word cmp_ft_s_f18 .word cmp_ft_s_f20 .word cmp_ft_s_f22 .word cmp_ft_s_f24 .word cmp_ft_s_f26 .word cmp_ft_s_f28 .word cmp_ft_s_f30 .text cmp_ft_s_f0: mfc1 ta0, $f0 b cmp_ft_s_done cmp_ft_s_f2: mfc1 ta0, $f2 b cmp_ft_s_done cmp_ft_s_f4: mfc1 ta0, $f4 b cmp_ft_s_done cmp_ft_s_f6: mfc1 ta0, $f6 b cmp_ft_s_done cmp_ft_s_f8: mfc1 ta0, $f8 b cmp_ft_s_done cmp_ft_s_f10: mfc1 ta0, $f10 b cmp_ft_s_done cmp_ft_s_f12: mfc1 ta0, $f12 b cmp_ft_s_done cmp_ft_s_f14: mfc1 ta0, $f14 b cmp_ft_s_done cmp_ft_s_f16: mfc1 ta0, $f16 b cmp_ft_s_done cmp_ft_s_f18: mfc1 ta0, $f18 b cmp_ft_s_done cmp_ft_s_f20: mfc1 ta0, $f20 b cmp_ft_s_done cmp_ft_s_f22: mfc1 ta0, $f22 b cmp_ft_s_done cmp_ft_s_f24: mfc1 ta0, $f24 b cmp_ft_s_done cmp_ft_s_f26: mfc1 ta0, $f26 b cmp_ft_s_done cmp_ft_s_f28: mfc1 ta0, $f28 b cmp_ft_s_done cmp_ft_s_f30: mfc1 ta0, $f30 cmp_ft_s_done: srl ta1, ta0, 23 # get exponent and ta1, ta1, 0xFF and ta2, ta0, 0x7FFFFF # get fraction srl ta0, ta0, 31 # get sign j ra END(get_cmp_s) /*---------------------------------------------------------------------------- * get_cmp_d -- * * Read (double precision) the FS register (bits 15-11) and * the FT register (bits 20-16) and break up into fields. * This is an internal routine used by MipsEmulateFP only. * * Results: * t0 contains the sign * t1 contains the (biased) exponent * t2 contains the fraction * t3 contains the remaining fraction * ta0 contains the sign * ta1 contains the (biased) exponent * ta2 contains the fraction * ta3 contains the remaining fraction * *---------------------------------------------------------------------------- */ LEAF(get_cmp_d) srl a3, a0, 12 - 2 # get FS field (even regs only) and a3, a3, 0xF << 2 # mask FS field lw a3, cmp_fs_d_tbl(a3) # switch on register number j a3 .rdata cmp_fs_d_tbl: .word cmp_fs_d_f0 .word cmp_fs_d_f2 .word cmp_fs_d_f4 .word cmp_fs_d_f6 .word cmp_fs_d_f8 .word cmp_fs_d_f10 .word cmp_fs_d_f12 .word cmp_fs_d_f14 .word cmp_fs_d_f16 .word cmp_fs_d_f18 .word cmp_fs_d_f20 .word cmp_fs_d_f22 .word cmp_fs_d_f24 .word cmp_fs_d_f26 .word cmp_fs_d_f28 .word cmp_fs_d_f30 .text cmp_fs_d_f0: mfc1 t3, $f0 mfc1 t0, $f1 b cmp_fs_d_done cmp_fs_d_f2: mfc1 t3, $f2 mfc1 t0, $f3 b cmp_fs_d_done cmp_fs_d_f4: mfc1 t3, $f4 mfc1 t0, $f5 b cmp_fs_d_done cmp_fs_d_f6: mfc1 t3, $f6 mfc1 t0, $f7 b cmp_fs_d_done cmp_fs_d_f8: mfc1 t3, $f8 mfc1 t0, $f9 b cmp_fs_d_done cmp_fs_d_f10: mfc1 t3, $f10 mfc1 t0, $f11 b cmp_fs_d_done cmp_fs_d_f12: mfc1 t3, $f12 mfc1 t0, $f13 b cmp_fs_d_done cmp_fs_d_f14: mfc1 t3, $f14 mfc1 t0, $f15 b cmp_fs_d_done cmp_fs_d_f16: mfc1 t3, $f16 mfc1 t0, $f17 b cmp_fs_d_done cmp_fs_d_f18: mfc1 t3, $f18 mfc1 t0, $f19 b cmp_fs_d_done cmp_fs_d_f20: mfc1 t3, $f20 mfc1 t0, $f21 b cmp_fs_d_done cmp_fs_d_f22: mfc1 t3, $f22 mfc1 t0, $f23 b cmp_fs_d_done cmp_fs_d_f24: mfc1 t3, $f24 mfc1 t0, $f25 b cmp_fs_d_done cmp_fs_d_f26: mfc1 t3, $f26 mfc1 t0, $f27 b cmp_fs_d_done cmp_fs_d_f28: mfc1 t3, $f28 mfc1 t0, $f29 b cmp_fs_d_done cmp_fs_d_f30: mfc1 t3, $f30 mfc1 t0, $f31 cmp_fs_d_done: srl t1, t0, 20 # get exponent and t1, t1, 0x7FF and t2, t0, 0xFFFFF # get fraction srl t0, t0, 31 # get sign srl a3, a0, 17 - 2 # get FT field (even regs only) and a3, a3, 0xF << 2 # mask FT field lw a3, cmp_ft_d_tbl(a3) # switch on register number j a3 .rdata cmp_ft_d_tbl: .word cmp_ft_d_f0 .word cmp_ft_d_f2 .word cmp_ft_d_f4 .word cmp_ft_d_f6 .word cmp_ft_d_f8 .word cmp_ft_d_f10 .word cmp_ft_d_f12 .word cmp_ft_d_f14 .word cmp_ft_d_f16 .word cmp_ft_d_f18 .word cmp_ft_d_f20 .word cmp_ft_d_f22 .word cmp_ft_d_f24 .word cmp_ft_d_f26 .word cmp_ft_d_f28 .word cmp_ft_d_f30 .text cmp_ft_d_f0: mfc1 ta3, $f0 mfc1 ta0, $f1 b cmp_ft_d_done cmp_ft_d_f2: mfc1 ta3, $f2 mfc1 ta0, $f3 b cmp_ft_d_done cmp_ft_d_f4: mfc1 ta3, $f4 mfc1 ta0, $f5 b cmp_ft_d_done cmp_ft_d_f6: mfc1 ta3, $f6 mfc1 ta0, $f7 b cmp_ft_d_done cmp_ft_d_f8: mfc1 ta3, $f8 mfc1 ta0, $f9 b cmp_ft_d_done cmp_ft_d_f10: mfc1 ta3, $f10 mfc1 ta0, $f11 b cmp_ft_d_done cmp_ft_d_f12: mfc1 ta3, $f12 mfc1 ta0, $f13 b cmp_ft_d_done cmp_ft_d_f14: mfc1 ta3, $f14 mfc1 ta0, $f15 b cmp_ft_d_done cmp_ft_d_f16: mfc1 ta3, $f16 mfc1 ta0, $f17 b cmp_ft_d_done cmp_ft_d_f18: mfc1 ta3, $f18 mfc1 ta0, $f19 b cmp_ft_d_done cmp_ft_d_f20: mfc1 ta3, $f20 mfc1 ta0, $f21 b cmp_ft_d_done cmp_ft_d_f22: mfc1 ta3, $f22 mfc1 ta0, $f23 b cmp_ft_d_done cmp_ft_d_f24: mfc1 ta3, $f24 mfc1 ta0, $f25 b cmp_ft_d_done cmp_ft_d_f26: mfc1 ta3, $f26 mfc1 ta0, $f27 b cmp_ft_d_done cmp_ft_d_f28: mfc1 ta3, $f28 mfc1 ta0, $f29 b cmp_ft_d_done cmp_ft_d_f30: mfc1 ta3, $f30 mfc1 ta0, $f31 cmp_ft_d_done: srl ta1, ta0, 20 # get exponent and ta1, ta1, 0x7FF and ta2, ta0, 0xFFFFF # get fraction srl ta0, ta0, 31 # get sign j ra END(get_cmp_d) /*---------------------------------------------------------------------------- * set_fd_s -- * * Write (single precision) the FD register (bits 10-6). * This is an internal routine used by MipsEmulateFP only. * * Arguments: * a0 contains the FP instruction * t0 contains the sign * t1 contains the (biased) exponent * t2 contains the fraction * * set_fd_word -- * * Write (integer) the FD register (bits 10-6). * This is an internal routine used by MipsEmulateFP only. * * Arguments: * a0 contains the FP instruction * t2 contains the integer * *---------------------------------------------------------------------------- */ LEAF(set_fd_s) sll t0, t0, 31 # position sign sll t1, t1, 23 # position exponent or t2, t2, t0 or t2, t2, t1 XLEAF(set_fd_word) srl a3, a0, 7 - 2 # get FD field (even regs only) and a3, a3, 0xF << 2 # mask FT field lw a3, set_fd_s_tbl(a3) # switch on register number j a3 .rdata set_fd_s_tbl: .word set_fd_s_f0 .word set_fd_s_f2 .word set_fd_s_f4 .word set_fd_s_f6 .word set_fd_s_f8 .word set_fd_s_f10 .word set_fd_s_f12 .word set_fd_s_f14 .word set_fd_s_f16 .word set_fd_s_f18 .word set_fd_s_f20 .word set_fd_s_f22 .word set_fd_s_f24 .word set_fd_s_f26 .word set_fd_s_f28 .word set_fd_s_f30 .text set_fd_s_f0: mtc1 t2, $f0 j ra set_fd_s_f2: mtc1 t2, $f2 j ra set_fd_s_f4: mtc1 t2, $f4 j ra set_fd_s_f6: mtc1 t2, $f6 j ra set_fd_s_f8: mtc1 t2, $f8 j ra set_fd_s_f10: mtc1 t2, $f10 j ra set_fd_s_f12: mtc1 t2, $f12 j ra set_fd_s_f14: mtc1 t2, $f14 j ra set_fd_s_f16: mtc1 t2, $f16 j ra set_fd_s_f18: mtc1 t2, $f18 j ra set_fd_s_f20: mtc1 t2, $f20 j ra set_fd_s_f22: mtc1 t2, $f22 j ra set_fd_s_f24: mtc1 t2, $f24 j ra set_fd_s_f26: mtc1 t2, $f26 j ra set_fd_s_f28: mtc1 t2, $f28 j ra set_fd_s_f30: mtc1 t2, $f30 j ra END(set_fd_s) /*---------------------------------------------------------------------------- * set_fd_d -- * * Write (double precision) the FT register (bits 10-6). * This is an internal routine used by MipsEmulateFP only. * * Arguments: * a0 contains the FP instruction * t0 contains the sign * t1 contains the (biased) exponent * t2 contains the fraction * t3 contains the remaining fraction * *---------------------------------------------------------------------------- */ LEAF(set_fd_d) sll t0, t0, 31 # set sign sll t1, t1, 20 # set exponent or t0, t0, t1 or t0, t0, t2 # set fraction srl a3, a0, 7 - 2 # get FD field (even regs only) and a3, a3, 0xF << 2 # mask FD field lw a3, set_fd_d_tbl(a3) # switch on register number j a3 .rdata set_fd_d_tbl: .word set_fd_d_f0 .word set_fd_d_f2 .word set_fd_d_f4 .word set_fd_d_f6 .word set_fd_d_f8 .word set_fd_d_f10 .word set_fd_d_f12 .word set_fd_d_f14 .word set_fd_d_f16 .word set_fd_d_f18 .word set_fd_d_f20 .word set_fd_d_f22 .word set_fd_d_f24 .word set_fd_d_f26 .word set_fd_d_f28 .word set_fd_d_f30 .text set_fd_d_f0: mtc1 t3, $f0 mtc1 t0, $f1 j ra set_fd_d_f2: mtc1 t3, $f2 mtc1 t0, $f3 j ra set_fd_d_f4: mtc1 t3, $f4 mtc1 t0, $f5 j ra set_fd_d_f6: mtc1 t3, $f6 mtc1 t0, $f7 j ra set_fd_d_f8: mtc1 t3, $f8 mtc1 t0, $f9 j ra set_fd_d_f10: mtc1 t3, $f10 mtc1 t0, $f11 j ra set_fd_d_f12: mtc1 t3, $f12 mtc1 t0, $f13 j ra set_fd_d_f14: mtc1 t3, $f14 mtc1 t0, $f15 j ra set_fd_d_f16: mtc1 t3, $f16 mtc1 t0, $f17 j ra set_fd_d_f18: mtc1 t3, $f18 mtc1 t0, $f19 j ra set_fd_d_f20: mtc1 t3, $f20 mtc1 t0, $f21 j ra set_fd_d_f22: mtc1 t3, $f22 mtc1 t0, $f23 j ra set_fd_d_f24: mtc1 t3, $f24 mtc1 t0, $f25 j ra set_fd_d_f26: mtc1 t3, $f26 mtc1 t0, $f27 j ra set_fd_d_f28: mtc1 t3, $f28 mtc1 t0, $f29 j ra set_fd_d_f30: mtc1 t3, $f30 mtc1 t0, $f31 j ra END(set_fd_d) /*---------------------------------------------------------------------------- * renorm_fs_s -- * * Results: * t1 unbiased exponent * t2 normalized fraction * *---------------------------------------------------------------------------- */ LEAF(renorm_fs_s) /* * Find out how many leading zero bits are in t2 and put in t9. */ move v0, t2 move t9, zero srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift t2 the correct number of bits. */ 1: subu t9, t9, SLEAD_ZEROS # dont count normal leading zeros li t1, SEXP_MIN subu t1, t1, t9 # adjust exponent sll t2, t2, t9 j ra END(renorm_fs_s) /*---------------------------------------------------------------------------- * renorm_fs_d -- * * Results: * t1 unbiased exponent * t2,t3 normalized fraction * *---------------------------------------------------------------------------- */ LEAF(renorm_fs_d) /* * Find out how many leading zero bits are in t2,t3 and put in t9. */ move v0, t2 move t9, zero bne t2, zero, 1f move v0, t3 addu t9, 32 1: srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift t2,t3 the correct number of bits. */ 1: subu t9, t9, DLEAD_ZEROS # dont count normal leading zeros li t1, DEXP_MIN subu t1, t1, t9 # adjust exponent li v0, 32 blt t9, v0, 1f subu t9, t9, v0 # shift fraction left >= 32 bits sll t2, t3, t9 move t3, zero j ra 1: subu v0, v0, t9 # shift fraction left < 32 bits sll t2, t2, t9 srl v1, t3, v0 or t2, t2, v1 sll t3, t3, t9 j ra END(renorm_fs_d) /*---------------------------------------------------------------------------- * renorm_ft_s -- * * Results: * ta1 unbiased exponent * ta2 normalized fraction * *---------------------------------------------------------------------------- */ LEAF(renorm_ft_s) /* * Find out how many leading zero bits are in ta2 and put in t9. */ move v0, ta2 move t9, zero srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift ta2 the correct number of bits. */ 1: subu t9, t9, SLEAD_ZEROS # dont count normal leading zeros li ta1, SEXP_MIN subu ta1, ta1, t9 # adjust exponent sll ta2, ta2, t9 j ra END(renorm_ft_s) /*---------------------------------------------------------------------------- * renorm_ft_d -- * * Results: * ta1 unbiased exponent * ta2,ta3 normalized fraction * *---------------------------------------------------------------------------- */ LEAF(renorm_ft_d) /* * Find out how many leading zero bits are in ta2,ta3 and put in t9. */ move v0, ta2 move t9, zero bne ta2, zero, 1f move v0, ta3 addu t9, 32 1: srl v1, v0, 16 bne v1, zero, 1f addu t9, 16 sll v0, 16 1: srl v1, v0, 24 bne v1, zero, 1f addu t9, 8 sll v0, 8 1: srl v1, v0, 28 bne v1, zero, 1f addu t9, 4 sll v0, 4 1: srl v1, v0, 30 bne v1, zero, 1f addu t9, 2 sll v0, 2 1: srl v1, v0, 31 bne v1, zero, 1f addu t9, 1 /* * Now shift ta2,ta3 the correct number of bits. */ 1: subu t9, t9, DLEAD_ZEROS # dont count normal leading zeros li ta1, DEXP_MIN subu ta1, ta1, t9 # adjust exponent li v0, 32 blt t9, v0, 1f subu t9, t9, v0 # shift fraction left >= 32 bits sll ta2, ta3, t9 move ta3, zero j ra 1: subu v0, v0, t9 # shift fraction left < 32 bits sll ta2, ta2, t9 srl v1, ta3, v0 or ta2, ta2, v1 sll ta3, ta3, t9 j ra END(renorm_ft_d) Index: head/sys/mips/mips/locore.S =================================================================== --- head/sys/mips/mips/locore.S (revision 331253) +++ head/sys/mips/mips/locore.S (revision 331254) @@ -1,205 +1,205 @@ /* $OpenBSD: locore.S,v 1.18 1998/09/15 10:58:53 pefo Exp $ */ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Digital Equipment Corporation and Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Copyright (C) 1989 Digital Equipment Corporation. * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby granted, * provided that the above copyright notice appears in all copies. * Digital Equipment Corporation makes no representations about the * suitability of this software for any purpose. It is provided "as is" * without express or implied warranty. * * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/loMem.s, * v 1.1 89/07/11 17:55:04 nelson Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/machAsm.s, * v 9.2 90/01/29 18:00:39 shirriff Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/vm/ds3100.md/vmPmaxAsm.s, * v 1.1 89/07/10 14:27:41 nelson Exp SPRITE (DECWRL) * * from: @(#)locore.s 8.5 (Berkeley) 1/4/94 * JNPR: locore.S,v 1.6.2.1 2007/08/29 12:24:49 girish * $FreeBSD$ */ /* * FREEBSD_DEVELOPERS_FIXME * The start routine below was written for a multi-core CPU * with each core being hyperthreaded. This serves as an example * for a complex CPU architecture. For a different CPU complex * please make necessary changes to read CPU-ID etc. * A clean solution would be to have a different locore file for * each CPU type. */ /* * Contains code that is the first executed at boot time plus * assembly language support routines. */ #include #include #include #include -#include "assym.s" +#include "assym.inc" .data #ifdef YAMON GLOBAL(fenvp) .space 4 # Assumes mips32? Is that OK? #endif .set noreorder .text GLOBAL(btext) ASM_ENTRY(_start) VECTOR(_locore, unknown) /* UNSAFE TO USE a0..a3, need to preserve the args from boot loader */ mtc0 zero, MIPS_COP_0_CAUSE # Clear soft interrupts #if defined(CPU_CNMIPS) /* * t1: Bits to set explicitly: * Enable FPU */ /* Set these bits */ li t1, (MIPS_SR_COP_0_BIT | MIPS_SR_PX | MIPS_SR_KX | MIPS_SR_UX | MIPS_SR_SX | MIPS_SR_BEV) /* Reset these bits */ li t0, ~(MIPS_SR_DE | MIPS_SR_SR | MIPS_SR_ERL | MIPS_SR_EXL | MIPS_SR_INT_IE | MIPS_SR_COP_2_BIT) #elif defined (CPU_RMI) || defined (CPU_NLM) /* Set these bits */ li t1, (MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT | MIPS_SR_KX | MIPS_SR_UX) /* Reset these bits */ li t0, ~(MIPS_SR_BEV | MIPS_SR_SR | MIPS_SR_INT_IE) #else /* * t0: Bits to preserve if set: * Soft reset * Boot exception vectors (firmware-provided) */ li t0, (MIPS_SR_BEV | MIPS_SR_SR) /* * t1: Bits to set explicitly: * Enable FPU */ li t1, MIPS_SR_COP_1_BIT #if defined(__mips_n32) || defined(__mips_n64) or t1, MIPS_SR_FR #endif #ifdef __mips_n64 or t1, MIPS_SR_KX | MIPS_SR_SX | MIPS_SR_UX #endif #endif /* * Read coprocessor 0 status register, clear bits not * preserved (namely, clearing interrupt bits), and set * bits we want to explicitly set. */ mfc0 t2, MIPS_COP_0_STATUS and t2, t0 or t2, t1 mtc0 t2, MIPS_COP_0_STATUS COP0_SYNC /* Make sure KSEG0 is cached */ li t0, MIPS_CCA_CACHED mtc0 t0, MIPS_COP_0_CONFIG COP0_SYNC /*xxximp * now that we pass a0...a3 to the platform_init routine, do we need * to stash this stuff here? */ #ifdef YAMON /* Save YAMON boot environment pointer */ sw a2, _C_LABEL(fenvp) #endif #if defined(CPU_CNMIPS) && defined(SMP) .set push .set mips32r2 rdhwr t2, $0 beqz t2, 1f nop j octeon_ap_wait nop .set pop 1: #endif #if defined(CPU_MALTA) && defined(SMP) .set push .set mips32r2 jal malta_cpu_configure nop jal platform_processor_id nop beqz v0, 1f nop j malta_ap_wait nop .set pop 1: #endif /* * Initialize stack and call machine startup. */ PTR_LA sp, _C_LABEL(pcpu_space) PTR_ADDU sp, (PAGE_SIZE * 2) - CALLFRAME_SIZ REG_S zero, CALLFRAME_RA(sp) # Zero out old ra for debugger REG_S zero, CALLFRAME_SP(sp) # Zero out old fp for debugger PTR_LA gp, _C_LABEL(_gp) /* Call the platform-specific startup code. */ jal _C_LABEL(platform_start) nop PTR_LA sp, _C_LABEL(thread0_st) PTR_L a0, TD_PCB(sp) REG_LI t0, ~7 and a0, a0, t0 PTR_SUBU sp, a0, CALLFRAME_SIZ jal _C_LABEL(mi_startup) # mi_startup(frame) sw zero, CALLFRAME_SIZ - 8(sp) # Zero out old fp for debugger PANIC("Startup failed!") VECTOR_END(_locore) Index: head/sys/mips/mips/mpboot.S =================================================================== --- head/sys/mips/mips/mpboot.S (revision 331253) +++ head/sys/mips/mips/mpboot.S (revision 331254) @@ -1,89 +1,89 @@ /*- * Copyright (c) 2010 Neelkanth Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include -#include "assym.s" +#include "assym.inc" .text .set noat .set noreorder /* XXX move this to a header file */ #if defined(CPU_CNMIPS) #define CLEAR_STATUS \ mfc0 a0, MIPS_COP_0_STATUS ;\ li a2, (MIPS_SR_KX | MIPS_SR_SX | MIPS_SR_UX) ; \ or a0, a0, a2 ; \ li a2, ~(MIPS_SR_INT_IE | MIPS_SR_EXL | MIPS_SR_KSU_USER | MIPS_SR_BEV) ; \ and a0, a0, a2 ; \ mtc0 a0, MIPS_COP_0_STATUS #elif defined(__mips_n64) #define CLEAR_STATUS \ li a0, (MIPS_SR_KX | MIPS_SR_UX) ; \ mtc0 a0, MIPS_COP_0_STATUS #else #define CLEAR_STATUS \ mtc0 zero, MIPS_COP_0_STATUS #endif GLOBAL(mpentry) CLEAR_STATUS /* disable interrupts */ mtc0 zero, MIPS_COP_0_CAUSE /* clear soft interrupts */ li t0, MIPS_CCA_CACHED /* make sure kseg0 is cached */ mtc0 t0, MIPS_COP_0_CONFIG COP0_SYNC jal platform_processor_id /* get the processor number */ nop move s0, v0 /* * Initialize stack and call machine startup */ PTR_LA sp, _C_LABEL(pcpu_space) addiu sp, (PAGE_SIZE * 2) - CALLFRAME_SIZ sll t0, s0, PAGE_SHIFT + 1 addu sp, sp, t0 /* Zero out old ra and old fp for debugger */ sw zero, CALLFRAME_SIZ - 4(sp) sw zero, CALLFRAME_SIZ - 8(sp) PTR_LA gp, _C_LABEL(_gp) jal platform_init_ap move a0, s0 jal smp_init_secondary move a0, s0 PANIC("AP startup failed!") Index: head/sys/mips/mips/octeon_cop2_swtch.S =================================================================== --- head/sys/mips/mips/octeon_cop2_swtch.S (revision 331253) +++ head/sys/mips/mips/octeon_cop2_swtch.S (revision 331254) @@ -1,246 +1,246 @@ /*- * Copyright (c) 2011 Oleksandr Tymoshenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include -#include "assym.s" +#include "assym.inc" .set noreorder #define SAVE_COP2_REGISTER(reg) \ dmfc2 t1, reg; sd t1, reg##_OFFSET(a0) #define RESTORE_COP2_REGISTER(reg) \ ld t1, reg##_OFFSET(a0); dmtc2 t1, reg##_SET LEAF(octeon_cop2_save) /* save original cop2 status in t2*/ mfc0 t2, MIPS_COP_0_STATUS or t0, t2, MIPS_SR_COP_2_BIT and t0, t0, ~MIPS_SR_INT_IE mtc0 t0, MIPS_COP_0_STATUS /* Get CvmCtl register */ dmfc0 t0, $9, 7 /* CRC state */ SAVE_COP2_REGISTER(COP2_CRC_IV) SAVE_COP2_REGISTER(COP2_CRC_LENGTH) SAVE_COP2_REGISTER(COP2_CRC_POLY) /* if CvmCtl[NODFA_CP2] -> save_nodfa */ bbit1 t0, 28, save_nodfa nop /* LLM state */ SAVE_COP2_REGISTER(COP2_LLM_DAT0) SAVE_COP2_REGISTER(COP2_LLM_DAT1) save_nodfa: /* crypto stuff is irrelevant if CvmCtl[NOCRYPTO] */ bbit1 t0, 26, save_done nop SAVE_COP2_REGISTER(COP2_3DES_IV) SAVE_COP2_REGISTER(COP2_3DES_KEY0) SAVE_COP2_REGISTER(COP2_3DES_KEY1) SAVE_COP2_REGISTER(COP2_3DES_KEY2) SAVE_COP2_REGISTER(COP2_3DES_RESULT) SAVE_COP2_REGISTER(COP2_AES_INP0) SAVE_COP2_REGISTER(COP2_AES_IV0) SAVE_COP2_REGISTER(COP2_AES_IV1) SAVE_COP2_REGISTER(COP2_AES_KEY0) SAVE_COP2_REGISTER(COP2_AES_KEY1) SAVE_COP2_REGISTER(COP2_AES_KEY2) SAVE_COP2_REGISTER(COP2_AES_KEY3) SAVE_COP2_REGISTER(COP2_AES_KEYLEN) SAVE_COP2_REGISTER(COP2_AES_RESULT0) SAVE_COP2_REGISTER(COP2_AES_RESULT1) dmfc0 t0, $15 li t1, 0x000d0000 /* Octeon Pass1 */ beq t0, t1, save_pass1 nop SAVE_COP2_REGISTER(COP2_HSH_DATW0) SAVE_COP2_REGISTER(COP2_HSH_DATW1) SAVE_COP2_REGISTER(COP2_HSH_DATW2) SAVE_COP2_REGISTER(COP2_HSH_DATW3) SAVE_COP2_REGISTER(COP2_HSH_DATW4) SAVE_COP2_REGISTER(COP2_HSH_DATW5) SAVE_COP2_REGISTER(COP2_HSH_DATW6) SAVE_COP2_REGISTER(COP2_HSH_DATW7) SAVE_COP2_REGISTER(COP2_HSH_DATW8) SAVE_COP2_REGISTER(COP2_HSH_DATW9) SAVE_COP2_REGISTER(COP2_HSH_DATW10) SAVE_COP2_REGISTER(COP2_HSH_DATW11) SAVE_COP2_REGISTER(COP2_HSH_DATW12) SAVE_COP2_REGISTER(COP2_HSH_DATW13) SAVE_COP2_REGISTER(COP2_HSH_DATW14) SAVE_COP2_REGISTER(COP2_HSH_IVW0) SAVE_COP2_REGISTER(COP2_HSH_IVW1) SAVE_COP2_REGISTER(COP2_HSH_IVW2) SAVE_COP2_REGISTER(COP2_HSH_IVW3) SAVE_COP2_REGISTER(COP2_HSH_IVW4) SAVE_COP2_REGISTER(COP2_HSH_IVW5) SAVE_COP2_REGISTER(COP2_HSH_IVW6) SAVE_COP2_REGISTER(COP2_HSH_IVW7) SAVE_COP2_REGISTER(COP2_GFM_MULT0) SAVE_COP2_REGISTER(COP2_GFM_MULT1) SAVE_COP2_REGISTER(COP2_GFM_POLY) SAVE_COP2_REGISTER(COP2_GFM_RESULT0) SAVE_COP2_REGISTER(COP2_GFM_RESULT1) /* restore saved COP2 status */ mtc0 t2, MIPS_COP_0_STATUS jr ra nop save_pass1: SAVE_COP2_REGISTER(COP2_HSH_DATW0_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW1_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW2_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW3_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW4_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW5_PASS1) SAVE_COP2_REGISTER(COP2_HSH_DATW6_PASS1) SAVE_COP2_REGISTER(COP2_HSH_IVW0_PASS1) SAVE_COP2_REGISTER(COP2_HSH_IVW1_PASS1) SAVE_COP2_REGISTER(COP2_HSH_IVW2_PASS1) save_done: /* restore saved COP2 status */ mtc0 t2, MIPS_COP_0_STATUS jr ra nop END(octeon_cop2_save) LEAF(octeon_cop2_restore) /* save original cop2 status in t2*/ mfc0 t2, MIPS_COP_0_STATUS or t0, t2, MIPS_SR_COP_2_BIT and t0, t0, ~MIPS_SR_INT_IE mtc0 t0, MIPS_COP_0_STATUS /* Get CvmCtl register */ dmfc0 t0, $9, 7 /* CRC state */ RESTORE_COP2_REGISTER(COP2_CRC_IV) RESTORE_COP2_REGISTER(COP2_CRC_LENGTH) RESTORE_COP2_REGISTER(COP2_CRC_POLY) /* if CvmCtl[NODFA_CP2] -> save_nodfa */ bbit1 t0, 28, restore_nodfa nop /* LLM state */ RESTORE_COP2_REGISTER(COP2_LLM_DAT0) RESTORE_COP2_REGISTER(COP2_LLM_DAT1) restore_nodfa: /* crypto stuff is irrelevant if CvmCtl[NOCRYPTO] */ bbit1 t0, 26, restore_done nop RESTORE_COP2_REGISTER(COP2_3DES_IV) RESTORE_COP2_REGISTER(COP2_3DES_KEY0) RESTORE_COP2_REGISTER(COP2_3DES_KEY1) RESTORE_COP2_REGISTER(COP2_3DES_KEY2) RESTORE_COP2_REGISTER(COP2_3DES_RESULT) RESTORE_COP2_REGISTER(COP2_AES_INP0) RESTORE_COP2_REGISTER(COP2_AES_IV0) RESTORE_COP2_REGISTER(COP2_AES_IV1) RESTORE_COP2_REGISTER(COP2_AES_KEY0) RESTORE_COP2_REGISTER(COP2_AES_KEY1) RESTORE_COP2_REGISTER(COP2_AES_KEY2) RESTORE_COP2_REGISTER(COP2_AES_KEY3) RESTORE_COP2_REGISTER(COP2_AES_KEYLEN) RESTORE_COP2_REGISTER(COP2_AES_RESULT0) RESTORE_COP2_REGISTER(COP2_AES_RESULT1) dmfc0 t0, $15 li t1, 0x000d0000 /* Octeon Pass1 */ beq t0, t1, restore_pass1 nop RESTORE_COP2_REGISTER(COP2_HSH_DATW0) RESTORE_COP2_REGISTER(COP2_HSH_DATW1) RESTORE_COP2_REGISTER(COP2_HSH_DATW2) RESTORE_COP2_REGISTER(COP2_HSH_DATW3) RESTORE_COP2_REGISTER(COP2_HSH_DATW4) RESTORE_COP2_REGISTER(COP2_HSH_DATW5) RESTORE_COP2_REGISTER(COP2_HSH_DATW6) RESTORE_COP2_REGISTER(COP2_HSH_DATW7) RESTORE_COP2_REGISTER(COP2_HSH_DATW8) RESTORE_COP2_REGISTER(COP2_HSH_DATW9) RESTORE_COP2_REGISTER(COP2_HSH_DATW10) RESTORE_COP2_REGISTER(COP2_HSH_DATW11) RESTORE_COP2_REGISTER(COP2_HSH_DATW12) RESTORE_COP2_REGISTER(COP2_HSH_DATW13) RESTORE_COP2_REGISTER(COP2_HSH_DATW14) RESTORE_COP2_REGISTER(COP2_HSH_IVW0) RESTORE_COP2_REGISTER(COP2_HSH_IVW1) RESTORE_COP2_REGISTER(COP2_HSH_IVW2) RESTORE_COP2_REGISTER(COP2_HSH_IVW3) RESTORE_COP2_REGISTER(COP2_HSH_IVW4) RESTORE_COP2_REGISTER(COP2_HSH_IVW5) RESTORE_COP2_REGISTER(COP2_HSH_IVW6) RESTORE_COP2_REGISTER(COP2_HSH_IVW7) RESTORE_COP2_REGISTER(COP2_GFM_MULT0) RESTORE_COP2_REGISTER(COP2_GFM_MULT1) RESTORE_COP2_REGISTER(COP2_GFM_POLY) RESTORE_COP2_REGISTER(COP2_GFM_RESULT0) RESTORE_COP2_REGISTER(COP2_GFM_RESULT1) /* restore saved COP2 status */ mtc0 t2, MIPS_COP_0_STATUS jr ra nop restore_pass1: RESTORE_COP2_REGISTER(COP2_HSH_DATW0_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW1_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW2_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW3_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW4_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW5_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_DATW6_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_IVW0_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_IVW1_PASS1) RESTORE_COP2_REGISTER(COP2_HSH_IVW2_PASS1) restore_done: /* restore saved COP2 status */ mtc0 t2, MIPS_COP_0_STATUS jr ra nop END(octeon_cop2_restore) Index: head/sys/mips/mips/support.S =================================================================== --- head/sys/mips/mips/support.S (revision 331253) +++ head/sys/mips/mips/support.S (revision 331254) @@ -1,1037 +1,1037 @@ /* $OpenBSD: locore.S,v 1.18 1998/09/15 10:58:53 pefo Exp $ */ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Digital Equipment Corporation and Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Copyright (C) 1989 Digital Equipment Corporation. * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby granted, * provided that the above copyright notice appears in all copies. * Digital Equipment Corporation makes no representations about the * suitability of this software for any purpose. It is provided "as is" * without express or implied warranty. * * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/loMem.s, * v 1.1 89/07/11 17:55:04 nelson Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/machAsm.s, * v 9.2 90/01/29 18:00:39 shirriff Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/vm/ds3100.md/vmPmaxAsm.s, * v 1.1 89/07/10 14:27:41 nelson Exp SPRITE (DECWRL) * * from: @(#)locore.s 8.5 (Berkeley) 1/4/94 * JNPR: support.S,v 1.5.2.2 2007/08/29 10:03:49 girish * $FreeBSD$ */ /* * Copyright (c) 1997 Jonathan Stone (hereinafter referred to as the author) * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Jonathan R. Stone for * the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Contains assembly language support routines. */ #include "opt_ddb.h" #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" .set noreorder # Noreorder is default style! /* * Primitives */ .text /* * See if access to addr with a len type instruction causes a machine check. * len is length of access (1=byte, 2=short, 4=int) * * badaddr(addr, len) * char *addr; * int len; */ LEAF(badaddr) PTR_LA v0, baderr GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) bne a1, 1, 2f PTR_S v0, U_PCB_ONFAULT(v1) b 5f lbu v0, (a0) 2: bne a1, 2, 4f nop b 5f lhu v0, (a0) 4: lw v0, (a0) 5: PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero # made it w/o errors baderr: j ra li v0, 1 # trap sends us here END(badaddr) /* * int copystr(void *kfaddr, void *kdaddr, size_t maxlen, size_t *lencopied) * Copy a NIL-terminated string, at most maxlen characters long. Return the * number of characters copied (including the NIL) in *lencopied. If the * string is too long, return ENAMETOOLONG; else return 0. */ LEAF(copystr) move t0, a2 beq a2, zero, 4f 1: lbu v0, 0(a0) PTR_SUBU a2, a2, 1 beq v0, zero, 2f sb v0, 0(a1) # each byte until NIL PTR_ADDU a0, a0, 1 bne a2, zero, 1b # less than maxlen PTR_ADDU a1, a1, 1 4: li v0, ENAMETOOLONG # run out of space 2: beq a3, zero, 3f # return num. of copied bytes PTR_SUBU a2, t0, a2 # if the 4th arg was non-NULL PTR_S a2, 0(a3) 3: j ra # v0 is 0 or ENAMETOOLONG nop END(copystr) /* * Copy a null terminated string from the user address space into * the kernel address space. * * copyinstr(fromaddr, toaddr, maxlength, &lencopied) * caddr_t fromaddr; * caddr_t toaddr; * u_int maxlength; * u_int *lencopied; */ NESTED(copyinstr, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a0, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(copystr) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S zero, U_PCB_ONFAULT(v1) j ra PTR_ADDU sp, sp, CALLFRAME_SIZ END(copyinstr) /* * Copy a null terminated string from the kernel address space into * the user address space. * * copyoutstr(fromaddr, toaddr, maxlength, &lencopied) * caddr_t fromaddr; * caddr_t toaddr; * u_int maxlength; * u_int *lencopied; */ NESTED(copyoutstr, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a1, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(copystr) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S zero, U_PCB_ONFAULT(v1) j ra PTR_ADDU sp, sp, CALLFRAME_SIZ END(copyoutstr) /* * Copy specified amount of data from user space into the kernel * copyin(from, to, len) * caddr_t *from; (user source address) * caddr_t *to; (kernel destination address) * unsigned len; */ NESTED(copyin, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a0, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(bcopy) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) # bcopy modified v1, so reload PTR_S zero, U_PCB_ONFAULT(v1) PTR_ADDU sp, sp, CALLFRAME_SIZ j ra move v0, zero END(copyin) /* * Copy specified amount of data from kernel to the user space * copyout(from, to, len) * caddr_t *from; (kernel source address) * caddr_t *to; (user destination address) * unsigned len; */ NESTED(copyout, CALLFRAME_SIZ, ra) PTR_SUBU sp, sp, CALLFRAME_SIZ .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) PTR_LA v0, copyerr blt a1, zero, _C_LABEL(copyerr) # make sure address is in user space REG_S ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) jal _C_LABEL(bcopy) PTR_S v0, U_PCB_ONFAULT(v1) REG_L ra, CALLFRAME_RA(sp) GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) # bcopy modified v1, so reload PTR_S zero, U_PCB_ONFAULT(v1) PTR_ADDU sp, sp, CALLFRAME_SIZ j ra move v0, zero END(copyout) LEAF(copyerr) REG_L ra, CALLFRAME_RA(sp) PTR_ADDU sp, sp, CALLFRAME_SIZ j ra li v0, EFAULT # return error END(copyerr) /* * {fu,su},{byte,sword,word}, fetch or store a byte, short or word to * user-space. */ #ifdef __mips_n64 LEAF(fueword64) XLEAF(fueword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) ld v0, 0(a0) # fetch word PTR_S zero, U_PCB_ONFAULT(v1) sd v0, 0(a1) # store word j ra li v0, 0 END(fueword64) #endif LEAF(fueword32) #ifndef __mips_n64 XLEAF(fueword) #endif PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lw v0, 0(a0) # fetch word PTR_S zero, U_PCB_ONFAULT(v1) sw v0, 0(a1) # store word j ra li v0, 0 END(fueword32) LEAF(fuesword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lhu v0, 0(a0) # fetch short PTR_S zero, U_PCB_ONFAULT(v1) sh v0, 0(a1) # store short j ra li v0, 0 END(fuesword) LEAF(fubyte) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lbu v0, 0(a0) # fetch byte j ra PTR_S zero, U_PCB_ONFAULT(v1) END(fubyte) LEAF(suword32) #ifndef __mips_n64 XLEAF(suword) #endif PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sw a1, 0(a0) # store word PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(suword32) #ifdef __mips_n64 LEAF(suword64) XLEAF(suword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sd a1, 0(a0) # store word PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(suword64) #endif /* * casueword(9) * u_long casueword(u_long *p, u_long oldval, u_long *oldval_p, * u_long newval) */ /* * casueword32(9) * uint32_t casueword(uint32_t *p, uint32_t oldval, * uint32_t newval) */ LEAF(casueword32) #ifndef __mips_n64 XLEAF(casueword) #endif PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) 1: move t0, a3 ll t1, 0(a0) bne a1, t1, 2f nop sc t0, 0(a0) # store word beqz t0, 1b nop j 3f li v0, 0 2: li v0, -1 3: PTR_S zero, U_PCB_ONFAULT(v1) jr ra sw t1, 0(a2) # unconditionally store old word END(casueword32) #ifdef __mips_n64 LEAF(casueword64) XLEAF(casueword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) 1: move t0, a3 lld t1, 0(a0) bne a1, t1, 2f nop scd t0, 0(a0) # store double word beqz t0, 1b nop j 3f li v0, 0 2: li v0, -1 3: PTR_S zero, U_PCB_ONFAULT(v1) jr ra sd t1, 0(a2) # unconditionally store old word END(casueword64) #endif /* * Will have to flush the instruction cache if byte merging is done in hardware. */ LEAF(susword) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sh a1, 0(a0) # store short PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(susword) LEAF(subyte) PTR_LA v0, fswberr blt a0, zero, fswberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sb a1, 0(a0) # store byte PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(subyte) LEAF(fswberr) j ra li v0, -1 END(fswberr) /* * fuswintr and suswintr are just like fusword and susword except that if * the page is not in memory or would cause a trap, then we return an error. * The important thing is to prevent sleep() and switch(). */ LEAF(fuswintr) PTR_LA v0, fswintrberr blt a0, zero, fswintrberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) lhu v0, 0(a0) # fetch short j ra PTR_S zero, U_PCB_ONFAULT(v1) END(fuswintr) LEAF(suswintr) PTR_LA v0, fswintrberr blt a0, zero, fswintrberr # make sure address is in user space nop GET_CPU_PCPU(v1) PTR_L v1, PC_CURPCB(v1) PTR_S v0, U_PCB_ONFAULT(v1) sh a1, 0(a0) # store short PTR_S zero, U_PCB_ONFAULT(v1) j ra move v0, zero END(suswintr) LEAF(fswintrberr) j ra li v0, -1 END(fswintrberr) /* * memset(void *s1, int c, int len) * NetBSD: memset.S,v 1.3 2001/10/16 15:40:53 uch Exp */ LEAF(memset) .set noreorder blt a2, 12, memsetsmallclr # small amount to clear? move v0, a0 # save s1 for result sll t1, a1, 8 # compute c << 8 in t1 or t1, t1, a1 # compute c << 8 | c in 11 sll t2, t1, 16 # shift that left 16 or t1, t2, t1 # or together PTR_SUBU t0, zero, a0 # compute # bytes to word align address and t0, t0, 3 beq t0, zero, 1f # skip if word aligned PTR_SUBU a2, a2, t0 # subtract from remaining count SWHI t1, 0(a0) # store 1, 2, or 3 bytes to align PTR_ADDU a0, a0, t0 1: and v1, a2, 3 # compute number of whole words left PTR_SUBU t0, a2, v1 PTR_SUBU a2, a2, t0 PTR_ADDU t0, t0, a0 # compute ending address 2: PTR_ADDU a0, a0, 4 # clear words bne a0, t0, 2b # unrolling loop does not help sw t1, -4(a0) # since we are limited by memory speed memsetsmallclr: ble a2, zero, 2f PTR_ADDU t0, a2, a0 # compute ending address 1: PTR_ADDU a0, a0, 1 # clear bytes bne a0, t0, 1b sb a1, -1(a0) 2: j ra nop .set reorder END(memset) /* * bzero(s1, n) */ LEAF(bzero) XLEAF(blkclr) .set noreorder blt a1, 12, smallclr # small amount to clear? PTR_SUBU a3, zero, a0 # compute # bytes to word align address and a3, a3, 3 beq a3, zero, 1f # skip if word aligned PTR_SUBU a1, a1, a3 # subtract from remaining count SWHI zero, 0(a0) # clear 1, 2, or 3 bytes to align PTR_ADDU a0, a0, a3 1: and v0, a1, 3 # compute number of words left PTR_SUBU a3, a1, v0 move a1, v0 PTR_ADDU a3, a3, a0 # compute ending address 2: PTR_ADDU a0, a0, 4 # clear words bne a0, a3, 2b # unrolling loop does not help sw zero, -4(a0) # since we are limited by memory speed smallclr: ble a1, zero, 2f PTR_ADDU a3, a1, a0 # compute ending address 1: PTR_ADDU a0, a0, 1 # clear bytes bne a0, a3, 1b sb zero, -1(a0) 2: j ra nop END(bzero) /* * bcmp(s1, s2, n) */ LEAF(bcmp) .set noreorder blt a2, 16, smallcmp # is it worth any trouble? xor v0, a0, a1 # compare low two bits of addresses and v0, v0, 3 PTR_SUBU a3, zero, a1 # compute # bytes to word align address bne v0, zero, unalignedcmp # not possible to align addresses and a3, a3, 3 beq a3, zero, 1f PTR_SUBU a2, a2, a3 # subtract from remaining count move v0, v1 # init v0,v1 so unmodified bytes match LWHI v0, 0(a0) # read 1, 2, or 3 bytes LWHI v1, 0(a1) PTR_ADDU a1, a1, a3 bne v0, v1, nomatch PTR_ADDU a0, a0, a3 1: and a3, a2, ~3 # compute number of whole words left PTR_SUBU a2, a2, a3 # which has to be >= (16-3) & ~3 PTR_ADDU a3, a3, a0 # compute ending address 2: lw v0, 0(a0) # compare words lw v1, 0(a1) PTR_ADDU a0, a0, 4 bne v0, v1, nomatch PTR_ADDU a1, a1, 4 bne a0, a3, 2b nop b smallcmp # finish remainder nop unalignedcmp: beq a3, zero, 2f PTR_SUBU a2, a2, a3 # subtract from remaining count PTR_ADDU a3, a3, a0 # compute ending address 1: lbu v0, 0(a0) # compare bytes until a1 word aligned lbu v1, 0(a1) PTR_ADDU a0, a0, 1 bne v0, v1, nomatch PTR_ADDU a1, a1, 1 bne a0, a3, 1b nop 2: and a3, a2, ~3 # compute number of whole words left PTR_SUBU a2, a2, a3 # which has to be >= (16-3) & ~3 PTR_ADDU a3, a3, a0 # compute ending address 3: LWHI v0, 0(a0) # compare words a0 unaligned, a1 aligned LWLO v0, 3(a0) lw v1, 0(a1) PTR_ADDU a0, a0, 4 bne v0, v1, nomatch PTR_ADDU a1, a1, 4 bne a0, a3, 3b nop smallcmp: ble a2, zero, match PTR_ADDU a3, a2, a0 # compute ending address 1: lbu v0, 0(a0) lbu v1, 0(a1) PTR_ADDU a0, a0, 1 bne v0, v1, nomatch PTR_ADDU a1, a1, 1 bne a0, a3, 1b nop match: j ra move v0, zero nomatch: j ra li v0, 1 END(bcmp) /* * bit = ffs(value) */ LEAF(ffs) .set noreorder beq a0, zero, 2f move v0, zero 1: and v1, a0, 1 # bit set? addu v0, v0, 1 beq v1, zero, 1b # no, continue srl a0, a0, 1 2: j ra nop END(ffs) /** * void * atomic_set_16(u_int16_t *a, u_int16_t b) * { * *a |= b; * } */ LEAF(atomic_set_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 andi a1, a1, 0xffff 1: ll t0, 0(a0) or t0, t0, a1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_set_16) /** * void * atomic_clear_16(u_int16_t *a, u_int16_t b) * { * *a &= ~b; * } */ LEAF(atomic_clear_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 nor a1, zero, a1 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xffff # t1 has the original lower 16 bits and t1, t1, a1 # t1 has the new lower 16 bits srl t0, t0, 16 # preserve original top 16 bits sll t0, t0, 16 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_clear_16) /** * void * atomic_subtract_16(uint16_t *a, uint16_t b) * { * *a -= b; * } */ LEAF(atomic_subtract_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xffff # t1 has the original lower 16 bits subu t1, t1, a1 andi t1, t1, 0xffff # t1 has the new lower 16 bits srl t0, t0, 16 # preserve original top 16 bits sll t0, t0, 16 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_subtract_16) /** * void * atomic_add_16(uint16_t *a, uint16_t b) * { * *a += b; * } */ LEAF(atomic_add_16) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xffff # t1 has the original lower 16 bits addu t1, t1, a1 andi t1, t1, 0xffff # t1 has the new lower 16 bits srl t0, t0, 16 # preserve original top 16 bits sll t0, t0, 16 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_add_16) /** * void * atomic_add_8(uint8_t *a, uint8_t b) * { * *a += b; * } */ LEAF(atomic_add_8) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xff # t1 has the original lower 8 bits addu t1, t1, a1 andi t1, t1, 0xff # t1 has the new lower 8 bits srl t0, t0, 8 # preserve original top 24 bits sll t0, t0, 8 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_add_8) /** * void * atomic_subtract_8(uint8_t *a, uint8_t b) * { * *a += b; * } */ LEAF(atomic_subtract_8) .set noreorder srl a0, a0, 2 # round down address to be 32-bit aligned sll a0, a0, 2 1: ll t0, 0(a0) move t1, t0 andi t1, t1, 0xff # t1 has the original lower 8 bits subu t1, t1, a1 andi t1, t1, 0xff # t1 has the new lower 8 bits srl t0, t0, 8 # preserve original top 24 bits sll t0, t0, 8 or t0, t0, t1 sc t0, 0(a0) beq t0, zero, 1b nop j ra nop END(atomic_subtract_8) .set noreorder # Noreorder is default style! #if defined(DDB) || defined(DEBUG) LEAF(kdbpeek) PTR_LA v1, ddberr and v0, a0, 3 # unaligned ? GET_CPU_PCPU(t1) PTR_L t1, PC_CURPCB(t1) bne v0, zero, 1f PTR_S v1, U_PCB_ONFAULT(t1) lw v0, (a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) 1: LWHI v0, 0(a0) LWLO v0, 3(a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) END(kdbpeek) LEAF(kdbpeekd) PTR_LA v1, ddberr and v0, a0, 3 # unaligned ? GET_CPU_PCPU(t1) PTR_L t1, PC_CURPCB(t1) bne v0, zero, 1f PTR_S v1, U_PCB_ONFAULT(t1) ld v0, (a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) 1: REG_LHI v0, 0(a0) REG_LLO v0, 7(a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) END(kdbpeekd) ddberr: jr ra nop #if defined(DDB) LEAF(kdbpoke) PTR_LA v1, ddberr and v0, a0, 3 # unaligned ? GET_CPU_PCPU(t1) PTR_L t1, PC_CURPCB(t1) bne v0, zero, 1f PTR_S v1, U_PCB_ONFAULT(t1) sw a1, (a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) 1: SWHI a1, 0(a0) SWLO a1, 3(a0) jr ra PTR_S zero, U_PCB_ONFAULT(t1) END(kdbpoke) .data .globl esym esym: .word 0 #endif /* DDB */ #endif /* DDB || DEBUG */ .text LEAF(breakpoint) break MIPS_BREAK_SOVER_VAL jr ra nop END(breakpoint) LEAF(setjmp) mfc0 v0, MIPS_COP_0_STATUS # Later the "real" spl value! REG_S s0, (SZREG * PCB_REG_S0)(a0) REG_S s1, (SZREG * PCB_REG_S1)(a0) REG_S s2, (SZREG * PCB_REG_S2)(a0) REG_S s3, (SZREG * PCB_REG_S3)(a0) REG_S s4, (SZREG * PCB_REG_S4)(a0) REG_S s5, (SZREG * PCB_REG_S5)(a0) REG_S s6, (SZREG * PCB_REG_S6)(a0) REG_S s7, (SZREG * PCB_REG_S7)(a0) REG_S s8, (SZREG * PCB_REG_S8)(a0) REG_S sp, (SZREG * PCB_REG_SP)(a0) REG_S ra, (SZREG * PCB_REG_RA)(a0) REG_S v0, (SZREG * PCB_REG_SR)(a0) jr ra li v0, 0 # setjmp return END(setjmp) LEAF(longjmp) REG_L v0, (SZREG * PCB_REG_SR)(a0) REG_L ra, (SZREG * PCB_REG_RA)(a0) REG_L s0, (SZREG * PCB_REG_S0)(a0) REG_L s1, (SZREG * PCB_REG_S1)(a0) REG_L s2, (SZREG * PCB_REG_S2)(a0) REG_L s3, (SZREG * PCB_REG_S3)(a0) REG_L s4, (SZREG * PCB_REG_S4)(a0) REG_L s5, (SZREG * PCB_REG_S5)(a0) REG_L s6, (SZREG * PCB_REG_S6)(a0) REG_L s7, (SZREG * PCB_REG_S7)(a0) REG_L s8, (SZREG * PCB_REG_S8)(a0) REG_L sp, (SZREG * PCB_REG_SP)(a0) mtc0 v0, MIPS_COP_0_STATUS # Later the "real" spl value! ITLBNOPFIX jr ra li v0, 1 # longjmp return END(longjmp) LEAF(mips3_ld) .set push .set noreorder .set mips64 #if defined(__mips_o32) mfc0 t0, MIPS_COP_0_STATUS # turn off interrupts and t1, t0, ~(MIPS_SR_INT_IE) mtc0 t1, MIPS_COP_0_STATUS COP0_SYNC nop nop nop ld v0, 0(a0) #if _BYTE_ORDER == _BIG_ENDIAN dsll v1, v0, 32 dsra v1, v1, 32 # low word in v1 dsra v0, v0, 32 # high word in v0 #else dsra v1, v0, 32 # high word in v1 dsll v0, v0, 32 dsra v0, v0, 32 # low word in v0 #endif mtc0 t0, MIPS_COP_0_STATUS # restore intr status. COP0_SYNC nop #else /* !__mips_o32 */ ld v0, 0(a0) #endif /* !__mips_o32 */ jr ra nop .set pop END(mips3_ld) LEAF(mips3_sd) .set push .set mips64 .set noreorder #if defined(__mips_o32) mfc0 t0, MIPS_COP_0_STATUS # turn off interrupts and t1, t0, ~(MIPS_SR_INT_IE) mtc0 t1, MIPS_COP_0_STATUS COP0_SYNC nop nop nop # NOTE: a1 is padding! #if _BYTE_ORDER == _BIG_ENDIAN dsll a2, a2, 32 # high word in a2 dsll a3, a3, 32 # low word in a3 dsrl a3, a3, 32 #else dsll a2, a2, 32 # low word in a2 dsrl a2, a2, 32 dsll a3, a3, 32 # high word in a3 #endif or a1, a2, a3 sd a1, 0(a0) mtc0 t0, MIPS_COP_0_STATUS # restore intr status. COP0_SYNC nop #else /* !__mips_o32 */ sd a1, 0(a0) #endif /* !__mips_o32 */ jr ra nop .set pop END(mips3_sd) Index: head/sys/mips/mips/swtch.S =================================================================== --- head/sys/mips/mips/swtch.S (revision 331253) +++ head/sys/mips/mips/swtch.S (revision 331254) @@ -1,689 +1,689 @@ /* $OpenBSD: locore.S,v 1.18 1998/09/15 10:58:53 pefo Exp $ */ /*- * Copyright (c) 1992, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Digital Equipment Corporation and Ralph Campbell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Copyright (C) 1989 Digital Equipment Corporation. * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby granted, * provided that the above copyright notice appears in all copies. * Digital Equipment Corporation makes no representations about the * suitability of this software for any purpose. It is provided "as is" * without express or implied warranty. * * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/loMem.s, * v 1.1 89/07/11 17:55:04 nelson Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/machAsm.s, * v 9.2 90/01/29 18:00:39 shirriff Exp SPRITE (DECWRL) * from: Header: /sprite/src/kernel/vm/ds3100.md/vmPmaxAsm.s, * v 1.1 89/07/10 14:27:41 nelson Exp SPRITE (DECWRL) * * from: @(#)locore.s 8.5 (Berkeley) 1/4/94 * JNPR: swtch.S,v 1.6.2.1 2007/09/10 10:36:50 girish * $FreeBSD$ */ /* * Contains code that is the first executed at boot time plus * assembly language support routines. */ #include "opt_compat.h" #include #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" .set noreorder # Noreorder is default style! /* * Setup for and return to user. */ LEAF(fork_trampoline) move a0,s0 move a1,s1 jal _C_LABEL(fork_exit) move a2,s2 #BDSlot DO_AST mfc0 v0, MIPS_COP_0_STATUS and v0, ~(MIPS_SR_INT_IE) mtc0 v0, MIPS_COP_0_STATUS # disable interrupts COP0_SYNC /* * The use of k1 for storing the PCB pointer must be done only * after interrupts are disabled. Otherwise it will get overwritten * by the interrupt code. */ .set noat GET_CPU_PCPU(k1) PTR_L k1, PC_CURPCB(k1) RESTORE_U_PCB_REG(t0, MULLO, k1) RESTORE_U_PCB_REG(t1, MULHI, k1) mtlo t0 mthi t1 RESTORE_U_PCB_REG(a0, PC, k1) RESTORE_U_PCB_REG(AT, AST, k1) RESTORE_U_PCB_REG(v0, V0, k1) MTC0 a0, MIPS_COP_0_EXC_PC # set return address RESTORE_U_PCB_REG(v1, V1, k1) RESTORE_U_PCB_REG(a0, A0, k1) RESTORE_U_PCB_REG(a1, A1, k1) RESTORE_U_PCB_REG(a2, A2, k1) RESTORE_U_PCB_REG(a3, A3, k1) RESTORE_U_PCB_REG(t0, T0, k1) RESTORE_U_PCB_REG(t1, T1, k1) RESTORE_U_PCB_REG(t2, T2, k1) RESTORE_U_PCB_REG(t3, T3, k1) RESTORE_U_PCB_REG(ta0, TA0, k1) RESTORE_U_PCB_REG(ta1, TA1, k1) RESTORE_U_PCB_REG(ta2, TA2, k1) RESTORE_U_PCB_REG(ta3, TA3, k1) RESTORE_U_PCB_REG(s0, S0, k1) RESTORE_U_PCB_REG(s1, S1, k1) RESTORE_U_PCB_REG(s2, S2, k1) RESTORE_U_PCB_REG(s3, S3, k1) RESTORE_U_PCB_REG(s4, S4, k1) RESTORE_U_PCB_REG(s5, S5, k1) RESTORE_U_PCB_REG(s6, S6, k1) RESTORE_U_PCB_REG(s7, S7, k1) RESTORE_U_PCB_REG(t8, T8, k1) RESTORE_U_PCB_REG(t9, T9, k1) RESTORE_U_PCB_REG(k0, SR, k1) RESTORE_U_PCB_REG(gp, GP, k1) RESTORE_U_PCB_REG(s8, S8, k1) RESTORE_U_PCB_REG(ra, RA, k1) RESTORE_U_PCB_REG(sp, SP, k1) li k1, ~MIPS_SR_INT_MASK and k0, k0, k1 mfc0 k1, MIPS_COP_0_STATUS and k1, k1, MIPS_SR_INT_MASK or k0, k0, k1 mtc0 k0, MIPS_COP_0_STATUS # switch to user mode (when eret...) HAZARD_DELAY sync eret .set at END(fork_trampoline) /* * Update pcb, saving current processor state. * Note: this only works if pcbp != curproc's pcb since * cpu_switch() will copy over pcb_context. * * savectx(struct pcb *pcbp); */ LEAF(savectx) SAVE_U_PCB_CONTEXT(s0, PCB_REG_S0, a0) SAVE_U_PCB_CONTEXT(s1, PCB_REG_S1, a0) SAVE_U_PCB_CONTEXT(s2, PCB_REG_S2, a0) SAVE_U_PCB_CONTEXT(s3, PCB_REG_S3, a0) mfc0 v0, MIPS_COP_0_STATUS SAVE_U_PCB_CONTEXT(s4, PCB_REG_S4, a0) SAVE_U_PCB_CONTEXT(s5, PCB_REG_S5, a0) SAVE_U_PCB_CONTEXT(s6, PCB_REG_S6, a0) SAVE_U_PCB_CONTEXT(s7, PCB_REG_S7, a0) SAVE_U_PCB_CONTEXT(sp, PCB_REG_SP, a0) SAVE_U_PCB_CONTEXT(s8, PCB_REG_S8, a0) SAVE_U_PCB_CONTEXT(ra, PCB_REG_RA, a0) SAVE_U_PCB_CONTEXT(v0, PCB_REG_SR, a0) SAVE_U_PCB_CONTEXT(gp, PCB_REG_GP, a0) move v0, ra /* save 'ra' before we trash it */ jal 1f nop 1: SAVE_U_PCB_CONTEXT(ra, PCB_REG_PC, a0) move ra, v0 /* restore 'ra' before returning */ j ra move v0, zero END(savectx) NESTED(cpu_throw, CALLFRAME_SIZ, ra) mfc0 t0, MIPS_COP_0_STATUS # t0 = saved status register nop nop and a3, t0, ~(MIPS_SR_INT_IE) mtc0 a3, MIPS_COP_0_STATUS # Disable all interrupts ITLBNOPFIX j mips_sw1 # We're not interested in old # thread's context, so jump # right to action nop # BDSLOT END(cpu_throw) /* * cpu_switch(struct thread *old, struct thread *new, struct mutex *mtx); * a0 - old * a1 - new * a2 - mtx * Find the highest priority process and resume it. */ NESTED(cpu_switch, CALLFRAME_SIZ, ra) mfc0 t0, MIPS_COP_0_STATUS # t0 = saved status register nop nop and a3, t0, ~(MIPS_SR_INT_IE) mtc0 a3, MIPS_COP_0_STATUS # Disable all interrupts ITLBNOPFIX beqz a0, mips_sw1 move a3, a0 PTR_L a0, TD_PCB(a0) # load PCB addr of curproc SAVE_U_PCB_CONTEXT(sp, PCB_REG_SP, a0) # save old sp PTR_SUBU sp, sp, CALLFRAME_SIZ REG_S ra, CALLFRAME_RA(sp) .mask 0x80000000, (CALLFRAME_RA - CALLFRAME_SIZ) SAVE_U_PCB_CONTEXT(s0, PCB_REG_S0, a0) # do a 'savectx()' SAVE_U_PCB_CONTEXT(s1, PCB_REG_S1, a0) SAVE_U_PCB_CONTEXT(s2, PCB_REG_S2, a0) SAVE_U_PCB_CONTEXT(s3, PCB_REG_S3, a0) SAVE_U_PCB_CONTEXT(s4, PCB_REG_S4, a0) SAVE_U_PCB_CONTEXT(s5, PCB_REG_S5, a0) SAVE_U_PCB_CONTEXT(s6, PCB_REG_S6, a0) SAVE_U_PCB_CONTEXT(s7, PCB_REG_S7, a0) SAVE_U_PCB_CONTEXT(s8, PCB_REG_S8, a0) SAVE_U_PCB_CONTEXT(ra, PCB_REG_RA, a0) # save return address SAVE_U_PCB_CONTEXT(t0, PCB_REG_SR, a0) # save status register SAVE_U_PCB_CONTEXT(gp, PCB_REG_GP, a0) jal getpc nop getpc: SAVE_U_PCB_CONTEXT(ra, PCB_REG_PC, a0) # save return address #ifdef CPU_CNMIPS lw t2, TD_MDFLAGS(a3) # get md_flags and t1, t2, MDTD_COP2USED beqz t1, cop2_untouched nop /* Clear cop2used flag */ and t2, t2, ~MDTD_COP2USED sw t2, TD_MDFLAGS(a3) and t2, t0, ~MIPS_SR_COP_2_BIT # clear COP_2 enable bit SAVE_U_PCB_CONTEXT(t2, PCB_REG_SR, a0) # save status register RESTORE_U_PCB_REG(t0, PS, a0) # get CPU status register and t2, t0, ~MIPS_SR_COP_2_BIT # clear COP_2 enable bit SAVE_U_PCB_REG(t2, PS, a0) # save stratus register /* preserve a0..a3 */ move s0, a0 move s1, a1 move s2, a2 move s3, a3 /* does kernel own COP2 context? */ lw t1, TD_COP2OWNER(a3) # get md_cop2owner beqz t1, userland_cop2 # 0 - it's userland context nop PTR_L a0, TD_COP2(a3) beqz a0, no_cop2_context nop j do_cop2_save nop userland_cop2: PTR_L a0, TD_UCOP2(a3) beqz a0, no_cop2_context nop do_cop2_save: jal octeon_cop2_save nop no_cop2_context: move a3, s3 move a2, s2 move a1, s1 move a0, s0 cop2_untouched: #endif PTR_S a2, TD_LOCK(a3) # Switchout td_lock mips_sw1: #if defined(SMP) && defined(SCHED_ULE) PTR_LA t0, _C_LABEL(blocked_lock) blocked_loop: PTR_L t1, TD_LOCK(a1) beq t0, t1, blocked_loop nop #endif move s7, a1 # Store newthread /* * Switch to new context. */ GET_CPU_PCPU(a3) PTR_S a1, PC_CURTHREAD(a3) PTR_L a2, TD_PCB(a1) PTR_S a2, PC_CURPCB(a3) PTR_L v0, TD_KSTACK(a1) #if defined(__mips_n64) PTR_LI s0, MIPS_XKSEG_START #else PTR_LI s0, MIPS_KSEG2_START # If Uarea addr is below kseg2, #endif bltu v0, s0, sw2 # no need to insert in TLB. PTE_L a1, TD_UPTE + 0(s7) # a1 = u. pte #0 PTE_L a2, TD_UPTE + PTESIZE(s7) # a2 = u. pte #1 /* * Wiredown the USPACE of newproc in TLB entry#0. Check whether target * USPACE is already in another place of TLB before that, and if so * invalidate that TLB entry. * NOTE: This is hard coded to UPAGES == 2. * Also, there should be no TLB faults at this point. */ MTC0 v0, MIPS_COP_0_TLB_HI # VPN = va HAZARD_DELAY tlbp # probe VPN HAZARD_DELAY mfc0 s0, MIPS_COP_0_TLB_INDEX HAZARD_DELAY PTR_LI t1, MIPS_KSEG0_START # invalidate tlb entry bltz s0, entry0set nop sll s0, PAGE_SHIFT + 1 addu t1, s0 MTC0 t1, MIPS_COP_0_TLB_HI PTE_MTC0 zero, MIPS_COP_0_TLB_LO0 PTE_MTC0 zero, MIPS_COP_0_TLB_LO1 HAZARD_DELAY tlbwi HAZARD_DELAY MTC0 v0, MIPS_COP_0_TLB_HI # set VPN again entry0set: /* SMP!! - Works only for unshared TLB case - i.e. no v-cpus */ mtc0 zero, MIPS_COP_0_TLB_INDEX # TLB entry #0 HAZARD_DELAY PTE_MTC0 a1, MIPS_COP_0_TLB_LO0 # upte[0] HAZARD_DELAY PTE_MTC0 a2, MIPS_COP_0_TLB_LO1 # upte[1] HAZARD_DELAY tlbwi # set TLB entry #0 HAZARD_DELAY /* * Now running on new u struct. */ sw2: PTR_L s0, TD_PCB(s7) RESTORE_U_PCB_CONTEXT(sp, PCB_REG_SP, s0) PTR_LA t1, _C_LABEL(pmap_activate) # s7 = new proc pointer jalr t1 # s7 = new proc pointer move a0, s7 # BDSLOT /* * Restore registers and return. */ move a0, s0 move a1, s7 RESTORE_U_PCB_CONTEXT(gp, PCB_REG_GP, a0) RESTORE_U_PCB_CONTEXT(v0, PCB_REG_SR, a0) # restore kernel context RESTORE_U_PCB_CONTEXT(ra, PCB_REG_RA, a0) RESTORE_U_PCB_CONTEXT(s0, PCB_REG_S0, a0) RESTORE_U_PCB_CONTEXT(s1, PCB_REG_S1, a0) RESTORE_U_PCB_CONTEXT(s2, PCB_REG_S2, a0) RESTORE_U_PCB_CONTEXT(s3, PCB_REG_S3, a0) RESTORE_U_PCB_CONTEXT(s4, PCB_REG_S4, a0) RESTORE_U_PCB_CONTEXT(s5, PCB_REG_S5, a0) RESTORE_U_PCB_CONTEXT(s6, PCB_REG_S6, a0) RESTORE_U_PCB_CONTEXT(s7, PCB_REG_S7, a0) RESTORE_U_PCB_CONTEXT(s8, PCB_REG_S8, a0) mfc0 t0, MIPS_COP_0_STATUS and t0, t0, MIPS_SR_INT_MASK and v0, v0, ~MIPS_SR_INT_MASK or v0, v0, t0 mtc0 v0, MIPS_COP_0_STATUS ITLBNOPFIX /* * Set the new thread's TLS pointer. * * Note that this code is removed if the CPU doesn't support ULRI by * remove_userlocal_code() in cpu.c. */ PTR_L t0, TD_MDTLS(a1) # Get TLS pointer PTR_L t1, TD_MDTLS_TCB_OFFSET(a1) # Get TLS/TCB offset PTR_ADDU v0, t0, t1 MTC0 v0, MIPS_COP_0_USERLOCAL, 2 # write it to ULR for rdhwr j ra nop END(cpu_switch) /*---------------------------------------------------------------------------- * * MipsSwitchFPState -- * * Save the current state into 'from' and restore it from 'to'. * * MipsSwitchFPState(from, to) * struct thread *from; * struct trapframe *to; * * Results: * None. * * Side effects: * None. * *---------------------------------------------------------------------------- */ LEAF(MipsSwitchFPState) .set push .set hardfloat mfc0 t1, MIPS_COP_0_STATUS # Save old SR HAZARD_DELAY #if defined(__mips_n32) || defined(__mips_n64) or t0, t1, MIPS_SR_COP_1_BIT | MIPS_SR_FR # enable the coprocessor #else or t0, t1, MIPS_SR_COP_1_BIT # enable the coprocessor #endif mtc0 t0, MIPS_COP_0_STATUS HAZARD_DELAY ITLBNOPFIX beq a0, zero, 1f # skip save if NULL pointer nop /* * First read out the status register to make sure that all FP operations * have completed. */ PTR_L a0, TD_PCB(a0) # get pointer to pcb for proc cfc1 t0, MIPS_FPU_CSR # stall til FP done cfc1 t0, MIPS_FPU_CSR # now get status li t3, ~MIPS_SR_COP_1_BIT RESTORE_U_PCB_REG(t2, PS, a0) # get CPU status register SAVE_U_PCB_FPSR(t0, FSR_NUM, a0) # save FP status and t2, t2, t3 # clear COP_1 enable bit SAVE_U_PCB_REG(t2, PS, a0) # save new status register /* * Save the floating point registers. */ SAVE_U_PCB_FPREG($f0, F0_NUM, a0) SAVE_U_PCB_FPREG($f1, F1_NUM, a0) SAVE_U_PCB_FPREG($f2, F2_NUM, a0) SAVE_U_PCB_FPREG($f3, F3_NUM, a0) SAVE_U_PCB_FPREG($f4, F4_NUM, a0) SAVE_U_PCB_FPREG($f5, F5_NUM, a0) SAVE_U_PCB_FPREG($f6, F6_NUM, a0) SAVE_U_PCB_FPREG($f7, F7_NUM, a0) SAVE_U_PCB_FPREG($f8, F8_NUM, a0) SAVE_U_PCB_FPREG($f9, F9_NUM, a0) SAVE_U_PCB_FPREG($f10, F10_NUM, a0) SAVE_U_PCB_FPREG($f11, F11_NUM, a0) SAVE_U_PCB_FPREG($f12, F12_NUM, a0) SAVE_U_PCB_FPREG($f13, F13_NUM, a0) SAVE_U_PCB_FPREG($f14, F14_NUM, a0) SAVE_U_PCB_FPREG($f15, F15_NUM, a0) SAVE_U_PCB_FPREG($f16, F16_NUM, a0) SAVE_U_PCB_FPREG($f17, F17_NUM, a0) SAVE_U_PCB_FPREG($f18, F18_NUM, a0) SAVE_U_PCB_FPREG($f19, F19_NUM, a0) SAVE_U_PCB_FPREG($f20, F20_NUM, a0) SAVE_U_PCB_FPREG($f21, F21_NUM, a0) SAVE_U_PCB_FPREG($f22, F22_NUM, a0) SAVE_U_PCB_FPREG($f23, F23_NUM, a0) SAVE_U_PCB_FPREG($f24, F24_NUM, a0) SAVE_U_PCB_FPREG($f25, F25_NUM, a0) SAVE_U_PCB_FPREG($f26, F26_NUM, a0) SAVE_U_PCB_FPREG($f27, F27_NUM, a0) SAVE_U_PCB_FPREG($f28, F28_NUM, a0) SAVE_U_PCB_FPREG($f29, F29_NUM, a0) SAVE_U_PCB_FPREG($f30, F30_NUM, a0) SAVE_U_PCB_FPREG($f31, F31_NUM, a0) 1: /* * Restore the floating point registers. */ RESTORE_U_PCB_FPSR(t0, FSR_NUM, a1) # get status register RESTORE_U_PCB_FPREG($f0, F0_NUM, a1) RESTORE_U_PCB_FPREG($f1, F1_NUM, a1) RESTORE_U_PCB_FPREG($f2, F2_NUM, a1) RESTORE_U_PCB_FPREG($f3, F3_NUM, a1) RESTORE_U_PCB_FPREG($f4, F4_NUM, a1) RESTORE_U_PCB_FPREG($f5, F5_NUM, a1) RESTORE_U_PCB_FPREG($f6, F6_NUM, a1) RESTORE_U_PCB_FPREG($f7, F7_NUM, a1) RESTORE_U_PCB_FPREG($f8, F8_NUM, a1) RESTORE_U_PCB_FPREG($f9, F9_NUM, a1) RESTORE_U_PCB_FPREG($f10, F10_NUM, a1) RESTORE_U_PCB_FPREG($f11, F11_NUM, a1) RESTORE_U_PCB_FPREG($f12, F12_NUM, a1) RESTORE_U_PCB_FPREG($f13, F13_NUM, a1) RESTORE_U_PCB_FPREG($f14, F14_NUM, a1) RESTORE_U_PCB_FPREG($f15, F15_NUM, a1) RESTORE_U_PCB_FPREG($f16, F16_NUM, a1) RESTORE_U_PCB_FPREG($f17, F17_NUM, a1) RESTORE_U_PCB_FPREG($f18, F18_NUM, a1) RESTORE_U_PCB_FPREG($f19, F19_NUM, a1) RESTORE_U_PCB_FPREG($f20, F20_NUM, a1) RESTORE_U_PCB_FPREG($f21, F21_NUM, a1) RESTORE_U_PCB_FPREG($f22, F22_NUM, a1) RESTORE_U_PCB_FPREG($f23, F23_NUM, a1) RESTORE_U_PCB_FPREG($f24, F24_NUM, a1) RESTORE_U_PCB_FPREG($f25, F25_NUM, a1) RESTORE_U_PCB_FPREG($f26, F26_NUM, a1) RESTORE_U_PCB_FPREG($f27, F27_NUM, a1) RESTORE_U_PCB_FPREG($f28, F28_NUM, a1) RESTORE_U_PCB_FPREG($f29, F29_NUM, a1) RESTORE_U_PCB_FPREG($f30, F30_NUM, a1) RESTORE_U_PCB_FPREG($f31, F31_NUM, a1) and t0, t0, ~MIPS_FPU_EXCEPTION_BITS ctc1 t0, MIPS_FPU_CSR nop mtc0 t1, MIPS_COP_0_STATUS # Restore the status register. ITLBNOPFIX j ra nop .set pop END(MipsSwitchFPState) /*---------------------------------------------------------------------------- * * MipsFPID -- * * Read and return the floating point implementation register. * * uint32_t * MipsFPID(void) * * Results: * Floating point implementation register. * * Side effects: * None. * *---------------------------------------------------------------------------- */ LEAF(MipsFPID) .set push .set hardfloat mfc0 t1, MIPS_COP_0_STATUS # Save the status register. HAZARD_DELAY #if defined(__mips_n32) || defined(__mips_n64) or t0, t1, MIPS_SR_COP_1_BIT | MIPS_SR_FR #else or t0, t1, MIPS_SR_COP_1_BIT #endif mtc0 t0, MIPS_COP_0_STATUS # Enable the coprocessor HAZARD_DELAY ITLBNOPFIX cfc1 v0, MIPS_FPU_ID mtc0 t1, MIPS_COP_0_STATUS # Restore the status register. ITLBNOPFIX j ra nop .set pop END(MipsFPID) /*---------------------------------------------------------------------------- * * MipsSaveCurFPState -- * * Save the current floating point coprocessor state. * * MipsSaveCurFPState(td) * struct thread *td; * * Results: * None. * * Side effects: * machFPCurProcPtr is cleared. * *---------------------------------------------------------------------------- */ LEAF(MipsSaveCurFPState) .set push .set hardfloat PTR_L a0, TD_PCB(a0) # get pointer to pcb for thread mfc0 t1, MIPS_COP_0_STATUS # Disable interrupts and HAZARD_DELAY #if defined(__mips_n32) || defined(__mips_n64) or t0, t1, MIPS_SR_COP_1_BIT | MIPS_SR_FR # enable the coprocessor #else or t0, t1, MIPS_SR_COP_1_BIT # enable the coprocessor #endif mtc0 t0, MIPS_COP_0_STATUS HAZARD_DELAY ITLBNOPFIX GET_CPU_PCPU(a1) PTR_S zero, PC_FPCURTHREAD(a1) # indicate state has been saved /* * First read out the status register to make sure that all FP operations * have completed. */ RESTORE_U_PCB_REG(t2, PS, a0) # get CPU status register li t3, ~MIPS_SR_COP_1_BIT and t2, t2, t3 # clear COP_1 enable bit cfc1 t0, MIPS_FPU_CSR # stall til FP done cfc1 t0, MIPS_FPU_CSR # now get status SAVE_U_PCB_REG(t2, PS, a0) # save new status register SAVE_U_PCB_FPSR(t0, FSR_NUM, a0) # save FP status /* * Save the floating point registers. */ SAVE_U_PCB_FPREG($f0, F0_NUM, a0) SAVE_U_PCB_FPREG($f1, F1_NUM, a0) SAVE_U_PCB_FPREG($f2, F2_NUM, a0) SAVE_U_PCB_FPREG($f3, F3_NUM, a0) SAVE_U_PCB_FPREG($f4, F4_NUM, a0) SAVE_U_PCB_FPREG($f5, F5_NUM, a0) SAVE_U_PCB_FPREG($f6, F6_NUM, a0) SAVE_U_PCB_FPREG($f7, F7_NUM, a0) SAVE_U_PCB_FPREG($f8, F8_NUM, a0) SAVE_U_PCB_FPREG($f9, F9_NUM, a0) SAVE_U_PCB_FPREG($f10, F10_NUM, a0) SAVE_U_PCB_FPREG($f11, F11_NUM, a0) SAVE_U_PCB_FPREG($f12, F12_NUM, a0) SAVE_U_PCB_FPREG($f13, F13_NUM, a0) SAVE_U_PCB_FPREG($f14, F14_NUM, a0) SAVE_U_PCB_FPREG($f15, F15_NUM, a0) SAVE_U_PCB_FPREG($f16, F16_NUM, a0) SAVE_U_PCB_FPREG($f17, F17_NUM, a0) SAVE_U_PCB_FPREG($f18, F18_NUM, a0) SAVE_U_PCB_FPREG($f19, F19_NUM, a0) SAVE_U_PCB_FPREG($f20, F20_NUM, a0) SAVE_U_PCB_FPREG($f21, F21_NUM, a0) SAVE_U_PCB_FPREG($f22, F22_NUM, a0) SAVE_U_PCB_FPREG($f23, F23_NUM, a0) SAVE_U_PCB_FPREG($f24, F24_NUM, a0) SAVE_U_PCB_FPREG($f25, F25_NUM, a0) SAVE_U_PCB_FPREG($f26, F26_NUM, a0) SAVE_U_PCB_FPREG($f27, F27_NUM, a0) SAVE_U_PCB_FPREG($f28, F28_NUM, a0) SAVE_U_PCB_FPREG($f29, F29_NUM, a0) SAVE_U_PCB_FPREG($f30, F30_NUM, a0) SAVE_U_PCB_FPREG($f31, F31_NUM, a0) mtc0 t1, MIPS_COP_0_STATUS # Restore the status register. ITLBNOPFIX j ra nop .set pop END(MipsSaveCurFPState) /* * This code is copied the user's stack for returning from signal handlers * (see sendsig() and sigreturn()). We have to compute the address * of the sigcontext struct for the sigreturn call. */ .globl _C_LABEL(sigcode) _C_LABEL(sigcode): PTR_ADDU a0, sp, SIGF_UC # address of ucontext li v0, SYS_sigreturn # sigreturn (ucp) syscall break 0 # just in case sigreturn fails .globl _C_LABEL(esigcode) _C_LABEL(esigcode): .data .globl szsigcode szsigcode: .long esigcode-sigcode .text #if (defined(__mips_n32) || defined(__mips_n64)) && defined(COMPAT_FREEBSD32) .globl _C_LABEL(sigcode32) _C_LABEL(sigcode32): addu a0, sp, SIGF32_UC # address of ucontext li v0, SYS_sigreturn # sigreturn (ucp) syscall break 0 # just in case sigreturn fails .globl _C_LABEL(esigcode32) _C_LABEL(esigcode32): .data .globl szsigcode32 szsigcode32: .long esigcode32-sigcode32 .text #endif Index: head/sys/mips/nlm/mpreset.S =================================================================== --- head/sys/mips/nlm/mpreset.S (revision 331253) +++ head/sys/mips/nlm/mpreset.S (revision 331254) @@ -1,201 +1,201 @@ /*- * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. * * NETLOGIC_BSD * $FreeBSD$ */ #include #include #include #include #include #include #define SYS_REG_KSEG1(node, reg) (0xa0000000 + XLP_DEFAULT_IO_BASE + \ XLP_IO_SYS_OFFSET(node) + XLP_IO_PCI_HDRSZ + (reg) * 4) -#include "assym.s" +#include "assym.inc" .text .set noat .set noreorder .set mips64 #define MFCR(rt,rs) .word ((0x1c<<26)|((rs)<<21)|((rt)<<16)|(0x18)) #define MTCR(rt,rs) .word ((0x1c<<26)|((rs)<<21)|((rt)<<16)|(0x19)) /* * We need to do this to really flush the dcache before splitting it */ .macro flush_l1_dcache .set push .set noreorder li $8, LSU_DEBUG_DATA0 /* use register number to handle */ li $9, LSU_DEBUG_ADDR /* different ABIs */ li t2, 0 /* index */ li t3, 0x1000 /* loop count, 512 sets * 8 whatever? */ 1: sll v0, t2, 5 MTCR(0, 8) ori v1, v0, 0x3 /* way0 | write_enable | write_active */ MTCR(3, 9) 2: MFCR(3, 9) andi v1, 0x1 /* wait for write_active == 0 */ bnez v1, 2b nop MTCR(0, 8) ori v1, v0, 0x7 /* way1 | write_enable | write_active */ MTCR(3, 9) 3: MFCR(3, 9) andi v1, 0x1 /* wait for write_active == 0 */ bnez v1, 3b nop addi t2, 1 bne t3, t2, 1b nop .set pop .endm VECTOR(XLPResetEntry, unknown) mfc0 t0, MIPS_COP_0_STATUS li t1, 0x80000 and t1, t0, t1 bnez t1, nmi_handler nop #ifdef SMP /* Reset entry for secordary cores */ mfc0 t0, MIPS_COP_0_PRID, 1 srl t0, t0, 2 /* discard thread id */ andi t0, t0, 0x7 /* core id */ li t1, 1 sll t0, t1, t0 nor t0, t0, zero /* mask with core id bit clear */ /* clear CPU non-coherent bit */ li t2, SYS_REG_KSEG1(0, SYS_CPU_NONCOHERENT_MODE) lw t1, 0(t2) and t1, t1, t0 sw t1, 0(t2) lw t1, 0(t2) /* read-back ensures operation complete */ sync dla t2, mpentry jr t2 nop #endif nop /* NOT REACHED */ VECTOR_END(XLPResetEntry) /* Not yet */ nmi_handler: nop nop j nmi_handler #ifdef SMP /* * Enable other threads in the core, called from thread 0 * of the core */ LEAF(xlp_enable_threads) /* * Save and restore callee saved registers of all ABIs * Enabling threads trashes the registers */ dmtc0 sp, $4, 2 /* SP saved in UserLocal */ ori sp, sp, 0x7 xori sp, sp, 0x7 /* align 64 bit */ addiu sp, sp, -128 mfc0 t1, MIPS_COP_0_STATUS sd s0, 0(sp) sd s1, 8(sp) sd s2, 16(sp) sd s3, 24(sp) sd s4, 32(sp) sd s5, 40(sp) sd s6, 48(sp) sd s7, 56(sp) sd s8, 64(sp) sd t1, 72(sp) sd gp, 80(sp) sd ra, 88(sp) flush_l1_dcache /* Use register number to work in o32 and n32 */ li $9, ((CPU_BLOCKID_MAP << 8) | MAP_THREADMODE) move $8, a0 sync MTCR(8, 9) mfc0 t0, MIPS_COP_0_PRID, 1 andi t0, 0x3 beqz t0, 2f nop dla t1, mpentry /* child thread, go to hardware init */ jr t1 nop 2: /* * Parent hardware thread, restore registers, return */ #if 1 /* * A0 Errata - Write MMU_SETUP after changing thread mode register. */ li $9, 0x400 li $8, 0 MTCR(8, 9) sll zero,3 /* ehb */ #endif dmfc0 t0, $4, 2 /* SP saved in UserLocal */ ori sp, t0, 0x7 xori sp, sp, 0x7 /* align 64 bit */ addiu sp, sp, -128 ld s0, 0(sp) ld s1, 8(sp) ld s2, 16(sp) ld s3, 24(sp) ld s4, 32(sp) ld s5, 40(sp) ld s6, 48(sp) ld s7, 56(sp) ld s8, 64(sp) ld t1, 72(sp) ld gp, 80(sp) ld ra, 88(sp) mfc0 t1, MIPS_COP_0_STATUS move sp, t0 /* Restore the real SP */ jr.hb ra nop END(xlp_enable_threads) #endif Index: head/sys/modules/dtrace/dtrace/Makefile =================================================================== --- head/sys/modules/dtrace/dtrace/Makefile (revision 331253) +++ head/sys/modules/dtrace/dtrace/Makefile (revision 331254) @@ -1,65 +1,65 @@ # $FreeBSD$ SYSDIR?= ${SRCTOP}/sys ARCHDIR= ${MACHINE_CPUARCH} .PATH: ${SYSDIR}/cddl/contrib/opensolaris/uts/common/dtrace .PATH: ${SYSDIR}/cddl/compat/opensolaris/kern .PATH: ${SYSDIR}/cddl/kern .PATH: ${SYSDIR}/cddl/dev/dtrace .PATH: ${SYSDIR}/cddl/dev/dtrace/${ARCHDIR} KMOD= dtrace SRCS= dtrace.c \ dtrace_xoroshiro128_plus.c \ dtrace_asm.S \ dtrace_subr.c .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" .PATH: ${SYSDIR}/cddl/dev/dtrace/x86 SRCS+= dis_tables.c \ instr_size.c CFLAGS+= -I${SYSDIR}/cddl/contrib/opensolaris/uts/intel \ -I${SYSDIR}/cddl/dev/dtrace/x86 .endif SRCS+= bus_if.h device_if.h vnode_if.h # Needed for dtrace_asm.S -DPSRCS+= assym.s +DPSRCS+= assym.inc -# These are needed for assym.s +# These are needed for assym.inc SRCS+= opt_compat.h opt_kstack_pages.h opt_nfs.h opt_hwpmc_hooks.h #This is needed for dtrace.c SRCS += opensolaris_taskq.c .if ${MACHINE_CPUARCH} == "i386" SRCS+= opt_apic.h .endif CFLAGS+= -I${SYSDIR}/cddl/compat/opensolaris \ -I${SYSDIR}/cddl/dev/dtrace \ -I${SYSDIR}/cddl/dev/dtrace/${ARCHDIR} \ -I${SYSDIR}/cddl/contrib/opensolaris/uts/common \ -I${SYSDIR}/cddl/contrib/opensolaris/uts/common/dtrace \ -I${SYSDIR}/cddl/contrib/opensolaris/common/util \ -I${SYSDIR} -DDIS_MEM CFLAGS+= -DSMP EXPORT_SYMS= dtrace_register \ dtrace_unregister \ dtrace_probe_lookup -dtrace_asm.o: assym.s +dtrace_asm.o: assym.inc .include CFLAGS+= -include ${SYSDIR}/cddl/compat/opensolaris/sys/debug_compat.h CWARNFLAGS+= -Wno-parentheses CWARNFLAGS+= -Wno-uninitialized CWARNFLAGS+= -Wno-cast-qual CWARNFLAGS+= -Wno-unused Index: head/sys/modules/hyperv/vmbus/Makefile =================================================================== --- head/sys/modules/hyperv/vmbus/Makefile (revision 331253) +++ head/sys/modules/hyperv/vmbus/Makefile (revision 331254) @@ -1,35 +1,35 @@ # $FreeBSD$ .PATH: ${SRCTOP}/sys/dev/hyperv/vmbus \ ${SRCTOP}/sys/dev/hyperv/vmbus/${MACHINE_CPUARCH} KMOD= hv_vmbus SRCS= hyperv.c \ hyperv_busdma.c \ hyperv_machdep.c \ vmbus.c \ vmbus_br.c \ vmbus_chan.c \ vmbus_et.c \ vmbus_if.c \ vmbus_res.c \ vmbus_vector.S \ vmbus_xact.c SRCS+= acpi_if.h bus_if.h device_if.h opt_acpi.h pci_if.h pcib_if.h vmbus_if.h -# XXX: for assym.s +# XXX: for assym.inc SRCS+= opt_kstack_pages.h opt_nfs.h opt_hwpmc_hooks.h opt_compat.h .if ${MACHINE_CPUARCH} == "i386" SRCS+= opt_apic.h .endif -DPSRCS+= assym.s +DPSRCS= assym.inc vmbus_vector.o: ${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \ ${.IMPSRC} -o ${.TARGET} CFLAGS+= -I${SRCTOP}/sys/dev/hyperv/include \ -I${SRCTOP}/sys/dev/hyperv/vmbus .include Index: head/sys/modules/linux/Makefile =================================================================== --- head/sys/modules/linux/Makefile (revision 331253) +++ head/sys/modules/linux/Makefile (revision 331254) @@ -1,87 +1,87 @@ # $FreeBSD$ .if ${MACHINE_CPUARCH} == "amd64" SFX= 32 CFLAGS+=-DCOMPAT_FREEBSD32 -DCOMPAT_LINUX32 .endif .PATH: ${SRCTOP}/sys/compat/linux ${SRCTOP}/sys/${MACHINE_CPUARCH}/linux${SFX} VDSO= linux${SFX}_vdso KMOD= linux SRCS= linux_fork.c linux${SFX}_dummy.c linux_file.c linux_event.c \ linux_futex.c linux_getcwd.c linux_ioctl.c linux_ipc.c \ linux${SFX}_machdep.c linux_misc.c linux_signal.c \ linux_socket.c linux_stats.c linux_sysctl.c linux${SFX}_sysent.c \ linux${SFX}_sysvec.c linux_uid16.c linux_time.c \ linux_timer.c linux_vdso.c \ opt_inet6.h opt_compat.h opt_posix.h opt_usb.h vnode_if.h \ device_if.h bus_if.h \ linux${SFX}_support.s -DPSRCS= assym.s linux${SFX}_genassym.c +DPSRCS= assym.inc linux${SFX}_genassym.c -# XXX: for assym.s +# XXX: for assym.inc SRCS+= opt_kstack_pages.h opt_nfs.h opt_compat.h opt_hwpmc_hooks.h .if ${MACHINE_CPUARCH} == "i386" SRCS+= opt_apic.h .endif OBJS= ${VDSO}.so .if ${MACHINE_CPUARCH} == "i386" SRCS+= linux_ptrace.c imgact_linux.c linux_util.c linux_mib.c linux_mmap.c \ linux_emul.c linux_errno.c opt_cpu.h linux.c .endif .if ${MACHINE_CPUARCH} == "i386" EXPORT_SYMS= EXPORT_SYMS+= linux_emul_path EXPORT_SYMS+= linux_get_osname EXPORT_SYMS+= linux_get_osrelease EXPORT_SYMS+= linux_ioctl_register_handler EXPORT_SYMS+= linux_ioctl_unregister_handler .endif CLEANFILES= linux${SFX}_assym.h linux${SFX}_genassym.o linux${SFX}_locore.o linux${SFX}_assym.h: linux${SFX}_genassym.o sh ${SYSDIR}/kern/genassym.sh linux${SFX}_genassym.o > ${.TARGET} -linux${SFX}_locore.o: linux${SFX}_assym.h assym.s +linux${SFX}_locore.o: linux${SFX}_assym.h assym.inc ${CC} -x assembler-with-cpp -DLOCORE -m32 -shared -s \ -pipe -I. -I${SYSDIR} -Werror -Wall -fno-common -nostdinc -nostdlib \ -fno-omit-frame-pointer -fPIC \ -Wl,-T${SRCTOP}/sys/${MACHINE_CPUARCH}/linux${SFX}/${VDSO}.lds.s \ -Wl,-soname=${VDSO}.so.1,--eh-frame-hdr,-warn-common \ ${.IMPSRC} -o ${.TARGET} -linux${SFX}_support.o: linux${SFX}_assym.h assym.s +linux${SFX}_support.o: linux${SFX}_assym.h assym.inc ${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \ ${.IMPSRC} -o ${.TARGET} .if ${MACHINE_CPUARCH} == "amd64" ${VDSO}.so: linux${SFX}_locore.o ${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd \ --binary-architecture i386 linux${SFX}_locore.o ${.TARGET} strip -N _binary_linux${SFX}_locore_o_size ${.TARGET} .else ${VDSO}.so: linux${SFX}_locore.o ${OBJCOPY} --input-target binary --output-target elf32-i386-freebsd \ --binary-architecture i386 linux${SFX}_locore.o ${.TARGET} strip -N _binary_linux_locore_o_size ${.TARGET} .endif linux${SFX}_genassym.o: ${CC} -c ${CFLAGS:N-flto:N-fno-common} ${.IMPSRC} .if !defined(KERNBUILDDIR) .if defined(KTR) CFLAGS+= -DKTR .endif .if defined(DEBUG) CFLAGS+= -DDEBUG .endif .endif .include Index: head/sys/modules/linux64/Makefile =================================================================== --- head/sys/modules/linux64/Makefile (revision 331253) +++ head/sys/modules/linux64/Makefile (revision 331254) @@ -1,59 +1,59 @@ # $FreeBSD$ .PATH: ${SRCTOP}/sys/compat/linux ${SRCTOP}/sys/${MACHINE}/linux VDSO= linux_vdso KMOD= linux64 SRCS= linux_fork.c linux_dummy.c linux_file.c linux_event.c \ linux_futex.c linux_getcwd.c linux_ioctl.c linux_ipc.c \ linux_machdep.c linux_misc.c linux_ptrace.c linux_signal.c \ linux_socket.c linux_stats.c linux_sysctl.c linux_sysent.c \ linux_sysvec.c linux_time.c linux_vdso.c linux_timer.c \ opt_inet6.h opt_compat.h opt_posix.h opt_usb.h \ vnode_if.h device_if.h bus_if.h \ linux_support.s -DPSRCS= assym.s linux_genassym.c +DPSRCS= assym.inc linux_genassym.c -# XXX: for assym.s +# XXX: for assym.inc SRCS+= opt_kstack_pages.h opt_nfs.h opt_hwpmc_hooks.h .if ${MACHINE_CPUARCH} == "i386" SRCS+= opt_apic.h .endif CLEANFILES= linux_assym.h linux_genassym.o linux_locore.o OBJS= ${VDSO}.so linux_assym.h: linux_genassym.o sh ${SYSDIR}/kern/genassym.sh linux_genassym.o > ${.TARGET} linux_locore.o: linux_locore.s linux_assym.h ${CC} -x assembler-with-cpp -DLOCORE -shared -mcmodel=small \ -pipe -I. -I${SYSDIR} -Werror -Wall -fno-common -fPIC -nostdinc \ -Wl,-T${SRCTOP}/sys/${MACHINE}/linux/${VDSO}.lds.s \ -Wl,-soname=${VDSO}.so.1,-warn-common -nostdlib \ ${.IMPSRC} -o ${.TARGET} ${VDSO}.so: linux_locore.o ${OBJCOPY} --input-target binary --output-target elf64-x86-64-freebsd \ -S -g --binary-architecture i386:x86-64 linux_locore.o ${.TARGET} strip -N _binary_linux_locore_o_size ${.TARGET} -linux_support.o: assym.s linux_assym.h +linux_support.o: assym.inc linux_assym.h ${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \ ${.IMPSRC} -o ${.TARGET} linux_genassym.o: ${CC} -c ${CFLAGS:N-flto:N-fno-common} ${.IMPSRC} .if !defined(KERNBUILDDIR) .if defined(DEBUG) CFLAGS+=-DDEBUG .endif .if defined(KTR) CFLAGS+=-DKTR .endif .endif .include Index: head/sys/modules/sgx/Makefile =================================================================== --- head/sys/modules/sgx/Makefile (revision 331253) +++ head/sys/modules/sgx/Makefile (revision 331254) @@ -1,14 +1,14 @@ # $FreeBSD$ .PATH: ${SRCTOP}/sys/amd64/sgx KMOD= sgx SRCS= sgx.c sgxvar.h sgx_support.S SRCS+= opt_compat.h opt_hwpmc_hooks.h opt_kstack_pages.h -DPSRCS= assym.s +DPSRCS= assym.inc -sgx_support.o: assym.s +sgx_support.o: assym.inc ${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \ ${.IMPSRC} -o ${.TARGET} .include Index: head/sys/powerpc/aim/locore32.S =================================================================== --- head/sys/powerpc/aim/locore32.S (revision 331253) +++ head/sys/powerpc/aim/locore32.S (revision 331254) @@ -1,117 +1,117 @@ /* $FreeBSD$ */ /*- * Copyright (C) 2010-2016 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "assym.s" +#include "assym.inc" #include #include #include #include #include #include "opt_platform.h" /* Locate the per-CPU data structure */ #define GET_CPUINFO(r) \ mfsprg0 r /* * Compiled KERNBASE location and the kernel load address */ .globl kernbase .set kernbase, KERNBASE /* * Globals */ .data .align 3 GLOBAL(__startkernel) .long begin GLOBAL(__endkernel) .long end .align 4 #define TMPSTKSZ 8192 /* 8K temporary stack */ GLOBAL(tmpstk) .space TMPSTKSZ .text .globl btext btext: /* * Main kernel entry point. */ .text .globl __start __start: /* Figure out where we are */ bl 1f .long _DYNAMIC-. .long _GLOBAL_OFFSET_TABLE_-. .long tmpstk-. 1: mflr %r30 /* Set up temporary stack pointer */ lwz %r1,8(%r30) add %r1,%r1,%r30 addi %r1,%r1,(8+TMPSTKSZ-40) /* Relocate self */ stw %r3,16(%r1) stw %r4,20(%r1) stw %r5,24(%r1) stw %r6,28(%r1) stw %r7,32(%r1) lwz %r3,0(%r30) /* _DYNAMIC in %r3 */ add %r3,%r3,%r30 lwz %r4,4(%r30) /* GOT pointer */ add %r4,%r4,%r30 lwz %r4,4(%r4) /* got[0] is _DYNAMIC link addr */ subf %r4,%r4,%r3 /* subtract to calculate relocbase */ bl elf_reloc_self lwz %r3,16(%r1) lwz %r4,20(%r1) lwz %r5,24(%r1) lwz %r6,28(%r1) lwz %r7,32(%r1) /* MD setup */ bl powerpc_init /* Set stack pointer to new value and branch to mi_startup */ mr %r1, %r3 li %r3, 0 stw %r3, 0(%r1) bl mi_startup /* mi_startup() does not return */ b . #include Index: head/sys/powerpc/aim/locore64.S =================================================================== --- head/sys/powerpc/aim/locore64.S (revision 331253) +++ head/sys/powerpc/aim/locore64.S (revision 331254) @@ -1,210 +1,210 @@ /* $FreeBSD$ */ /*- * Copyright (C) 2010-2016 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ -#include "assym.s" +#include "assym.inc" #include #include #include #include #include #ifdef _CALL_ELF .abiversion _CALL_ELF #endif /* Glue for linker script */ .globl kernbase .set kernbase, KERNBASE /* * Globals */ .data .align 3 GLOBAL(__startkernel) .llong begin GLOBAL(__endkernel) .llong end GLOBAL(can_wakeup) .llong 0x0 .align 4 #define TMPSTKSZ 16384 /* 16K temporary stack */ GLOBAL(tmpstk) .space TMPSTKSZ TOC_ENTRY(tmpstk) TOC_ENTRY(can_wakeup) /* * Entry point for bootloaders that do not fully implement ELF and start * at the beginning of the image (kexec, notably). In its own section so * that it ends up before any linker-generated call stubs and actually at * the beginning of the image. kexec on some systems also enters at * (start of image) + 0x60, so put a spin loop there. */ .section ".text.kboot", "x", @progbits kbootentry: b __start . = kbootentry + 0x40 /* Magic address used in platform layer */ .global smp_spin_sem ap_kexec_spin_sem: .long -1 . = kbootentry + 0x60 /* Entry point for kexec APs */ ap_kexec_start: /* At 0x60 past start, copied to 0x60 by kexec */ /* r3 set to CPU ID by kexec */ /* Invalidate icache for low-memory copy and jump there */ li %r0,0x80 dcbst 0,%r0 sync icbi 0,%r0 isync ba 0x80 /* Absolute branch to next inst */ . = kbootentry + 0x80 /* Aligned to cache line */ 1: or 31,31,31 /* yield */ sync lwz %r1,0x40(0) /* Spin on ap_kexec_spin_sem */ cmpw %r1,%r3 /* Until it equals our CPU ID */ bne 1b /* Released */ or 2,2,2 /* unyield */ /* Make sure that it will be software reset. Clear SRR1 */ li %r1,0 mtsrr1 %r1 ba EXC_RST /* * Now start the real text section */ .text .globl btext btext: /* * Main kernel entry point. * * Calling convention: * r3: Flattened Device Tree pointer (or zero) * r4: ignored * r5: OF client interface pointer (or zero) * r6: Loader metadata pointer (or zero) * r7: Magic cookie (0xfb5d104d) to indicate that r6 has loader metadata */ .text ASENTRY_NOPROF(__start) /* Set 64-bit mode if not yet set before branching to C */ mfmsr %r20 li %r21,1 insrdi %r20,%r21,1,0 mtmsrd %r20 isync nop /* Make this block a multiple of 8 bytes */ /* Set up the TOC pointer */ b 0f .align 3 0: nop bl 1f .llong __tocbase + 0x8000 - . 1: mflr %r2 ld %r1,0(%r2) add %r2,%r1,%r2 /* Get load offset */ ld %r31,-0x8000(%r2) /* First TOC entry is TOC base */ subf %r31,%r31,%r2 /* Subtract from real TOC base to get base */ /* Set up the stack pointer */ bl 1f .llong tmpstk + TMPSTKSZ - 96 - . 1: mflr %r30 ld %r1,0(%r30) add %r1,%r1,%r30 nop /* Relocate kernel */ std %r3,48(%r1) std %r4,56(%r1) std %r5,64(%r1) std %r6,72(%r1) std %r7,80(%r1) bl 1f .llong _DYNAMIC-. 1: mflr %r3 ld %r4,0(%r3) add %r3,%r4,%r3 mr %r4,%r31 bl elf_reloc_self nop ld %r3,48(%r1) ld %r4,56(%r1) ld %r5,64(%r1) ld %r6,72(%r1) ld %r7,80(%r1) /* Begin CPU init */ mr %r4,%r2 /* Replace ignored r4 with tocbase for trap handlers */ bl powerpc_init nop /* Set stack pointer to new value and branch to mi_startup */ mr %r1, %r3 li %r3, 0 std %r3, 0(%r1) bl mi_startup nop /* Unreachable */ b . ASENTRY_NOPROF(__restartkernel) /* * r3-r7: arguments to go to __start * r8: offset from current kernel address to apply * r9: MSR to set when (atomically) jumping to __start + r8 */ mtsrr1 %r9 bl 1f 1: mflr %r25 add %r25,%r8,%r25 addi %r25,%r25,2f-1b mtsrr0 %r25 rfid 2: bl __start nop #include Index: head/sys/powerpc/booke/locore.S =================================================================== --- head/sys/powerpc/booke/locore.S (revision 331253) +++ head/sys/powerpc/booke/locore.S (revision 331254) @@ -1,1004 +1,1004 @@ /*- * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski * Copyright (C) 2006 Semihalf, Marian Balakowicz * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ -#include "assym.s" +#include "assym.inc" #include "opt_hwpmc_hooks.h" #include #include #include #include #include #include #include #include #define TMPSTACKSZ 16384 #ifdef __powerpc64__ #define GET_TOCBASE(r) \ mfspr r, SPR_SPRG8 #define TOC_RESTORE nop #define CMPI cmpdi #define CMPL cmpld #define LOAD ld #define LOADX ldarx #define STORE std #define STOREX stdcx. #define STU stdu #define CALLSIZE 48 #define REDZONE 288 #define THREAD_REG %r13 #define ADDR(x) \ .llong x #define WORD_SIZE 8 #else #define GET_TOCBASE(r) #define TOC_RESTORE #define CMPI cmpwi #define CMPL cmplw #define LOAD lwz #define LOADX lwarx #define STOREX stwcx. #define STORE stw #define STU stwu #define CALLSIZE 8 #define REDZONE 0 #define THREAD_REG %r2 #define ADDR(x) \ .long x #define WORD_SIZE 4 #endif .text .globl btext btext: /* * This symbol is here for the benefit of kvm_mkdb, and is supposed to * mark the start of kernel text. */ .globl kernel_text kernel_text: /* * Startup entry. Note, this must be the first thing in the text segment! */ .text .globl __start __start: /* * Assumptions on the boot loader: * - System memory starts from physical address 0 * - It's mapped by a single TLB1 entry * - TLB1 mapping is 1:1 pa to va * - Kernel is loaded at 64MB boundary * - All PID registers are set to the same value * - CPU is running in AS=0 * * Registers contents provided by the loader(8): * r1 : stack pointer * r3 : metadata pointer * * We rearrange the TLB1 layout as follows: * - Find TLB1 entry we started in * - Make sure it's protected, invalidate other entries * - Create temp entry in the second AS (make sure it's not TLB[1]) * - Switch to temp mapping * - Map 64MB of RAM in TLB1[1] * - Use AS=1, set EPN to KERNBASE and RPN to kernel load address * - Switch to TLB1[1] mapping * - Invalidate temp mapping * * locore registers use: * r1 : stack pointer * r2 : trace pointer (AP only, for early diagnostics) * r3-r27 : scratch registers * r28 : temp TLB1 entry * r29 : initial TLB1 entry we started in * r30-r31 : arguments (metadata pointer) */ /* * Keep arguments in r30 & r31 for later use. */ mr %r30, %r3 mr %r31, %r4 /* * Initial cleanup */ li %r3, PSL_DE /* Keep debug exceptions for CodeWarrior. */ #ifdef __powerpc64__ oris %r3, %r3, PSL_CM@h #endif mtmsr %r3 isync /* * Initial HIDs configuration */ 1: mfpvr %r3 rlwinm %r3, %r3, 16, 16, 31 lis %r4, HID0_E500_DEFAULT_SET@h ori %r4, %r4, HID0_E500_DEFAULT_SET@l /* Check for e500mc and e5500 */ cmpli 0, 0, %r3, FSL_E500mc bne 2f lis %r4, HID0_E500MC_DEFAULT_SET@h ori %r4, %r4, HID0_E500MC_DEFAULT_SET@l b 3f 2: cmpli 0, 0, %r3, FSL_E5500 bne 3f lis %r4, HID0_E5500_DEFAULT_SET@h ori %r4, %r4, HID0_E5500_DEFAULT_SET@l 3: mtspr SPR_HID0, %r4 isync /* * E500mc and E5500 do not have HID1 register, so skip HID1 setup on * this core. */ cmpli 0, 0, %r3, FSL_E500mc beq 1f cmpli 0, 0, %r3, FSL_E5500 beq 1f cmpli 0, 0, %r3, FSL_E6500 beq 1f lis %r3, HID1_E500_DEFAULT_SET@h ori %r3, %r3, HID1_E500_DEFAULT_SET@l mtspr SPR_HID1, %r3 isync 1: /* Invalidate all entries in TLB0 */ li %r3, 0 bl tlb_inval_all cmpwi %r30, 0 beq done_mapping /* * Locate the TLB1 entry that maps this code */ bl 1f 1: mflr %r3 bl tlb1_find_current /* the entry found is returned in r29 */ bl tlb1_inval_all_but_current /* * Create temporary mapping in AS=1 and switch to it */ bl tlb1_temp_mapping_as1 mfmsr %r3 ori %r3, %r3, (PSL_IS | PSL_DS) bl 2f 2: mflr %r4 addi %r4, %r4, (3f - 2b) mtspr SPR_SRR0, %r4 mtspr SPR_SRR1, %r3 rfi /* Switch context */ /* * Invalidate initial entry */ 3: mr %r3, %r29 bl tlb1_inval_entry /* * Setup final mapping in TLB1[1] and switch to it */ /* Final kernel mapping, map in 64 MB of RAM */ lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */ li %r4, 0 /* Entry 0 */ rlwimi %r3, %r4, 16, 10, 15 mtspr SPR_MAS0, %r3 isync li %r3, (TLB_SIZE_64M << MAS1_TSIZE_SHIFT)@l oris %r3, %r3, (MAS1_VALID | MAS1_IPROT)@h mtspr SPR_MAS1, %r3 /* note TS was not filled, so it's TS=0 */ isync LOAD_ADDR(%r3, KERNBASE) ori %r3, %r3, (_TLB_ENTRY_SHARED | MAS2_M)@l /* WIMGE = 0b00100 */ mtspr SPR_MAS2, %r3 isync /* Discover phys load address */ bl 3f 3: mflr %r4 /* Use current address */ rlwinm %r4, %r4, 0, 0, 5 /* 64MB alignment mask */ ori %r4, %r4, (MAS3_SX | MAS3_SW | MAS3_SR)@l mtspr SPR_MAS3, %r4 /* Set RPN and protection */ isync bl zero_mas7 bl zero_mas8 tlbwe isync msync /* Switch to the above TLB1[1] mapping */ bl 4f 4: mflr %r4 #ifdef __powerpc64__ clrldi %r4, %r4, 38 clrrdi %r3, %r3, 12 #else rlwinm %r4, %r4, 0, 6, 31 /* Current offset from kernel load address */ rlwinm %r3, %r3, 0, 0, 19 #endif add %r4, %r4, %r3 /* Convert to kernel virtual address */ addi %r4, %r4, (5f - 4b) li %r3, PSL_DE /* Note AS=0 */ #ifdef __powerpc64__ oris %r3, %r3, PSL_CM@h #endif mtspr SPR_SRR0, %r4 mtspr SPR_SRR1, %r3 rfi /* * Invalidate temp mapping */ 5: mr %r3, %r28 bl tlb1_inval_entry done_mapping: #ifdef __powerpc64__ /* Set up the TOC pointer */ b 0f .align 3 0: nop bl 1f .llong __tocbase + 0x8000 - . 1: mflr %r2 ld %r1,0(%r2) add %r2,%r1,%r2 mtspr SPR_SPRG8, %r2 /* Get load offset */ ld %r31,-0x8000(%r2) /* First TOC entry is TOC base */ subf %r31,%r31,%r2 /* Subtract from real TOC base to get base */ /* Set up the stack pointer */ ld %r1,TOC_REF(tmpstack)(%r2) addi %r1,%r1,TMPSTACKSZ-96 add %r1,%r1,%r31 bl 1f .llong _DYNAMIC-. 1: mflr %r3 ld %r4,0(%r3) add %r3,%r4,%r3 mr %r4,%r31 #else /* * Setup a temporary stack */ bl 1f .long tmpstack-. 1: mflr %r1 lwz %r2,0(%r1) add %r1,%r1,%r2 addi %r1, %r1, (TMPSTACKSZ - 16) /* * Relocate kernel */ bl 1f .long _DYNAMIC-. .long _GLOBAL_OFFSET_TABLE_-. 1: mflr %r5 lwz %r3,0(%r5) /* _DYNAMIC in %r3 */ add %r3,%r3,%r5 lwz %r4,4(%r5) /* GOT pointer */ add %r4,%r4,%r5 lwz %r4,4(%r4) /* got[0] is _DYNAMIC link addr */ subf %r4,%r4,%r3 /* subtract to calculate relocbase */ #endif bl CNAME(elf_reloc_self) TOC_RESTORE /* * Initialise exception vector offsets */ bl CNAME(ivor_setup) TOC_RESTORE /* * Set up arguments and jump to system initialization code */ mr %r3, %r30 mr %r4, %r31 /* Prepare core */ bl CNAME(booke_init) TOC_RESTORE /* Switch to thread0.td_kstack now */ mr %r1, %r3 li %r3, 0 STORE %r3, 0(%r1) /* Machine independet part, does not return */ bl CNAME(mi_startup) TOC_RESTORE /* NOT REACHED */ 5: b 5b #ifdef SMP /************************************************************************/ /* AP Boot page */ /************************************************************************/ .text .globl __boot_page .align 12 __boot_page: bl 1f .globl bp_trace bp_trace: .long 0 .globl bp_kernload bp_kernload: .long 0 /* * Initial configuration */ 1: mflr %r31 /* r31 hold the address of bp_trace */ /* Set HIDs */ mfpvr %r3 rlwinm %r3, %r3, 16, 16, 31 /* HID0 for E500 is default */ lis %r4, HID0_E500_DEFAULT_SET@h ori %r4, %r4, HID0_E500_DEFAULT_SET@l cmpli 0, 0, %r3, FSL_E500mc bne 2f lis %r4, HID0_E500MC_DEFAULT_SET@h ori %r4, %r4, HID0_E500MC_DEFAULT_SET@l b 3f 2: cmpli 0, 0, %r3, FSL_E5500 bne 3f lis %r4, HID0_E5500_DEFAULT_SET@h ori %r4, %r4, HID0_E5500_DEFAULT_SET@l 3: mtspr SPR_HID0, %r4 isync /* Enable branch prediction */ li %r3, BUCSR_BPEN mtspr SPR_BUCSR, %r3 isync /* Invalidate all entries in TLB0 */ li %r3, 0 bl tlb_inval_all /* * Find TLB1 entry which is translating us now */ bl 2f 2: mflr %r3 bl tlb1_find_current /* the entry number found is in r29 */ bl tlb1_inval_all_but_current /* * Create temporary translation in AS=1 and switch to it */ bl tlb1_temp_mapping_as1 mfmsr %r3 ori %r3, %r3, (PSL_IS | PSL_DS) #ifdef __powerpc64__ oris %r3, %r3, PSL_CM@h #endif bl 3f 3: mflr %r4 addi %r4, %r4, (4f - 3b) mtspr SPR_SRR0, %r4 mtspr SPR_SRR1, %r3 rfi /* Switch context */ /* * Invalidate initial entry */ 4: mr %r3, %r29 bl tlb1_inval_entry /* * Setup final mapping in TLB1[1] and switch to it */ /* Final kernel mapping, map in 64 MB of RAM */ lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */ li %r4, 0 /* Entry 0 */ rlwimi %r3, %r4, 16, 4, 15 mtspr SPR_MAS0, %r3 isync li %r3, (TLB_SIZE_64M << MAS1_TSIZE_SHIFT)@l oris %r3, %r3, (MAS1_VALID | MAS1_IPROT)@h mtspr SPR_MAS1, %r3 /* note TS was not filled, so it's TS=0 */ isync LOAD_ADDR(%r3, KERNBASE) ori %r3, %r3, (_TLB_ENTRY_SHARED | MAS2_M)@l /* WIMGE = 0b00100 */ mtspr SPR_MAS2, %r3 isync /* Retrieve kernel load [physical] address from bp_kernload */ #ifdef __powerpc64__ b 0f .align 3 0: nop #endif bl 5f ADDR(bp_kernload) ADDR(__boot_page) 5: mflr %r3 #ifdef __powerpc64__ ld %r4, 0(%r3) ld %r5, 8(%r3) clrrdi %r3, %r3, 12 #else lwz %r4, 0(%r3) lwz %r5, 4(%r3) rlwinm %r3, %r3, 0, 0, 19 #endif sub %r4, %r4, %r5 /* offset of bp_kernload within __boot_page */ lwzx %r3, %r4, %r3 /* Set RPN and protection */ ori %r3, %r3, (MAS3_SX | MAS3_SW | MAS3_SR)@l mtspr SPR_MAS3, %r3 isync bl zero_mas7 bl zero_mas8 tlbwe isync msync /* Switch to the final mapping */ bl 6f 6: mflr %r3 rlwinm %r3, %r3, 0, 0xfff /* Offset from boot page start */ add %r3, %r3, %r5 /* Make this virtual address */ addi %r3, %r3, (7f - 6b) #ifdef __powerpc64__ lis %r4, PSL_CM@h /* Note AS=0 */ #else li %r4, 0 /* Note AS=0 */ #endif mtspr SPR_SRR0, %r3 mtspr SPR_SRR1, %r4 rfi 7: /* * At this point we're running at virtual addresses KERNBASE and beyond so * it's allowed to directly access all locations the kernel was linked * against. */ /* * Invalidate temp mapping */ mr %r3, %r28 bl tlb1_inval_entry #ifdef __powerpc64__ /* Set up the TOC pointer */ b 0f .align 3 0: nop bl 1f .llong __tocbase + 0x8000 - . 1: mflr %r2 ld %r1,0(%r2) add %r2,%r1,%r2 mtspr SPR_SPRG8, %r2 /* Get load offset */ ld %r31,-0x8000(%r2) /* First TOC entry is TOC base */ subf %r31,%r31,%r2 /* Subtract from real TOC base to get base */ /* Set up the stack pointer */ ld %r1,TOC_REF(tmpstack)(%r2) addi %r1,%r1,TMPSTACKSZ-96 add %r1,%r1,%r31 #else /* * Setup a temporary stack */ bl 1f .long tmpstack-. 1: mflr %r1 lwz %r2,0(%r1) add %r1,%r1,%r2 stw %r1, 0(%r1) addi %r1, %r1, (TMPSTACKSZ - 16) #endif /* * Initialise exception vector offsets */ bl CNAME(ivor_setup) TOC_RESTORE /* * Assign our pcpu instance */ bl 1f .long ap_pcpu-. 1: mflr %r4 lwz %r3, 0(%r4) add %r3, %r3, %r4 LOAD %r3, 0(%r3) mtsprg0 %r3 bl CNAME(pmap_bootstrap_ap) TOC_RESTORE bl CNAME(cpudep_ap_bootstrap) TOC_RESTORE /* Switch to the idle thread's kstack */ mr %r1, %r3 bl CNAME(machdep_ap_bootstrap) TOC_RESTORE /* NOT REACHED */ 6: b 6b #endif /* SMP */ #if defined (BOOKE_E500) /* * Invalidate all entries in the given TLB. * * r3 TLBSEL */ tlb_inval_all: rlwinm %r3, %r3, 3, (1 << 3) /* TLBSEL */ ori %r3, %r3, (1 << 2) /* INVALL */ tlbivax 0, %r3 isync msync tlbsync msync blr /* * expects address to look up in r3, returns entry number in r29 * * FIXME: the hidden assumption is we are now running in AS=0, but we should * retrieve actual AS from MSR[IS|DS] and put it in MAS6[SAS] */ tlb1_find_current: mfspr %r17, SPR_PID0 slwi %r17, %r17, MAS6_SPID0_SHIFT mtspr SPR_MAS6, %r17 isync tlbsx 0, %r3 mfspr %r17, SPR_MAS0 rlwinm %r29, %r17, 16, 26, 31 /* MAS0[ESEL] -> r29 */ /* Make sure we have IPROT set on the entry */ mfspr %r17, SPR_MAS1 oris %r17, %r17, MAS1_IPROT@h mtspr SPR_MAS1, %r17 isync tlbwe isync msync blr /* * Invalidates a single entry in TLB1. * * r3 ESEL * r4-r5 scratched */ tlb1_inval_entry: lis %r4, MAS0_TLBSEL1@h /* Select TLB1 */ rlwimi %r4, %r3, 16, 10, 15 /* Select our entry */ mtspr SPR_MAS0, %r4 isync tlbre li %r5, 0 /* MAS1[V] = 0 */ mtspr SPR_MAS1, %r5 isync tlbwe isync msync blr /* * r29 current entry number * r28 returned temp entry * r3-r5 scratched */ tlb1_temp_mapping_as1: /* Read our current translation */ lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */ rlwimi %r3, %r29, 16, 10, 15 /* Select our current entry */ mtspr SPR_MAS0, %r3 isync tlbre /* * Prepare and write temp entry * * FIXME this is not robust against overflow i.e. when the current * entry is the last in TLB1 */ lis %r3, MAS0_TLBSEL1@h /* Select TLB1 */ addi %r28, %r29, 1 /* Use next entry. */ rlwimi %r3, %r28, 16, 10, 15 /* Select temp entry */ mtspr SPR_MAS0, %r3 isync mfspr %r5, SPR_MAS1 li %r4, 1 /* AS=1 */ rlwimi %r5, %r4, 12, 19, 19 li %r4, 0 /* Global mapping, TID=0 */ rlwimi %r5, %r4, 16, 8, 15 oris %r5, %r5, (MAS1_VALID | MAS1_IPROT)@h mtspr SPR_MAS1, %r5 isync mflr %r3 bl zero_mas7 bl zero_mas8 mtlr %r3 tlbwe isync msync blr /* * Loops over TLB1, invalidates all entries skipping the one which currently * maps this code. * * r29 current entry * r3-r5 scratched */ tlb1_inval_all_but_current: mfspr %r3, SPR_TLB1CFG /* Get number of entries */ andi. %r3, %r3, TLBCFG_NENTRY_MASK@l li %r4, 0 /* Start from Entry 0 */ 1: lis %r5, MAS0_TLBSEL1@h rlwimi %r5, %r4, 16, 10, 15 mtspr SPR_MAS0, %r5 isync tlbre mfspr %r5, SPR_MAS1 cmpw %r4, %r29 /* our current entry? */ beq 2f rlwinm %r5, %r5, 0, 2, 31 /* clear VALID and IPROT bits */ mtspr SPR_MAS1, %r5 isync tlbwe isync msync 2: addi %r4, %r4, 1 cmpw %r4, %r3 /* Check if this is the last entry */ bne 1b blr /* * MAS7 and MAS8 conditional zeroing. */ .globl zero_mas7 zero_mas7: mfpvr %r20 rlwinm %r20, %r20, 16, 16, 31 cmpli 0, 0, %r20, FSL_E500v1 beq 1f li %r20, 0 mtspr SPR_MAS7, %r20 isync 1: blr .globl zero_mas8 zero_mas8: mfpvr %r20 rlwinm %r20, %r20, 16, 16, 31 cmpli 0, 0, %r20, FSL_E500mc beq 1f cmpli 0, 0, %r20, FSL_E5500 beq 1f blr 1: li %r20, 0 mtspr SPR_MAS8, %r20 isync blr #endif #ifdef SMP .globl __boot_tlb1 /* * The __boot_tlb1 table is used to hold BSP TLB1 entries * marked with _TLB_ENTRY_SHARED flag during AP bootstrap. * The BSP fills in the table in tlb_ap_prep() function. Next, * AP loads its contents to TLB1 hardware in pmap_bootstrap_ap(). */ __boot_tlb1: .space TLB1_MAX_ENTRIES * TLB_ENTRY_SIZE __boot_page_padding: /* * Boot page needs to be exactly 4K, with the last word of this page * acting as the reset vector, so we need to stuff the remainder. * Upon release from holdoff CPU fetches the last word of the boot * page. */ .space 4092 - (__boot_page_padding - __boot_page) b __boot_page #endif /* SMP */ /************************************************************************/ /* locore subroutines */ /************************************************************************/ /* * Cache disable/enable/inval sequences according * to section 2.16 of E500CORE RM. */ ENTRY(dcache_inval) /* Invalidate d-cache */ mfspr %r3, SPR_L1CSR0 ori %r3, %r3, (L1CSR0_DCFI | L1CSR0_DCLFR)@l msync isync mtspr SPR_L1CSR0, %r3 isync 1: mfspr %r3, SPR_L1CSR0 andi. %r3, %r3, L1CSR0_DCFI bne 1b blr ENTRY(dcache_disable) /* Disable d-cache */ mfspr %r3, SPR_L1CSR0 li %r4, L1CSR0_DCE@l not %r4, %r4 and %r3, %r3, %r4 msync isync mtspr SPR_L1CSR0, %r3 isync blr ENTRY(dcache_enable) /* Enable d-cache */ mfspr %r3, SPR_L1CSR0 oris %r3, %r3, (L1CSR0_DCPE | L1CSR0_DCE)@h ori %r3, %r3, (L1CSR0_DCPE | L1CSR0_DCE)@l msync isync mtspr SPR_L1CSR0, %r3 isync blr ENTRY(icache_inval) /* Invalidate i-cache */ mfspr %r3, SPR_L1CSR1 ori %r3, %r3, (L1CSR1_ICFI | L1CSR1_ICLFR)@l isync mtspr SPR_L1CSR1, %r3 isync 1: mfspr %r3, SPR_L1CSR1 andi. %r3, %r3, L1CSR1_ICFI bne 1b blr ENTRY(icache_disable) /* Disable i-cache */ mfspr %r3, SPR_L1CSR1 li %r4, L1CSR1_ICE@l not %r4, %r4 and %r3, %r3, %r4 isync mtspr SPR_L1CSR1, %r3 isync blr ENTRY(icache_enable) /* Enable i-cache */ mfspr %r3, SPR_L1CSR1 oris %r3, %r3, (L1CSR1_ICPE | L1CSR1_ICE)@h ori %r3, %r3, (L1CSR1_ICPE | L1CSR1_ICE)@l isync mtspr SPR_L1CSR1, %r3 isync blr /* * L2 cache disable/enable/inval sequences for E500mc. */ ENTRY(l2cache_inval) mfspr %r3, SPR_L2CSR0 oris %r3, %r3, (L2CSR0_L2FI | L2CSR0_L2LFC)@h ori %r3, %r3, (L2CSR0_L2FI | L2CSR0_L2LFC)@l isync mtspr SPR_L2CSR0, %r3 isync 1: mfspr %r3, SPR_L2CSR0 andis. %r3, %r3, L2CSR0_L2FI@h bne 1b blr ENTRY(l2cache_enable) mfspr %r3, SPR_L2CSR0 oris %r3, %r3, (L2CSR0_L2E | L2CSR0_L2PE)@h isync mtspr SPR_L2CSR0, %r3 isync blr /* * Branch predictor setup. */ ENTRY(bpred_enable) mfspr %r3, SPR_BUCSR ori %r3, %r3, BUCSR_BBFI isync mtspr SPR_BUCSR, %r3 isync ori %r3, %r3, BUCSR_BPEN isync mtspr SPR_BUCSR, %r3 isync blr ENTRY(dataloss_erratum_access) /* Lock two cache lines into I-Cache */ sync mfspr %r11, SPR_L1CSR1 rlwinm %r11, %r11, 0, ~L1CSR1_ICUL sync isync mtspr SPR_L1CSR1, %r11 isync lis %r8, 2f@h ori %r8, %r8, 2f@l icbtls 0, 0, %r8 addi %r9, %r8, 64 sync mfspr %r11, SPR_L1CSR1 3: andi. %r11, %r11, L1CSR1_ICUL bne 3b icbtls 0, 0, %r9 sync mfspr %r11, SPR_L1CSR1 3: andi. %r11, %r11, L1CSR1_ICUL bne 3b b 2f .align 6 /* Inside a locked cacheline, wait a while, write, then wait a while */ 2: sync mfspr %r5, TBR_TBL 4: addis %r11, %r5, 0x100000@h /* wait around one million timebase ticks */ mfspr %r5, TBR_TBL subf. %r5, %r5, %r11 bgt 4b stw %r4, 0(%r3) mfspr %r5, TBR_TBL 4: addis %r11, %r5, 0x100000@h /* wait around one million timebase ticks */ mfspr %r5, TBR_TBL subf. %r5, %r5, %r11 bgt 4b sync /* * Fill out the rest of this cache line and the next with nops, * to ensure that nothing outside the locked area will be * fetched due to a branch. */ .rept 19 nop .endr icblc 0, 0, %r8 icblc 0, 0, %r9 blr /* * XXX: This should be moved to a shared AIM/booke asm file, if one ever is * created. */ ENTRY(get_spr) mfspr %r3, 0 blr /************************************************************************/ /* Data section */ /************************************************************************/ .data .align 3 GLOBAL(__startkernel) ADDR(begin) GLOBAL(__endkernel) ADDR(end) .align 4 tmpstack: .space TMPSTACKSZ tmpstackbound: .space 10240 /* XXX: this really should not be necessary */ #ifdef __powerpc64__ TOC_ENTRY(tmpstack) TOC_ENTRY(bp_kernload) #endif /* * Compiled KERNBASE locations */ .globl kernbase .set kernbase, KERNBASE #include Index: head/sys/powerpc/powerpc/cpu_subr64.S =================================================================== --- head/sys/powerpc/powerpc/cpu_subr64.S (revision 331253) +++ head/sys/powerpc/powerpc/cpu_subr64.S (revision 331254) @@ -1,97 +1,97 @@ /*- * Copyright (c) 2017-2018 QCM Technologies. * Copyright (c) 2017-2018 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#include "assym.s" +#include "assym.inc" #include .globl CNAME(power_save_sequence) .p2align 3 ENTRY(enter_idle_powerx) mfsprg0 %r3 /* Get the pcpu pointer */ ld %r3,PC_CURTHREAD(%r3) /* Get current thread */ ld %r3,TD_PCB(%r3) /* Get PCB of current thread */ std %r12,PCB_CONTEXT(%r3) /* Save the non-volatile GP regs. */ std %r13,PCB_CONTEXT+1*8(%r3) std %r14,PCB_CONTEXT+2*8(%r3) std %r15,PCB_CONTEXT+3*8(%r3) std %r16,PCB_CONTEXT+4*8(%r3) std %r17,PCB_CONTEXT+5*8(%r3) std %r18,PCB_CONTEXT+6*8(%r3) std %r19,PCB_CONTEXT+7*8(%r3) std %r20,PCB_CONTEXT+8*8(%r3) std %r21,PCB_CONTEXT+9*8(%r3) std %r22,PCB_CONTEXT+10*8(%r3) std %r23,PCB_CONTEXT+11*8(%r3) std %r24,PCB_CONTEXT+12*8(%r3) std %r25,PCB_CONTEXT+13*8(%r3) std %r26,PCB_CONTEXT+14*8(%r3) std %r27,PCB_CONTEXT+15*8(%r3) std %r28,PCB_CONTEXT+16*8(%r3) std %r29,PCB_CONTEXT+17*8(%r3) std %r30,PCB_CONTEXT+18*8(%r3) std %r31,PCB_CONTEXT+19*8(%r3) mfcr %r16 /* Save the condition register */ std %r16,PCB_CR(%r3) mflr %r16 /* Save the link register */ std %r16,PCB_LR(%r3) std %r1,PCB_SP(%r3) /* Save the stack pointer */ std %r2,PCB_TOC(%r3) /* Save the TOC pointer */ /* Set where we want to jump */ bl 1f .llong power_save_sequence /* Remember about 8 byte alignment */ 1: mflr %r3 ld %r3,0(%r3) mtsrr0 %r3 /* Set MSR */ li %r3,0 ori %r3,%r3,(PSL_ME | PSL_RI) li %r8,0x9 /* PSL_SF and PSL_HV */ insrdi %r3,%r8,4,0 mtsrr1 %r3 rfid .p2align 2 CNAME(power_save_sequence): bl 1f .llong 0x0 /* Playground for power-save sequence */ 1: mflr %r3 /* Start power-save sequence */ std %r2,0(%r3) ptesync ld %r2,0(%r3) 2: cmpd %r2,%r2 bne 2b nap b . Index: head/sys/powerpc/powerpc/sigcode32.S =================================================================== --- head/sys/powerpc/powerpc/sigcode32.S (revision 331253) +++ head/sys/powerpc/powerpc/sigcode32.S (revision 331254) @@ -1,59 +1,59 @@ /* $FreeBSD$ */ /* $NetBSD: sigcode.S,v 1.1 1999/11/17 14:56:11 kleink Exp $ */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include -#include "assym.s" +#include "assym.inc" /* * The following code gets copied to the top of the user stack on process * execution. It does signal trampolining on signal delivery. * * On entry r1 points to a struct sigframe at bottom of current stack. * All other registers are unchanged. */ .globl CNAME(sigcode32),CNAME(szsigcode32) CNAME(sigcode32): addi 1,1,-32 /* reserved space for callee */ blrl addi 3,1,32+SF_UC /* restore sp, and get &frame->sf_uc */ li 0,SYS_sigreturn sc /* sigreturn(scp) */ li 0,SYS_exit sc /* exit(errno) */ endsigcode32: .data CNAME(szsigcode32): .long endsigcode32 - CNAME(sigcode32) Index: head/sys/powerpc/powerpc/sigcode64.S =================================================================== --- head/sys/powerpc/powerpc/sigcode64.S (revision 331253) +++ head/sys/powerpc/powerpc/sigcode64.S (revision 331254) @@ -1,76 +1,76 @@ /* $FreeBSD$ */ /* $NetBSD: sigcode.S,v 1.1 1999/11/17 14:56:11 kleink Exp $ */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include -#include "assym.s" +#include "assym.inc" /* * The following code gets copied to the top of the user stack on process * execution. It does signal trampolining on signal delivery. * * On entry r1 points to a struct sigframe at bottom of current stack. * All other registers are unchanged. * * Entered midway through for v2 ELF binaries that don't need to deal with * function descriptors. * */ .globl CNAME(sigcode64),CNAME(szsigcode64) .globl CNAME(sigcode64_elfv2),CNAME(szsigcode64_elfv2) CNAME(sigcode64): mflr 2 /* resolve function descriptor */ ld 0,0(2) ld 2,8(2) mtlr 0 CNAME(sigcode64_elfv2): addi 1,1,-112 /* reserved space for callee */ mflr 12 /* ELFv2 wants the address in r12 */ blrl addi 3,1,112+SF_UC /* restore sp, and get &frame->sf_uc */ li 0,SYS_sigreturn sc /* sigreturn(scp) */ li 0,SYS_exit sc /* exit(errno) */ nop /* align to doubleword */ endsigcode64: .data CNAME(szsigcode64): .long endsigcode64 - CNAME(sigcode64) CNAME(szsigcode64_elfv2): .long endsigcode64 - CNAME(sigcode64_elfv2) Index: head/sys/powerpc/powerpc/swtch32.S =================================================================== --- head/sys/powerpc/powerpc/swtch32.S (revision 331253) +++ head/sys/powerpc/powerpc/swtch32.S (revision 331254) @@ -1,207 +1,207 @@ /* $FreeBSD$ */ /* $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $ */ /*- * Copyright (C) 2001 Benno Rice * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "assym.s" +#include "assym.inc" #include "opt_sched.h" #include #include #include #include #include /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) mr %r2, %r4 li %r14,0 /* Tell cpu_switchin not to release a thread */ b cpu_switchin /* * void cpu_switch(struct thread *old, * struct thread *new, * struct mutex *mtx); * * Switch to a new thread saving the current state in the old thread. */ ENTRY(cpu_switch) lwz %r6,TD_PCB(%r3) /* Get the old thread's PCB ptr */ stmw %r12,PCB_CONTEXT(%r6) /* Save the non-volatile GP regs. These can now be used for scratch */ mfcr %r16 /* Save the condition register */ stw %r16,PCB_CR(%r6) mflr %r16 /* Save the link register */ stw %r16,PCB_LR(%r6) stw %r1,PCB_SP(%r6) /* Save the stack pointer */ mr %r14,%r3 /* Copy the old thread ptr... */ mr %r2,%r4 /* and the new thread ptr in curthread */ mr %r16,%r5 /* and the new lock */ mr %r17,%r6 /* and the PCB */ lwz %r7,PCB_FLAGS(%r17) /* Save FPU context if needed */ andi. %r7, %r7, PCB_FPU beq .L1 bl save_fpu .L1: mr %r3,%r14 /* restore old thread ptr */ lwz %r7,PCB_FLAGS(%r17) /* Save Altivec context if needed */ andi. %r7, %r7, PCB_VEC beq .L2 bl save_vec .L2: mr %r3,%r14 /* restore old thread ptr */ bl pmap_deactivate /* Deactivate the current pmap */ sync /* Make sure all of that finished */ cpu_switchin: #if defined(SMP) && defined(SCHED_ULE) /* Wait for the new thread to become unblocked */ bl _GLOBAL_OFFSET_TABLE_@local-4 mflr %r6 lwz %r6,blocked_lock@got(%r6) blocked_loop: lwz %r7,TD_LOCK(%r2) cmpw %r6,%r7 beq- blocked_loop isync #endif lwz %r17,TD_PCB(%r2) /* Get new current PCB */ lwz %r1,PCB_SP(%r17) /* Load new stack pointer */ /* Release old thread now that we have a stack pointer set up */ cmpwi %r14,0 beq- 1f stw %r16,TD_LOCK(%r14) /* ULE: update old thread's lock */ 1: mfsprg %r7,0 /* Get the pcpu pointer */ stw %r2,PC_CURTHREAD(%r7) /* Store new current thread */ lwz %r17,TD_PCB(%r2) /* Store new current PCB */ stw %r17,PC_CURPCB(%r7) mr %r3,%r2 /* Get new thread ptr */ bl pmap_activate /* Activate the new address space */ lwz %r6, PCB_FLAGS(%r17) /* Restore FPU context if needed */ andi. %r6, %r6, PCB_FPU beq .L3 mr %r3,%r2 /* Pass curthread to enable_fpu */ bl enable_fpu .L3: lwz %r6, PCB_FLAGS(%r17) /* Restore Altivec context if needed */ andi. %r6, %r6, PCB_VEC beq .L4 mr %r3,%r2 /* Pass curthread to enable_vec */ bl enable_vec .L4: /* thread to restore is in r3 */ mr %r3,%r17 /* Recover PCB ptr */ lmw %r12,PCB_CONTEXT(%r3) /* Load the non-volatile GP regs */ lwz %r5,PCB_CR(%r3) /* Load the condition register */ mtcr %r5 lwz %r5,PCB_LR(%r3) /* Load the link register */ mtlr %r5 lwz %r1,PCB_SP(%r3) /* Load the stack pointer */ /* * Perform a dummy stwcx. to clear any reservations we may have * inherited from the previous thread. It doesn't matter if the * stwcx succeeds or not. pcb_context[0] can be clobbered. */ stwcx. %r1, 0, %r3 blr /* * savectx(pcb) * Update pcb, saving current processor state */ ENTRY(savectx) stmw %r12,PCB_CONTEXT(%r3) /* Save the non-volatile GP regs */ mfcr %r4 /* Save the condition register */ stw %r4,PCB_CR(%r3) mflr %r4 /* Save the link register */ stw %r4,PCB_LR(%r3) blr /* * fork_trampoline() * Set up the return from cpu_fork() */ ENTRY(fork_trampoline) lwz %r3,CF_FUNC(%r1) lwz %r4,CF_ARG0(%r1) lwz %r5,CF_ARG1(%r1) bl fork_exit addi %r1,%r1,CF_SIZE-FSP /* Allow 8 bytes in front of trapframe to simulate FRAME_SETUP does when allocating space for a frame pointer/saved LR */ b trapexit Index: head/sys/powerpc/powerpc/swtch64.S =================================================================== --- head/sys/powerpc/powerpc/swtch64.S (revision 331253) +++ head/sys/powerpc/powerpc/swtch64.S (revision 331254) @@ -1,285 +1,285 @@ /* $FreeBSD$ */ /* $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $ */ /*- * Copyright (C) 2001 Benno Rice * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (C) 1995, 1996 Wolfgang Solfrank. * Copyright (C) 1995, 1996 TooLs GmbH. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include "assym.s" +#include "assym.inc" #include "opt_sched.h" #include #include #include #include #ifdef _CALL_ELF .abiversion _CALL_ELF #endif TOC_ENTRY(blocked_lock) /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) mr %r13, %r4 li %r14,0 /* Tell cpu_switchin not to release a thread */ b cpu_switchin /* * void cpu_switch(struct thread *old, * struct thread *new, * struct mutex *mtx); * * Switch to a new thread saving the current state in the old thread. */ ENTRY(cpu_switch) ld %r6,TD_PCB(%r3) /* Get the old thread's PCB ptr */ std %r12,PCB_CONTEXT(%r6) /* Save the non-volatile GP regs. These can now be used for scratch */ std %r14,PCB_CONTEXT+2*8(%r6) std %r15,PCB_CONTEXT+3*8(%r6) std %r16,PCB_CONTEXT+4*8(%r6) std %r17,PCB_CONTEXT+5*8(%r6) std %r18,PCB_CONTEXT+6*8(%r6) std %r19,PCB_CONTEXT+7*8(%r6) std %r20,PCB_CONTEXT+8*8(%r6) std %r21,PCB_CONTEXT+9*8(%r6) std %r22,PCB_CONTEXT+10*8(%r6) std %r23,PCB_CONTEXT+11*8(%r6) std %r24,PCB_CONTEXT+12*8(%r6) std %r25,PCB_CONTEXT+13*8(%r6) std %r26,PCB_CONTEXT+14*8(%r6) std %r27,PCB_CONTEXT+15*8(%r6) std %r28,PCB_CONTEXT+16*8(%r6) std %r29,PCB_CONTEXT+17*8(%r6) std %r30,PCB_CONTEXT+18*8(%r6) std %r31,PCB_CONTEXT+19*8(%r6) mfcr %r16 /* Save the condition register */ std %r16,PCB_CR(%r6) mflr %r16 /* Save the link register */ std %r16,PCB_LR(%r6) std %r1,PCB_SP(%r6) /* Save the stack pointer */ std %r2,PCB_TOC(%r6) /* Save the TOC pointer */ mr %r14,%r3 /* Copy the old thread ptr... */ mr %r13,%r4 /* and the new thread ptr in curthread*/ mr %r16,%r5 /* and the new lock */ mr %r17,%r6 /* and the PCB */ stdu %r1,-48(%r1) lwz %r7,PCB_FLAGS(%r17) /* Save FPU context if needed */ andi. %r7, %r7, PCB_FPU beq .L1 bl save_fpu nop .L1: mr %r3,%r14 /* restore old thread ptr */ lwz %r7,PCB_FLAGS(%r17) /* Save Altivec context if needed */ andi. %r7, %r7, PCB_VEC beq .L2 bl save_vec nop .L2: mr %r3,%r14 /* restore old thread ptr */ bl pmap_deactivate /* Deactivate the current pmap */ nop sync /* Make sure all of that finished */ cpu_switchin: #if defined(SMP) && defined(SCHED_ULE) /* Wait for the new thread to become unblocked */ ld %r6,TOC_REF(blocked_lock)(%r2) blocked_loop: ld %r7,TD_LOCK(%r13) cmpd %r6,%r7 beq- blocked_loop isync #endif ld %r17,TD_PCB(%r13) /* Get new PCB */ ld %r1,PCB_SP(%r17) /* Load the stack pointer */ addi %r1,%r1,-48 /* Remember about cpu_switch stack frame */ /* Release old thread now that we have a stack pointer set up */ cmpdi %r14,0 beq- 1f std %r16,TD_LOCK(%r14) /* ULE: update old thread's lock */ 1: mfsprg %r7,0 /* Get the pcpu pointer */ std %r13,PC_CURTHREAD(%r7) /* Store new current thread */ ld %r17,TD_PCB(%r13) /* Store new current PCB */ std %r17,PC_CURPCB(%r7) mr %r3,%r13 /* Get new thread ptr */ bl pmap_activate /* Activate the new address space */ nop lwz %r6, PCB_FLAGS(%r17) /* Restore FPU context if needed */ andi. %r6, %r6, PCB_FPU beq .L3 mr %r3,%r13 /* Pass curthread to enable_fpu */ bl enable_fpu nop .L3: lwz %r6, PCB_FLAGS(%r17) /* Restore Altivec context if needed */ andi. %r6, %r6, PCB_VEC beq .L4 mr %r3,%r13 /* Pass curthread to enable_vec */ bl enable_vec nop /* thread to restore is in r3 */ .L4: addi %r1,%r1,48 mr %r3,%r17 /* Recover PCB ptr */ ld %r12,PCB_CONTEXT(%r3) /* Load the non-volatile GP regs. */ ld %r14,PCB_CONTEXT+2*8(%r3) ld %r15,PCB_CONTEXT+3*8(%r3) ld %r16,PCB_CONTEXT+4*8(%r3) ld %r17,PCB_CONTEXT+5*8(%r3) ld %r18,PCB_CONTEXT+6*8(%r3) ld %r19,PCB_CONTEXT+7*8(%r3) ld %r20,PCB_CONTEXT+8*8(%r3) ld %r21,PCB_CONTEXT+9*8(%r3) ld %r22,PCB_CONTEXT+10*8(%r3) ld %r23,PCB_CONTEXT+11*8(%r3) ld %r24,PCB_CONTEXT+12*8(%r3) ld %r25,PCB_CONTEXT+13*8(%r3) ld %r26,PCB_CONTEXT+14*8(%r3) ld %r27,PCB_CONTEXT+15*8(%r3) ld %r28,PCB_CONTEXT+16*8(%r3) ld %r29,PCB_CONTEXT+17*8(%r3) ld %r30,PCB_CONTEXT+18*8(%r3) ld %r31,PCB_CONTEXT+19*8(%r3) ld %r5,PCB_CR(%r3) /* Load the condition register */ mtcr %r5 ld %r5,PCB_LR(%r3) /* Load the link register */ mtlr %r5 ld %r1,PCB_SP(%r3) /* Load the stack pointer */ ld %r2,PCB_TOC(%r3) /* Load the TOC pointer */ /* * Perform a dummy stdcx. to clear any reservations we may have * inherited from the previous thread. It doesn't matter if the * stdcx succeeds or not. pcb_context[0] can be clobbered. */ stdcx. %r1, 0, %r3 blr /* * savectx(pcb) * Update pcb, saving current processor state */ ENTRY(savectx) std %r12,PCB_CONTEXT(%r3) /* Save the non-volatile GP regs. */ std %r13,PCB_CONTEXT+1*8(%r3) std %r14,PCB_CONTEXT+2*8(%r3) std %r15,PCB_CONTEXT+3*8(%r3) std %r16,PCB_CONTEXT+4*8(%r3) std %r17,PCB_CONTEXT+5*8(%r3) std %r18,PCB_CONTEXT+6*8(%r3) std %r19,PCB_CONTEXT+7*8(%r3) std %r20,PCB_CONTEXT+8*8(%r3) std %r21,PCB_CONTEXT+9*8(%r3) std %r22,PCB_CONTEXT+10*8(%r3) std %r23,PCB_CONTEXT+11*8(%r3) std %r24,PCB_CONTEXT+12*8(%r3) std %r25,PCB_CONTEXT+13*8(%r3) std %r26,PCB_CONTEXT+14*8(%r3) std %r27,PCB_CONTEXT+15*8(%r3) std %r28,PCB_CONTEXT+16*8(%r3) std %r29,PCB_CONTEXT+17*8(%r3) std %r30,PCB_CONTEXT+18*8(%r3) std %r31,PCB_CONTEXT+19*8(%r3) mfcr %r4 /* Save the condition register */ std %r4,PCB_CR(%r3) std %r2,PCB_TOC(%r3) /* Save the TOC pointer */ mflr %r4 /* Save the link register */ std %r4,PCB_LR(%r3) blr /* * fork_trampoline() * Set up the return from cpu_fork() */ ENTRY_NOPROF(fork_trampoline) ld %r3,CF_FUNC(%r1) ld %r4,CF_ARG0(%r1) ld %r5,CF_ARG1(%r1) stdu %r1,-48(%r1) bl fork_exit nop addi %r1,%r1,48+CF_SIZE-FSP /* Allow 8 bytes in front of trapframe to simulate FRAME_SETUP does when allocating space for a frame pointer/saved LR */ bl trapexit nop Index: head/sys/riscv/riscv/copyinout.S =================================================================== --- head/sys/riscv/riscv/copyinout.S (revision 331253) +++ head/sys/riscv/riscv/copyinout.S (revision 331254) @@ -1,137 +1,137 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include -#include "assym.s" +#include "assym.inc" /* * Fault handler for the copy{in,out} functions below. */ ENTRY(copyio_fault) SET_FAULT_HANDLER(x0, a1) /* Clear the handler */ copyio_fault_nopcb: li a0, EFAULT ret END(copyio_fault) /* * Copies from a kernel to user address * * int copyout(const void *kaddr, void *udaddr, size_t len) */ ENTRY(copyout) beqz a2, 2f /* If len == 0 then skip loop */ add a3, a1, a2 li a4, VM_MAXUSER_ADDRESS bgt a3, a4, copyio_fault_nopcb la a6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(a6, a7) /* Set the handler */ 1: lb a4, 0(a0) /* Load from kaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in uaddr */ addi a1, a1, 1 addi a2, a2, -1 /* len-- */ bnez a2, 1b SET_FAULT_HANDLER(x0, a7) /* Clear the handler */ 2: li a0, 0 /* return 0 */ ret END(copyout) /* * Copies from a user to kernel address * * int copyin(const void *uaddr, void *kdaddr, size_t len) */ ENTRY(copyin) beqz a2, 2f /* If len == 0 then skip loop */ add a3, a0, a2 li a4, VM_MAXUSER_ADDRESS bgt a3, a4, copyio_fault_nopcb la a6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(a6, a7) /* Set the handler */ 1: lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in kaddr */ addi a1, a1, 1 addi a2, a2, -1 /* len-- */ bnez a2, 1b SET_FAULT_HANDLER(x0, a7) /* Clear the handler */ 2: li a0, 0 /* return 0 */ ret END(copyin) /* * Copies a string from a user to kernel address * * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) */ ENTRY(copyinstr) mv a5, x0 /* count = 0 */ beqz a2, 3f /* If len == 0 then skip loop */ li a7, VM_MAXUSER_ADDRESS la a6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(a6, a7) /* Set the handler */ 1: bgt a7, a0, copyio_fault lb a4, 0(a0) /* Load from uaddr */ addi a0, a0, 1 sb a4, 0(a1) /* Store in kaddr */ addi a1, a1, 1 beqz a4, 2f addi a2, a2, -1 /* len-- */ addi a5, a5, 1 /* count++ */ bnez a2, 1b 2: SET_FAULT_HANDLER(x0, a7) /* Clear the handler */ 3: beqz a3, 4f /* Check if done != NULL */ addi a5, a5, 1 /* count++ */ sd a5, 0(a3) /* done = count */ 4: mv a0, x0 /* return 0 */ ret END(copyinstr) Index: head/sys/riscv/riscv/exception.S =================================================================== --- head/sys/riscv/riscv/exception.S (revision 331253) +++ head/sys/riscv/riscv/exception.S (revision 331254) @@ -1,240 +1,240 @@ /*- * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); -#include "assym.s" +#include "assym.inc" #include #include .macro save_registers el addi sp, sp, -(TF_SIZE) sd ra, (TF_RA)(sp) sd tp, (TF_TP)(sp) .if \el == 0 /* We came from userspace. Load our pcpu */ sd gp, (TF_GP)(sp) ld gp, (TF_SIZE)(sp) .endif sd t0, (TF_T + 0 * 8)(sp) sd t1, (TF_T + 1 * 8)(sp) sd t2, (TF_T + 2 * 8)(sp) sd t3, (TF_T + 3 * 8)(sp) sd t4, (TF_T + 4 * 8)(sp) sd t5, (TF_T + 5 * 8)(sp) sd t6, (TF_T + 6 * 8)(sp) sd s0, (TF_S + 0 * 8)(sp) sd s1, (TF_S + 1 * 8)(sp) sd s2, (TF_S + 2 * 8)(sp) sd s3, (TF_S + 3 * 8)(sp) sd s4, (TF_S + 4 * 8)(sp) sd s5, (TF_S + 5 * 8)(sp) sd s6, (TF_S + 6 * 8)(sp) sd s7, (TF_S + 7 * 8)(sp) sd s8, (TF_S + 8 * 8)(sp) sd s9, (TF_S + 9 * 8)(sp) sd s10, (TF_S + 10 * 8)(sp) sd s11, (TF_S + 11 * 8)(sp) sd a0, (TF_A + 0 * 8)(sp) sd a1, (TF_A + 1 * 8)(sp) sd a2, (TF_A + 2 * 8)(sp) sd a3, (TF_A + 3 * 8)(sp) sd a4, (TF_A + 4 * 8)(sp) sd a5, (TF_A + 5 * 8)(sp) sd a6, (TF_A + 6 * 8)(sp) sd a7, (TF_A + 7 * 8)(sp) #if 0 /* XXX: temporary test: spin if stack is not kernel one */ .if \el == 1 /* kernel */ mv t0, sp srli t0, t0, 63 1: beqz t0, 1b .endif #endif .if \el == 1 /* Store kernel sp */ li t1, TF_SIZE add t0, sp, t1 sd t0, (TF_SP)(sp) .else /* Store user sp */ csrr t0, sscratch sd t0, (TF_SP)(sp) .endif li t0, 0 csrw sscratch, t0 csrr t0, sepc sd t0, (TF_SEPC)(sp) csrr t0, sstatus sd t0, (TF_SSTATUS)(sp) csrr t0, sbadaddr sd t0, (TF_SBADADDR)(sp) csrr t0, scause sd t0, (TF_SCAUSE)(sp) .endm .macro load_registers el ld t0, (TF_SSTATUS)(sp) .if \el == 0 /* * Ensure user interrupts will be enabled on eret * and supervisor mode can access userspace on trap. */ li t1, (SSTATUS_SPIE | SSTATUS_SUM) or t0, t0, t1 .else /* * Disable interrupts for supervisor mode exceptions. * For user mode exceptions we have already done this * in do_ast. */ li t1, ~SSTATUS_SIE and t0, t0, t1 .endif csrw sstatus, t0 ld t0, (TF_SEPC)(sp) csrw sepc, t0 .if \el == 0 /* We go to userspace. Load user sp */ ld t0, (TF_SP)(sp) csrw sscratch, t0 /* And store our pcpu */ sd gp, (TF_SIZE)(sp) ld gp, (TF_GP)(sp) .endif ld ra, (TF_RA)(sp) ld tp, (TF_TP)(sp) ld t0, (TF_T + 0 * 8)(sp) ld t1, (TF_T + 1 * 8)(sp) ld t2, (TF_T + 2 * 8)(sp) ld t3, (TF_T + 3 * 8)(sp) ld t4, (TF_T + 4 * 8)(sp) ld t5, (TF_T + 5 * 8)(sp) ld t6, (TF_T + 6 * 8)(sp) ld s0, (TF_S + 0 * 8)(sp) ld s1, (TF_S + 1 * 8)(sp) ld s2, (TF_S + 2 * 8)(sp) ld s3, (TF_S + 3 * 8)(sp) ld s4, (TF_S + 4 * 8)(sp) ld s5, (TF_S + 5 * 8)(sp) ld s6, (TF_S + 6 * 8)(sp) ld s7, (TF_S + 7 * 8)(sp) ld s8, (TF_S + 8 * 8)(sp) ld s9, (TF_S + 9 * 8)(sp) ld s10, (TF_S + 10 * 8)(sp) ld s11, (TF_S + 11 * 8)(sp) ld a0, (TF_A + 0 * 8)(sp) ld a1, (TF_A + 1 * 8)(sp) ld a2, (TF_A + 2 * 8)(sp) ld a3, (TF_A + 3 * 8)(sp) ld a4, (TF_A + 4 * 8)(sp) ld a5, (TF_A + 5 * 8)(sp) ld a6, (TF_A + 6 * 8)(sp) ld a7, (TF_A + 7 * 8)(sp) addi sp, sp, (TF_SIZE) .endm .macro do_ast /* Disable interrupts */ csrr a4, sstatus 1: csrci sstatus, (SSTATUS_SIE) ld a1, PC_CURTHREAD(gp) lw a2, TD_FLAGS(a1) li a3, (TDF_ASTPENDING|TDF_NEEDRESCHED) and a2, a2, a3 beqz a2, 2f /* Restore interrupts */ andi a4, a4, (SSTATUS_SIE) csrs sstatus, a4 /* Handle the ast */ mv a0, sp call _C_LABEL(ast) /* Re-check for new ast scheduled */ j 1b 2: .endm ENTRY(cpu_exception_handler) csrrw sp, sscratch, sp beqz sp, 1f /* User mode detected */ csrrw sp, sscratch, sp j cpu_exception_handler_user 1: /* Supervisor mode detected */ csrrw sp, sscratch, sp j cpu_exception_handler_supervisor END(cpu_exception_handler) ENTRY(cpu_exception_handler_supervisor) save_registers 1 mv a0, sp call _C_LABEL(do_trap_supervisor) load_registers 1 sret END(cpu_exception_handler_supervisor) ENTRY(cpu_exception_handler_user) csrrw sp, sscratch, sp save_registers 0 mv a0, sp call _C_LABEL(do_trap_user) do_ast load_registers 0 csrrw sp, sscratch, sp sret END(cpu_exception_handler_user) Index: head/sys/riscv/riscv/locore.S =================================================================== --- head/sys/riscv/riscv/locore.S (revision 331253) +++ head/sys/riscv/riscv/locore.S (revision 331254) @@ -1,312 +1,312 @@ /*- * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#include "assym.s" +#include "assym.inc" #include #include #include #include #include #include .globl kernbase .set kernbase, KERNBASE /* Trap entries */ .text /* Reset vector */ .text .globl _start _start: /* Get the physical address kernel loaded to */ la t0, virt_map ld t1, 0(t0) sub t1, t1, t0 li t2, KERNBASE sub s9, t2, t1 /* s9 = physmem base */ mv s10, a0 /* s10 = hart id */ mv s11, a1 /* s11 = dtbp */ li t0, SSTATUS_SUM csrs sstatus, t0 /* Direct secondary cores to mpentry */ bnez s10, mpentry /* * Page tables */ /* Add L1 entry for kernel */ la s1, pagetable_l1 la s2, pagetable_l2 /* Link to next level PN */ srli s2, s2, PAGE_SHIFT li a5, KERNBASE srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ andi a5, a5, 0x1ff /* & 0x1ff */ li t4, PTE_V slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ or t6, t4, t5 /* Store L1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 add t0, s1, a5 sd t6, (t0) /* Level 2 superpages (512 x 2MiB) */ la s1, pagetable_l2 srli t4, s9, 21 /* Div physmem base by 2 MiB */ li t2, 512 /* Build 512 entries */ add t3, t4, t2 li t5, 0 2: li t0, (PTE_V | PTE_RWX | PTE_D) slli t2, t4, PTE_PPN1_S /* << PTE_PPN1_S */ or t5, t0, t2 sd t5, (s1) /* Store PTE entry to position */ addi s1, s1, PTE_SIZE addi t4, t4, 1 bltu t4, t3, 2b /* Create an L1 page for early devmap */ la s1, pagetable_l1 la s2, pagetable_l2_devmap /* Link to next level PN */ srli s2, s2, PAGE_SHIFT li a5, (VM_MAX_KERNEL_ADDRESS - L2_SIZE) srli a5, a5, L1_SHIFT /* >> L1_SHIFT */ andi a5, a5, 0x1ff /* & 0x1ff */ li t4, PTE_V slli t5, s2, PTE_PPN0_S /* (s2 << PTE_PPN0_S) */ or t6, t4, t5 /* Store single level1 PTE entry to position */ li a6, PTE_SIZE mulw a5, a5, a6 add t0, s1, a5 sd t6, (t0) /* Create an L2 page superpage for DTB */ la s1, pagetable_l2_devmap mv s2, s11 srli s2, s2, PAGE_SHIFT li t0, (PTE_V | PTE_RWX | PTE_D) slli t2, s2, PTE_PPN0_S /* << PTE_PPN0_S */ or t0, t0, t2 /* Store PTE entry to position */ li a6, PTE_SIZE li a5, 510 mulw a5, a5, a6 add t1, s1, a5 sd t0, (t1) /* Page tables END */ /* Setup supervisor trap vector */ la t0, va sub t0, t0, s9 li t1, KERNBASE add t0, t0, t1 csrw stvec, t0 /* Set page tables base register */ la s2, pagetable_l1 srli s2, s2, PAGE_SHIFT li t0, SATP_MODE_SV39 or s2, s2, t0 sfence.vma csrw sptbr, s2 va: /* Setup supervisor trap vector */ la t0, cpu_exception_handler csrw stvec, t0 /* Ensure sscratch is zero */ li t0, 0 csrw sscratch, t0 /* Initialize stack pointer */ la s3, initstack_end mv sp, s3 addi sp, sp, -PCB_SIZE /* Clear BSS */ la a0, _C_LABEL(__bss_start) la s1, _C_LABEL(_end) 1: sd zero, 0(a0) addi a0, a0, 8 bltu a0, s1, 1b /* Fill riscv_bootparams */ addi sp, sp, -40 la t0, pagetable_l1 sd t0, 0(sp) /* kern_l1pt */ sd s9, 8(sp) /* kern_phys */ la t0, initstack_end sd t0, 16(sp) /* kern_stack */ li t0, (VM_MAX_KERNEL_ADDRESS - 2 * L2_SIZE) sd t0, 24(sp) /* dtbp_virt */ sd s11, 32(sp) /* dtbp_phys */ mv a0, sp call _C_LABEL(initriscv) /* Off we go */ call _C_LABEL(mi_startup) .align 4 initstack: .space (PAGE_SIZE * KSTACK_PAGES) initstack_end: ENTRY(sigcode) mv a0, sp addi a0, a0, SF_UC 1: li t0, SYS_sigreturn ecall /* sigreturn failed, exit */ li t0, SYS_exit ecall j 1b END(sigcode) /* This may be copied to the stack, keep it 16-byte aligned */ .align 3 esigcode: .data .align 3 .global szsigcode szsigcode: .quad esigcode - sigcode .align 12 pagetable_l1: .space PAGE_SIZE pagetable_l2: .space PAGE_SIZE pagetable_l2_devmap: .space PAGE_SIZE .align 3 virt_map: .quad virt_map /* Not in use, but required for linking. */ .align 3 .globl __global_pointer$ __global_pointer$: .space 8 .globl init_pt_va init_pt_va: .quad pagetable_l2 /* XXX: Keep page tables VA */ #ifndef SMP ENTRY(mpentry) 1: wfi j 1b END(mpentry) #else /* * mpentry(unsigned long) * * Called by a core when it is being brought online. */ ENTRY(mpentry) /* Setup stack pointer */ la t0, secondary_stacks li t1, (PAGE_SIZE * KSTACK_PAGES) mulw t1, t1, s10 add t0, t0, t1 sub t0, t0, s9 li t1, KERNBASE add sp, t0, t1 /* Setup supervisor trap vector */ la t0, mpva sub t0, t0, s9 li t1, KERNBASE add t0, t0, t1 csrw stvec, t0 /* Set page tables base register */ la s2, pagetable_l1 srli s2, s2, PAGE_SHIFT li t0, SATP_MODE_SV39 or s2, s2, t0 sfence.vma csrw sptbr, s2 mpva: /* Setup supervisor trap vector */ la t0, cpu_exception_handler csrw stvec, t0 /* Ensure sscratch is zero */ li t0, 0 csrw sscratch, t0 /* * Calculate the offset to __riscv_boot_ap * for current core, cpuid in a0. */ li t1, 4 mulw t1, t1, a0 /* Get pointer */ la t0, __riscv_boot_ap add t0, t0, t1 1: /* Wait the kernel to be ready */ lw t1, 0(t0) beqz t1, 1b call init_secondary END(mpentry) #endif Index: head/sys/riscv/riscv/support.S =================================================================== --- head/sys/riscv/riscv/support.S (revision 331253) +++ head/sys/riscv/riscv/support.S (revision 331254) @@ -1,295 +1,295 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include -#include "assym.s" +#include "assym.inc" /* * One of the fu* or su* functions failed, return -1. */ ENTRY(fsu_fault) SET_FAULT_HANDLER(x0, a1) /* Reset the handler function */ fsu_fault_nopcb: li a0, -1 ret END(fsu_fault) /* * int casueword32(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) */ ENTRY(casueword32) li a4, (VM_MAXUSER_ADDRESS-3) bgt a0, a4, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a4) /* And set it */ 1: lr.w a4, 0(a0) /* Load-exclusive the data */ bne a4, a1, 2f /* If not equal then exit */ sc.w a5, a3, 0(a0) /* Store the new data */ bnez a5, 1b /* Retry on failure */ 2: SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ sw a4, 0(a2) /* Store the read data */ li a0, 0 /* Success */ ret /* Return */ END(casueword32) /* * int casueword(volatile u_long *, u_long, u_long *, u_long) */ ENTRY(casueword) li a4, (VM_MAXUSER_ADDRESS-7) bgt a0, a4, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a4) /* And set it */ 1: lr.d a4, 0(a0) /* Load-exclusive the data */ bne a4, a1, 2f /* If not equal then exit */ sc.d a5, a3, 0(a0) /* Store the new data */ bnez a5, 1b /* Retry on failure */ 2: SET_FAULT_HANDLER(x0, a5) /* Reset the fault handler */ sd a4, 0(a2) /* Store the read data */ li a0, 0 /* Success */ ret /* Return */ END(casueword) /* * int fubyte(volatile const void *) */ ENTRY(fubyte) li a1, VM_MAXUSER_ADDRESS bgt a0, a1, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a1) /* And set it */ lb a0, 0(a0) /* Try loading the data */ SET_FAULT_HANDLER(x0, a1) /* Reset the fault handler */ ret /* Return */ END(fubyte) /* * int fuword(volatile const void *) */ ENTRY(fuword16) li a1, (VM_MAXUSER_ADDRESS-1) bgt a0, a1, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a1) /* And set it */ lh a0, 0(a0) /* Try loading the data */ SET_FAULT_HANDLER(x0, a1) /* Reset the fault handler */ ret /* Return */ END(fuword16) /* * int32_t fueword32(volatile const void *, int32_t *) */ ENTRY(fueword32) li a2, (VM_MAXUSER_ADDRESS-3) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ lw a0, 0(a0) /* Try loading the data */ SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ sw a0, 0(a1) /* Save the data in kernel space */ li a0, 0 /* Success */ ret /* Return */ END(fueword32) /* * long fueword(volatile const void *, int64_t *) * int64_t fueword64(volatile const void *, int64_t *) */ ENTRY(fueword) EENTRY(fueword64) li a2, (VM_MAXUSER_ADDRESS-7) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ ld a0, 0(a0) /* Try loading the data */ SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ sd a0, 0(a1) /* Save the data in kernel space */ li a0, 0 /* Success */ ret /* Return */ EEND(fueword64) END(fueword) /* * int subyte(volatile void *, int) */ ENTRY(subyte) li a2, VM_MAXUSER_ADDRESS bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ sb a1, 0(a0) /* Try storing the data */ SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ END(subyte) /* * int suword16(volatile void *, int) */ ENTRY(suword16) li a2, (VM_MAXUSER_ADDRESS-1) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ sh a1, 0(a0) /* Try storing the data */ SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ END(suword16) /* * int suword32(volatile void *, int) */ ENTRY(suword32) li a2, (VM_MAXUSER_ADDRESS-3) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ sw a1, 0(a0) /* Try storing the data */ SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ END(suword32) /* * int suword(volatile void *, long) */ ENTRY(suword) EENTRY(suword64) li a2, (VM_MAXUSER_ADDRESS-7) bgt a0, a2, fsu_fault_nopcb la a6, fsu_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ sd a1, 0(a0) /* Try storing the data */ SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ EEND(suword64) END(suword) /* * fuswintr and suswintr are just like fusword and susword except that if * the page is not in memory or would cause a trap, then we return an error. * The important thing is to prevent sleep() and switch(). */ /* * Special handler so the trap code knows not to sleep. */ ENTRY(fsu_intr_fault) SET_FAULT_HANDLER(x0, a1) /* Reset the handler function */ li a0, -1 ret END(fsu_fault) /* * int fuswintr(void *) */ ENTRY(fuswintr) li a1, (VM_MAXUSER_ADDRESS-3) bgt a0, a1, fsu_fault_nopcb la a6, fsu_intr_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a1) /* And set it */ lw a0, 0(a0) /* Try loading the data */ SET_FAULT_HANDLER(x0, x1) /* Reset the fault handler */ ret /* Return */ END(fuswintr) /* * int suswintr(void *base, int word) */ ENTRY(suswintr) li a2, (VM_MAXUSER_ADDRESS-3) bgt a0, a2, fsu_fault_nopcb la a6, fsu_intr_fault /* Load the fault handler */ SET_FAULT_HANDLER(a6, a2) /* And set it */ sw a1, 0(a0) /* Try storing the data */ SET_FAULT_HANDLER(x0, a2) /* Reset the fault handler */ li a0, 0 /* Success */ ret /* Return */ END(suswintr) ENTRY(setjmp) /* Store the stack pointer */ sd sp, 0(a0) addi a0, a0, 8 /* Store the general purpose registers and ra */ sd s0, (0 * 8)(a0) sd s1, (1 * 8)(a0) sd s2, (2 * 8)(a0) sd s3, (3 * 8)(a0) sd s4, (4 * 8)(a0) sd s5, (5 * 8)(a0) sd s6, (6 * 8)(a0) sd s7, (7 * 8)(a0) sd s8, (8 * 8)(a0) sd s9, (9 * 8)(a0) sd s10, (10 * 8)(a0) sd s11, (11 * 8)(a0) sd ra, (12 * 8)(a0) /* Return value */ li a0, 0 ret END(setjmp) ENTRY(longjmp) /* Restore the stack pointer */ ld sp, 0(a0) addi a0, a0, 8 /* Restore the general purpose registers and ra */ ld s0, (0 * 8)(a0) ld s1, (1 * 8)(a0) ld s2, (2 * 8)(a0) ld s3, (3 * 8)(a0) ld s4, (4 * 8)(a0) ld s5, (5 * 8)(a0) ld s6, (6 * 8)(a0) ld s7, (7 * 8)(a0) ld s8, (8 * 8)(a0) ld s9, (9 * 8)(a0) ld s10, (10 * 8)(a0) ld s11, (11 * 8)(a0) ld ra, (12 * 8)(a0) /* Load the return value */ mv a0, a1 ret END(longjmp) Index: head/sys/riscv/riscv/swtch.S =================================================================== --- head/sys/riscv/riscv/swtch.S (revision 331253) +++ head/sys/riscv/riscv/swtch.S (revision 331254) @@ -1,440 +1,440 @@ /*- * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include "assym.s" +#include "assym.inc" #include "opt_sched.h" #include #include #include #include __FBSDID("$FreeBSD$"); #ifdef FPE .macro __fpe_state_save p /* * Enable FPE usage in supervisor mode, * so we can access registers. */ li t0, SSTATUS_FS_INITIAL csrs sstatus, t0 /* Store registers */ frcsr t0 sd t0, (PCB_FCSR)(\p) fsd f0, (PCB_X + 0 * 16)(\p) fsd f1, (PCB_X + 1 * 16)(\p) fsd f2, (PCB_X + 2 * 16)(\p) fsd f3, (PCB_X + 3 * 16)(\p) fsd f4, (PCB_X + 4 * 16)(\p) fsd f5, (PCB_X + 5 * 16)(\p) fsd f6, (PCB_X + 6 * 16)(\p) fsd f7, (PCB_X + 7 * 16)(\p) fsd f8, (PCB_X + 8 * 16)(\p) fsd f9, (PCB_X + 9 * 16)(\p) fsd f10, (PCB_X + 10 * 16)(\p) fsd f11, (PCB_X + 11 * 16)(\p) fsd f12, (PCB_X + 12 * 16)(\p) fsd f13, (PCB_X + 13 * 16)(\p) fsd f14, (PCB_X + 14 * 16)(\p) fsd f15, (PCB_X + 15 * 16)(\p) fsd f16, (PCB_X + 16 * 16)(\p) fsd f17, (PCB_X + 17 * 16)(\p) fsd f18, (PCB_X + 18 * 16)(\p) fsd f19, (PCB_X + 19 * 16)(\p) fsd f20, (PCB_X + 20 * 16)(\p) fsd f21, (PCB_X + 21 * 16)(\p) fsd f22, (PCB_X + 22 * 16)(\p) fsd f23, (PCB_X + 23 * 16)(\p) fsd f24, (PCB_X + 24 * 16)(\p) fsd f25, (PCB_X + 25 * 16)(\p) fsd f26, (PCB_X + 26 * 16)(\p) fsd f27, (PCB_X + 27 * 16)(\p) fsd f28, (PCB_X + 28 * 16)(\p) fsd f29, (PCB_X + 29 * 16)(\p) fsd f30, (PCB_X + 30 * 16)(\p) fsd f31, (PCB_X + 31 * 16)(\p) /* Disable FPE usage in supervisor mode. */ li t0, SSTATUS_FS_MASK csrc sstatus, t0 .endm .macro __fpe_state_load p /* * Enable FPE usage in supervisor mode, * so we can access registers. */ li t0, SSTATUS_FS_INITIAL csrs sstatus, t0 /* Restore registers */ ld t0, (PCB_FCSR)(\p) fscsr t0 fld f0, (PCB_X + 0 * 16)(\p) fld f1, (PCB_X + 1 * 16)(\p) fld f2, (PCB_X + 2 * 16)(\p) fld f3, (PCB_X + 3 * 16)(\p) fld f4, (PCB_X + 4 * 16)(\p) fld f5, (PCB_X + 5 * 16)(\p) fld f6, (PCB_X + 6 * 16)(\p) fld f7, (PCB_X + 7 * 16)(\p) fld f8, (PCB_X + 8 * 16)(\p) fld f9, (PCB_X + 9 * 16)(\p) fld f10, (PCB_X + 10 * 16)(\p) fld f11, (PCB_X + 11 * 16)(\p) fld f12, (PCB_X + 12 * 16)(\p) fld f13, (PCB_X + 13 * 16)(\p) fld f14, (PCB_X + 14 * 16)(\p) fld f15, (PCB_X + 15 * 16)(\p) fld f16, (PCB_X + 16 * 16)(\p) fld f17, (PCB_X + 17 * 16)(\p) fld f18, (PCB_X + 18 * 16)(\p) fld f19, (PCB_X + 19 * 16)(\p) fld f20, (PCB_X + 20 * 16)(\p) fld f21, (PCB_X + 21 * 16)(\p) fld f22, (PCB_X + 22 * 16)(\p) fld f23, (PCB_X + 23 * 16)(\p) fld f24, (PCB_X + 24 * 16)(\p) fld f25, (PCB_X + 25 * 16)(\p) fld f26, (PCB_X + 26 * 16)(\p) fld f27, (PCB_X + 27 * 16)(\p) fld f28, (PCB_X + 28 * 16)(\p) fld f29, (PCB_X + 29 * 16)(\p) fld f30, (PCB_X + 30 * 16)(\p) fld f31, (PCB_X + 31 * 16)(\p) /* Disable FPE usage in supervisor mode. */ li t0, SSTATUS_FS_MASK csrc sstatus, t0 .endm /* * void * fpe_state_save(struct thread *td) */ ENTRY(fpe_state_save) /* Get pointer to PCB */ ld a0, TD_PCB(a0) __fpe_state_save a0 ret END(fpe_state_save) #endif /* FPE */ /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) /* Store the new curthread */ sd a1, PC_CURTHREAD(gp) /* And the new pcb */ ld x13, TD_PCB(a1) sd x13, PC_CURPCB(gp) sfence.vma /* Switch to the new pmap */ ld t0, PCB_L1ADDR(x13) srli t0, t0, PAGE_SHIFT li t1, SATP_MODE_SV39 or t0, t0, t1 csrw sptbr, t0 /* TODO: Invalidate the TLB */ sfence.vma /* Load registers */ ld ra, (PCB_RA)(x13) ld sp, (PCB_SP)(x13) /* s[0-11] */ ld s0, (PCB_S + 0 * 8)(x13) ld s1, (PCB_S + 1 * 8)(x13) ld s2, (PCB_S + 2 * 8)(x13) ld s3, (PCB_S + 3 * 8)(x13) ld s4, (PCB_S + 4 * 8)(x13) ld s5, (PCB_S + 5 * 8)(x13) ld s6, (PCB_S + 6 * 8)(x13) ld s7, (PCB_S + 7 * 8)(x13) ld s8, (PCB_S + 8 * 8)(x13) ld s9, (PCB_S + 9 * 8)(x13) ld s10, (PCB_S + 10 * 8)(x13) ld s11, (PCB_S + 11 * 8)(x13) #ifdef FPE /* Is FPE enabled for new thread? */ ld t0, TD_FRAME(a1) ld t1, (TF_SSTATUS)(t0) li t2, SSTATUS_FS_MASK and t3, t1, t2 beqz t3, 1f /* No, skip. */ /* Restore registers. */ __fpe_state_load x13 1: #endif ret .Lcpu_throw_panic_str: .asciz "cpu_throw: %p\0" END(cpu_throw) /* * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) * * a0 = old * a1 = new * a2 = mtx * x3 to x7, x16 and x17 are caller saved */ ENTRY(cpu_switch) /* Store the new curthread */ sd a1, PC_CURTHREAD(gp) /* And the new pcb */ ld x13, TD_PCB(a1) sd x13, PC_CURPCB(gp) /* Save the old context. */ ld x13, TD_PCB(a0) /* Store ra, sp and the callee-saved registers */ sd ra, (PCB_RA)(x13) sd sp, (PCB_SP)(x13) /* s[0-11] */ sd s0, (PCB_S + 0 * 8)(x13) sd s1, (PCB_S + 1 * 8)(x13) sd s2, (PCB_S + 2 * 8)(x13) sd s3, (PCB_S + 3 * 8)(x13) sd s4, (PCB_S + 4 * 8)(x13) sd s5, (PCB_S + 5 * 8)(x13) sd s6, (PCB_S + 6 * 8)(x13) sd s7, (PCB_S + 7 * 8)(x13) sd s8, (PCB_S + 8 * 8)(x13) sd s9, (PCB_S + 9 * 8)(x13) sd s10, (PCB_S + 10 * 8)(x13) sd s11, (PCB_S + 11 * 8)(x13) #ifdef FPE /* * Is FPE enabled and is it in dirty state * for the old thread? */ ld t0, TD_FRAME(a0) ld t1, (TF_SSTATUS)(t0) li t2, SSTATUS_FS_MASK and t3, t1, t2 li t2, SSTATUS_FS_DIRTY bne t3, t2, 1f /* No, skip. */ /* Yes, mark FPE state clean and save registers. */ li t2, ~SSTATUS_FS_MASK and t3, t1, t2 li t2, SSTATUS_FS_CLEAN or t3, t3, t2 sd t3, (TF_SSTATUS)(t0) __fpe_state_save x13 1: #endif /* * Restore the saved context. */ ld x13, TD_PCB(a1) /* * TODO: We may need to flush the cache here if switching * to a user process. */ sfence.vma /* Switch to the new pmap */ ld t0, PCB_L1ADDR(x13) srli t0, t0, PAGE_SHIFT li t1, SATP_MODE_SV39 or t0, t0, t1 csrw sptbr, t0 /* TODO: Invalidate the TLB */ sfence.vma /* Release the old thread */ sd a2, TD_LOCK(a0) #if defined(SCHED_ULE) && defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ la a2, _C_LABEL(blocked_lock) 1: ld t0, TD_LOCK(a1) beq t0, a2, 1b #endif /* Restore the registers */ ld ra, (PCB_RA)(x13) ld sp, (PCB_SP)(x13) /* s[0-11] */ ld s0, (PCB_S + 0 * 8)(x13) ld s1, (PCB_S + 1 * 8)(x13) ld s2, (PCB_S + 2 * 8)(x13) ld s3, (PCB_S + 3 * 8)(x13) ld s4, (PCB_S + 4 * 8)(x13) ld s5, (PCB_S + 5 * 8)(x13) ld s6, (PCB_S + 6 * 8)(x13) ld s7, (PCB_S + 7 * 8)(x13) ld s8, (PCB_S + 8 * 8)(x13) ld s9, (PCB_S + 9 * 8)(x13) ld s10, (PCB_S + 10 * 8)(x13) ld s11, (PCB_S + 11 * 8)(x13) #ifdef FPE /* Is FPE enabled for new thread? */ ld t0, TD_FRAME(a1) ld t1, (TF_SSTATUS)(t0) li t2, SSTATUS_FS_MASK and t3, t1, t2 beqz t3, 1f /* No, skip. */ /* Restore registers. */ __fpe_state_load x13 1: #endif ret .Lcpu_switch_panic_str: .asciz "cpu_switch: %p\0" END(cpu_switch) /* * fork_exit(void (*callout)(void *, struct trapframe *), void *arg, * struct trapframe *frame) */ ENTRY(fork_trampoline) mv a0, s0 mv a1, s1 mv a2, sp call _C_LABEL(fork_exit) /* Restore sstatus */ ld t0, (TF_SSTATUS)(sp) /* Ensure interrupts disabled */ li t1, ~SSTATUS_SIE and t0, t0, t1 csrw sstatus, t0 /* Restore exception program counter */ ld t0, (TF_SEPC)(sp) csrw sepc, t0 /* Restore the registers */ ld t0, (TF_T + 0 * 8)(sp) ld t1, (TF_T + 1 * 8)(sp) ld t2, (TF_T + 2 * 8)(sp) ld t3, (TF_T + 3 * 8)(sp) ld t4, (TF_T + 4 * 8)(sp) ld t5, (TF_T + 5 * 8)(sp) ld t6, (TF_T + 6 * 8)(sp) ld s0, (TF_S + 0 * 8)(sp) ld s1, (TF_S + 1 * 8)(sp) ld s2, (TF_S + 2 * 8)(sp) ld s3, (TF_S + 3 * 8)(sp) ld s4, (TF_S + 4 * 8)(sp) ld s5, (TF_S + 5 * 8)(sp) ld s6, (TF_S + 6 * 8)(sp) ld s7, (TF_S + 7 * 8)(sp) ld s8, (TF_S + 8 * 8)(sp) ld s9, (TF_S + 9 * 8)(sp) ld s10, (TF_S + 10 * 8)(sp) ld s11, (TF_S + 11 * 8)(sp) ld a0, (TF_A + 0 * 8)(sp) ld a1, (TF_A + 1 * 8)(sp) ld a2, (TF_A + 2 * 8)(sp) ld a3, (TF_A + 3 * 8)(sp) ld a4, (TF_A + 4 * 8)(sp) ld a5, (TF_A + 5 * 8)(sp) ld a6, (TF_A + 6 * 8)(sp) ld a7, (TF_A + 7 * 8)(sp) /* Load user ra and sp */ ld tp, (TF_TP)(sp) ld ra, (TF_RA)(sp) /* * Store our pcpup on stack, we will load it back * on kernel mode trap. */ sd gp, (TF_SIZE)(sp) ld gp, (TF_GP)(sp) /* Save kernel stack so we can use it doing a user trap */ addi sp, sp, TF_SIZE csrw sscratch, sp /* Load user stack */ ld sp, (TF_SP - TF_SIZE)(sp) sret END(fork_trampoline) ENTRY(savectx) /* Store ra, sp and the callee-saved registers */ sd ra, (PCB_RA)(a0) sd sp, (PCB_SP)(a0) /* s[0-11] */ sd s0, (PCB_S + 0 * 8)(a0) sd s1, (PCB_S + 1 * 8)(a0) sd s2, (PCB_S + 2 * 8)(a0) sd s3, (PCB_S + 3 * 8)(a0) sd s4, (PCB_S + 4 * 8)(a0) sd s5, (PCB_S + 5 * 8)(a0) sd s6, (PCB_S + 6 * 8)(a0) sd s7, (PCB_S + 7 * 8)(a0) sd s8, (PCB_S + 8 * 8)(a0) sd s9, (PCB_S + 9 * 8)(a0) sd s10, (PCB_S + 10 * 8)(a0) sd s11, (PCB_S + 11 * 8)(a0) #ifdef FPE __fpe_state_save a0 #endif ret END(savectx) Index: head/sys/sparc64/sparc64/exception.S =================================================================== --- head/sys/sparc64/sparc64/exception.S (revision 331253) +++ head/sys/sparc64/sparc64/exception.S (revision 331254) @@ -1,3065 +1,3065 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * BSDI $Id: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp $ */ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" #define TSB_ASI 0x0 #define TSB_KERNEL 0x0 #define TSB_KERNEL_MASK 0x0 #define TSB_KERNEL_PHYS 0x0 #define TSB_KERNEL_PHYS_END 0x0 #define TSB_QUAD_LDD 0x0 .register %g2,#ignore .register %g3,#ignore .register %g6,#ignore .register %g7,#ignore /* * Atomically set a bit in a TTE. */ #define TTE_SET_BIT(r1, r2, r3, bit, a, asi) \ add r1, TTE_DATA, r1 ; \ LD(x, a) [r1] asi, r2 ; \ 9: or r2, bit, r3 ; \ CAS(x, a) [r1] asi, r2, r3 ; \ cmp r2, r3 ; \ bne,pn %xcc, 9b ; \ mov r3, r2 #define TTE_SET_REF(r1, r2, r3, a, asi) TTE_SET_BIT(r1, r2, r3, TD_REF, a, asi) #define TTE_SET_W(r1, r2, r3, a, asi) TTE_SET_BIT(r1, r2, r3, TD_W, a, asi) /* * Macros for spilling and filling live windows. * * NOTE: These macros use exactly 16 instructions, and it is assumed that the * handler will not use more than 24 instructions total, to leave room for * resume vectors which occupy the last 8 instructions. */ #define SPILL(storer, base, size, asi) \ storer %l0, [base + (0 * size)] asi ; \ storer %l1, [base + (1 * size)] asi ; \ storer %l2, [base + (2 * size)] asi ; \ storer %l3, [base + (3 * size)] asi ; \ storer %l4, [base + (4 * size)] asi ; \ storer %l5, [base + (5 * size)] asi ; \ storer %l6, [base + (6 * size)] asi ; \ storer %l7, [base + (7 * size)] asi ; \ storer %i0, [base + (8 * size)] asi ; \ storer %i1, [base + (9 * size)] asi ; \ storer %i2, [base + (10 * size)] asi ; \ storer %i3, [base + (11 * size)] asi ; \ storer %i4, [base + (12 * size)] asi ; \ storer %i5, [base + (13 * size)] asi ; \ storer %i6, [base + (14 * size)] asi ; \ storer %i7, [base + (15 * size)] asi #define FILL(loader, base, size, asi) \ loader [base + (0 * size)] asi, %l0 ; \ loader [base + (1 * size)] asi, %l1 ; \ loader [base + (2 * size)] asi, %l2 ; \ loader [base + (3 * size)] asi, %l3 ; \ loader [base + (4 * size)] asi, %l4 ; \ loader [base + (5 * size)] asi, %l5 ; \ loader [base + (6 * size)] asi, %l6 ; \ loader [base + (7 * size)] asi, %l7 ; \ loader [base + (8 * size)] asi, %i0 ; \ loader [base + (9 * size)] asi, %i1 ; \ loader [base + (10 * size)] asi, %i2 ; \ loader [base + (11 * size)] asi, %i3 ; \ loader [base + (12 * size)] asi, %i4 ; \ loader [base + (13 * size)] asi, %i5 ; \ loader [base + (14 * size)] asi, %i6 ; \ loader [base + (15 * size)] asi, %i7 #define ERRATUM50(reg) mov reg, reg #define KSTACK_SLOP 1024 /* * Sanity check the kernel stack and bail out if it's wrong. * XXX: doesn't handle being on the panic stack. */ #define KSTACK_CHECK \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ add %sp, SPOFF, %g1 ; \ andcc %g1, (1 << PTR_SHIFT) - 1, %g0 ; \ bnz,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [PCPU(CURTHREAD)], %g2 ; \ ldx [%g2 + TD_KSTACK], %g2 ; \ add %g2, KSTACK_SLOP, %g2 ; \ subcc %g1, %g2, %g1 ; \ ble,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ set KSTACK_PAGES * PAGE_SIZE, %g2 ; \ cmp %g1, %g2 ; \ bgt,a %xcc, tl1_kstack_fault ; \ inc 16, ASP_REG ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG .globl tl_text_begin tl_text_begin: nop ENTRY(tl1_kstack_fault) rdpr %tl, %g1 1: cmp %g1, 2 be,a 2f nop #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx" , %g2, %g3, %g4, 7, 8, 9) rdpr %tl, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %tnpc, %g3 stx %g3, [%g2 + KTR_PARM1] 9: #endif sub %g1, 1, %g1 wrpr %g1, 0, %tl ba,a %xcc, 1b nop 2: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx" , %g1, %g2, %g3, 7, 8, 9) add %sp, SPOFF, %g2 stx %g2, [%g1 + KTR_PARM1] ldx [PCPU(CURTHREAD)], %g2 ldx [%g2 + TD_KSTACK], %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %canrestore, %g2 stx %g2, [%g1 + KTR_PARM3] rdpr %cansave, %g2 stx %g2, [%g1 + KTR_PARM4] rdpr %otherwin, %g2 stx %g2, [%g1 + KTR_PARM5] rdpr %wstate, %g2 stx %g2, [%g1 + KTR_PARM6] 9: #endif wrpr %g0, 0, %canrestore wrpr %g0, 6, %cansave wrpr %g0, 0, %otherwin wrpr %g0, WSTATE_KERNEL, %wstate sub ASP_REG, SPOFF + CCFSZ, %sp clr %fp set trap, %o2 ba %xcc, tl1_trap mov T_KSTACK_FAULT | T_KERNEL, %o0 END(tl1_kstack_fault) /* * Magic to resume from a spill or fill trap. If we get an alignment or an * MMU fault during a spill or a fill, this macro will detect the fault and * resume at a set instruction offset in the trap handler. * * To check if the previous trap was a spill/fill we convert the trapped pc * to a trap type and verify that it is in the range of spill/fill vectors. * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the * tl bit allows us to detect both ranges with one test. * * This is: * 0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100 * * To calculate the new pc we take advantage of the xor feature of wrpr. * Forcing all the low bits of the trapped pc on we can produce any offset * into the spill/fill vector. The size of a spill/fill trap vector is 0x80. * * 0x7f ^ 0x1f == 0x60 * 0x1f == (0x80 - 0x60) - 1 * * Which are the offset and xor value used to resume from alignment faults. */ /* * Determine if we have trapped inside of a spill/fill vector, and if so resume * at a fixed instruction offset in the trap vector. Must be called on * alternate globals. */ #define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \ dec 16, ASP_REG ; \ stx %g1, [ASP_REG + 0] ; \ stx %g2, [ASP_REG + 8] ; \ rdpr %tpc, %g1 ; \ ERRATUM50(%g1) ; \ rdpr %tba, %g2 ; \ sub %g1, %g2, %g2 ; \ srlx %g2, 5, %g2 ; \ andn %g2, 0x200, %g2 ; \ cmp %g2, 0x80 ; \ blu,pt %xcc, 9f ; \ cmp %g2, 0x100 ; \ bgeu,pt %xcc, 9f ; \ or %g1, 0x7f, %g1 ; \ wrpr %g1, xor, %tnpc ; \ stxa_g0_sfsr ; \ ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG ; \ done ; \ 9: ldx [ASP_REG + 8], %g2 ; \ ldx [ASP_REG + 0], %g1 ; \ inc 16, ASP_REG /* * For certain faults we need to clear the SFSR MMU register before returning. */ #define RSF_CLR_SFSR \ wr %g0, ASI_DMMU, %asi ; \ stxa %g0, [%g0 + AA_DMMU_SFSR] %asi #define RSF_XOR(off) ((0x80 - off) - 1) /* * Instruction offsets in spill and fill trap handlers for handling certain * nested traps, and corresponding xor constants for wrpr. */ #define RSF_OFF_ALIGN 0x60 #define RSF_OFF_MMU 0x70 #define RESUME_SPILLFILL_ALIGN \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN)) #define RESUME_SPILLFILL_MMU \ RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU)) #define RESUME_SPILLFILL_MMU_CLR_SFSR \ RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU)) /* * Constant to add to %tnpc when taking a fill trap just before returning to * user mode. */ #define RSF_FILL_INC tl0_ret_fill_end - tl0_ret_fill /* * Generate a T_SPILL or T_FILL trap if the window operation fails. */ #define RSF_TRAP(type) \ ba %xcc, tl0_sftrap ; \ mov type, %g2 ; \ .align 16 /* * Game over if the window operation fails. */ #define RSF_FATAL(type) \ ba %xcc, rsf_fatal ; \ mov type, %g2 ; \ .align 16 /* * Magic to resume from a failed fill a few instructions after the corrsponding * restore. This is used on return from the kernel to usermode. */ #define RSF_FILL_MAGIC \ rdpr %tnpc, %g1 ; \ add %g1, RSF_FILL_INC, %g1 ; \ wrpr %g1, 0, %tnpc ; \ done ; \ .align 16 /* * Spill to the pcb if a spill to the user stack in kernel mode fails. */ #define RSF_SPILL_TOPCB \ ba,a %xcc, tl1_spill_topcb ; \ nop ; \ .align 16 ENTRY(rsf_fatal) #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" , %g1, %g3, %g4, 7, 8, 9) rdpr %tt, %g3 stx %g3, [%g1 + KTR_PARM1] stx %g2, [%g1 + KTR_PARM2] 9: #endif KSTACK_CHECK sir END(rsf_fatal) .data _ALIGN_DATA .globl intrnames, sintrnames intrnames: .space (IV_MAX + PIL_MAX) * (MAXCOMLEN + 1) sintrnames: .quad (IV_MAX + PIL_MAX) * (MAXCOMLEN + 1) .globl intrcnt, sintrcnt intrcnt: .space (IV_MAX + PIL_MAX) * 8 sintrcnt: .quad (IV_MAX + PIL_MAX) * 8 .text /* * Trap table and associated macros * * Due to its size a trap table is an inherently hard thing to represent in * code in a clean way. There are approximately 1024 vectors, of 8 or 32 * instructions each, many of which are identical. The way that this is * laid out is the instructions (8 or 32) for the actual trap vector appear * as an AS macro. In general this code branches to tl0_trap or tl1_trap, * but if not supporting code can be placed just after the definition of the * macro. The macros are then instantiated in a different section (.trap), * which is setup to be placed by the linker at the beginning of .text, and the * code around the macros is moved to the end of trap table. In this way the * code that must be sequential in memory can be split up, and located near * its supporting code so that it is easier to follow. */ /* * Clean window traps occur when %cleanwin is zero to ensure that data * is not leaked between address spaces in registers. */ .macro clean_window clr %o0 clr %o1 clr %o2 clr %o3 clr %o4 clr %o5 clr %o6 clr %o7 clr %l0 clr %l1 clr %l2 clr %l3 clr %l4 clr %l5 clr %l6 rdpr %cleanwin, %l7 inc %l7 wrpr %l7, 0, %cleanwin clr %l7 retry .align 128 .endm /* * Stack fixups for entry from user mode. We are still running on the * user stack, and with its live registers, so we must save soon. We * are on alternate globals so we do have some registers. Set the * transitional window state, and do the save. If this traps we * attempt to spill a window to the user stack. If this fails, we * spill the window to the pcb and continue. Spilling to the pcb * must not fail. * * NOTE: Must be called with alternate globals and clobbers %g1. */ .macro tl0_split rdpr %wstate, %g1 wrpr %g1, WSTATE_TRANSITION, %wstate save .endm .macro tl0_setup type tl0_split clr %o1 set trap, %o2 ba %xcc, tl0_utrap mov \type, %o0 .endm /* * Generic trap type. Call trap() with the specified type. */ .macro tl0_gen type tl0_setup \type .align 32 .endm /* * This is used to suck up the massive swaths of reserved trap types. * Generates count "reserved" trap vectors. */ .macro tl0_reserved count .rept \count tl0_gen T_RESERVED .endr .endm .macro tl1_split rdpr %wstate, %g1 wrpr %g1, WSTATE_NESTED, %wstate save %sp, -(CCFSZ + TF_SIZEOF), %sp .endm .macro tl1_setup type tl1_split clr %o1 set trap, %o2 ba %xcc, tl1_trap mov \type | T_KERNEL, %o0 .endm .macro tl1_gen type tl1_setup \type .align 32 .endm .macro tl1_reserved count .rept \count tl1_gen T_RESERVED .endr .endm .macro tl0_insn_excptn wrpr %g0, PSTATE_ALT, %pstate wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 /* * XXX in theory, a store to AA_IMMU_SFSR must be immediately * followed by a DONE, FLUSH or RETRY for USIII. In practice, * this triggers a RED state exception though. */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap mov T_INSTRUCTION_EXCEPTION, %g2 .align 32 .endm .macro tl0_data_excptn wrpr %g0, PSTATE_ALT, %pstate wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap mov T_DATA_EXCEPTION, %g2 .align 32 .endm .macro tl0_align wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 .align 32 .endm ENTRY(tl0_sfsr_trap) tl0_split clr %o1 set trap, %o2 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl0_utrap mov %g2, %o0 END(tl0_sfsr_trap) .macro tl0_intr level, mask tl0_split set \mask, %o1 ba %xcc, tl0_intr mov \level, %o0 .align 32 .endm #define INTR(level, traplvl) \ tl ## traplvl ## _intr level, 1 << level #define TICK(traplvl) \ tl ## traplvl ## _intr PIL_TICK, 0x10001 #define INTR_LEVEL(tl) \ INTR(1, tl) ; \ INTR(2, tl) ; \ INTR(3, tl) ; \ INTR(4, tl) ; \ INTR(5, tl) ; \ INTR(6, tl) ; \ INTR(7, tl) ; \ INTR(8, tl) ; \ INTR(9, tl) ; \ INTR(10, tl) ; \ INTR(11, tl) ; \ INTR(12, tl) ; \ INTR(13, tl) ; \ TICK(tl) ; \ INTR(15, tl) ; .macro tl0_intr_level INTR_LEVEL(0) .endm .macro intr_vector ldxa [%g0] ASI_INTR_RECEIVE, %g1 andcc %g1, IRSR_BUSY, %g0 bnz,a,pt %xcc, intr_vector nop ba,a,pt %xcc, intr_vector_stray nop .align 32 .endm .macro tl0_immu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g1 /* * Initialize the page size walker. */ mov TS_MIN, %g2 /* * Loop over all supported page sizes. */ /* * Compute the page shift for the page size we are currently looking * for. */ 1: add %g2, %g2, %g3 add %g3, %g2, %g3 add %g3, PAGE_SHIFT, %g3 /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g1, %g3, %g3 /* * Compute the TTE bucket address. */ ldxa [%g0 + AA_IMMU_TSB] %asi, %g5 and %g3, TSB_BUCKET_MASK, %g4 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 add %g4, %g5, %g4 /* * Compute the TTE tag target. */ sllx %g3, TV_SIZE_BITS, %g3 or %g3, %g2, %g3 /* * Loop over the TTEs in this bucket. */ /* * Load the TTE. Note that this instruction may fault, clobbering * the contents of the tag access register, %g5, %g6, and %g7. We * do not use %g5, and %g6 and %g7 are not used until this instruction * completes successfully. */ 2: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and executable and that the TTE tags match. */ brgez,pn %g7, 3f andcc %g7, TD_EXEC, %g0 bz,pn %xcc, 3f cmp %g3, %g6 bne,pn %xcc, 3f EMPTY /* * We matched a TTE, load the TLB. */ /* * Set the reference bit, if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pn %xcc, tl0_immu_miss_set_ref nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_IMMU_TAR] %asi stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG retry /* * Advance to the next TTE in this bucket, and check the low bits * of the bucket pointer to see if we've finished the bucket. */ 3: add %g4, 1 << TTE_SHIFT, %g4 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,pt %xcc, 2b EMPTY /* * See if we just checked the largest page size, and advance to the * next one if not. */ cmp %g2, TS_MAX bne,pt %xcc, 1b add %g2, 1, %g2 /* * Not in user TSB, call C code. */ ba,a %xcc, tl0_immu_miss_trap .align 128 .endm ENTRY(tl0_immu_miss_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g4, %g2, %g3, a, ASI_N) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g2, 1f nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_IMMU_TAR] %asi stxa %g2, [%g0] ASI_ITLB_DATA_IN_REG 1: retry END(tl0_immu_miss_set_ref) ENTRY(tl0_immu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ sethi %hi(KERNBASE), %g2 stxa %g1, [%g0 + AA_IMMU_TAR] %asi flush %g2 /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Reload the tag access register. */ ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 /* * Save the tag access register, and call common trap code. */ tl0_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl0_utrap mov T_INSTRUCTION_MISS, %o0 END(tl0_immu_miss_trap) .macro tl0_dmmu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 /* * Initialize the page size walker. */ tl1_dmmu_miss_user: mov TS_MIN, %g2 /* * Loop over all supported page sizes. */ /* * Compute the page shift for the page size we are currently looking * for. */ 1: add %g2, %g2, %g3 add %g3, %g2, %g3 add %g3, PAGE_SHIFT, %g3 /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g1, %g3, %g3 /* * Compute the TTE bucket address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 and %g3, TSB_BUCKET_MASK, %g4 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 add %g4, %g5, %g4 /* * Compute the TTE tag target. */ sllx %g3, TV_SIZE_BITS, %g3 or %g3, %g2, %g3 /* * Loop over the TTEs in this bucket. */ /* * Load the TTE. Note that this instruction may fault, clobbering * the contents of the tag access register, %g5, %g6, and %g7. We * do not use %g5, and %g6 and %g7 are not used until this instruction * completes successfully. */ 2: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and that the virtual page numbers match. */ brgez,pn %g7, 3f cmp %g3, %g6 bne,pn %xcc, 3f EMPTY /* * We matched a TTE, load the TLB. */ /* * Set the reference bit, if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pn %xcc, tl0_dmmu_miss_set_ref nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG retry /* * Advance to the next TTE in this bucket, and check the low bits * of the bucket pointer to see if we've finished the bucket. */ 3: add %g4, 1 << TTE_SHIFT, %g4 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,pt %xcc, 2b EMPTY /* * See if we just checked the largest page size, and advance to the * next one if not. */ cmp %g2, TS_MAX bne,pt %xcc, 1b add %g2, 1, %g2 /* * Not in user TSB, call C code. */ ba,a %xcc, tl0_dmmu_miss_trap .align 128 .endm ENTRY(tl0_dmmu_miss_set_ref) /* * Set the reference bit. */ TTE_SET_REF(%g4, %g2, %g3, a, ASI_N) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g2, 1f nop /* * Load the TTE tag and data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(tl0_dmmu_miss_set_ref) ENTRY(tl0_dmmu_miss_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Check if we actually came from the kernel. */ rdpr %tl, %g1 cmp %g1, 1 bgt,a,pn %xcc, 1f nop /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 /* * Save the tag access register and call common trap code. */ tl0_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl0_utrap mov T_DATA_MISS, %o0 /* * Handle faults during window spill/fill. */ 1: RESUME_SPILLFILL_MMU /* * Reload the tag access register. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 tl1_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl0_dmmu_miss_trap) .macro tl0_dmmu_prot ba,a %xcc, tl0_dmmu_prot_1 nop .align 128 .endm ENTRY(tl0_dmmu_prot_1) /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g1 /* * Initialize the page size walker. */ tl1_dmmu_prot_user: mov TS_MIN, %g2 /* * Loop over all supported page sizes. */ /* * Compute the page shift for the page size we are currently looking * for. */ 1: add %g2, %g2, %g3 add %g3, %g2, %g3 add %g3, PAGE_SHIFT, %g3 /* * Extract the virtual page number from the contents of the tag * access register. */ srlx %g1, %g3, %g3 /* * Compute the TTE bucket address. */ ldxa [%g0 + AA_DMMU_TSB] %asi, %g5 and %g3, TSB_BUCKET_MASK, %g4 sllx %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4 add %g4, %g5, %g4 /* * Compute the TTE tag target. */ sllx %g3, TV_SIZE_BITS, %g3 or %g3, %g2, %g3 /* * Loop over the TTEs in this bucket. */ /* * Load the TTE. Note that this instruction may fault, clobbering * the contents of the tag access register, %g5, %g6, and %g7. We * do not use %g5, and %g6 and %g7 are not used until this instruction * completes successfully. */ 2: ldda [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and writable and that the virtual page * numbers match. */ brgez,pn %g7, 4f andcc %g7, TD_SW, %g0 bz,pn %xcc, 4f cmp %g3, %g6 bne,pn %xcc, 4f nop /* * Set the hardware write bit. */ TTE_SET_W(%g4, %g2, %g3, a, ASI_N) /* * Delete the old TLB entry and clear the SFSR. */ srlx %g1, PAGE_SHIFT, %g3 sllx %g3, PAGE_SHIFT, %g3 stxa %g0, [%g3] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g2, 3f or %g2, TD_W, %g2 /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi stxa %g2, [%g0] ASI_DTLB_DATA_IN_REG 3: retry /* * Check the low bits to see if we've finished the bucket. */ 4: add %g4, 1 << TTE_SHIFT, %g4 andcc %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0 bnz,pt %xcc, 2b EMPTY /* * See if we just checked the largest page size, and advance to the * next one if not. */ cmp %g2, TS_MAX bne,pt %xcc, 1b add %g2, 1, %g2 /* * Not in user TSB, call C code. */ ba,a %xcc, tl0_dmmu_prot_trap nop END(tl0_dmmu_prot_1) ENTRY(tl0_dmmu_prot_trap) /* * Put back the contents of the tag access register, in case we * faulted. */ stxa %g1, [%g0 + AA_DMMU_TAR] %asi membar #Sync /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Check if we actually came from the kernel. */ rdpr %tl, %g1 cmp %g1, 1 bgt,a,pn %xcc, 1f nop /* * Load the SFAR, SFSR and TAR. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * Save the MMU registers and call common trap code. */ tl0_split clr %o1 set trap, %o2 mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl0_utrap mov T_DATA_PROTECTION, %o0 /* * Handle faults during window spill/fill. */ 1: RESUME_SPILLFILL_MMU_CLR_SFSR /* * Load the SFAR, SFSR and TAR. Clear the SFSR. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split clr %o1 set trap, %o2 mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl0_dmmu_prot_trap) .macro tl0_spill_0_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_spill_1_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_TRAP(T_SPILL) RSF_TRAP(T_SPILL) .endm .macro tl0_fill_0_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm .macro tl0_fill_1_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_TRAP(T_FILL) RSF_TRAP(T_FILL) .endm ENTRY(tl0_sftrap) rdpr %tstate, %g1 and %g1, TSTATE_CWP_MASK, %g1 wrpr %g1, 0, %cwp tl0_split clr %o1 set trap, %o2 ba %xcc, tl0_trap mov %g2, %o0 END(tl0_sftrap) .macro tl0_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl0_syscall tl0_split clr %o1 set syscall, %o2 ba %xcc, tl0_trap mov T_SYSCALL, %o0 .align 32 .endm .macro tl0_fp_restore ba,a %xcc, tl0_fp_restore nop .align 32 .endm ENTRY(tl0_fp_restore) ldx [PCB_REG + PCB_FLAGS], %g1 andn %g1, PCB_FEF, %g1 stx %g1, [PCB_REG + PCB_FLAGS] wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi ldda [PCB_REG + PCB_UFP + (0 * 64)] %asi, %f0 ldda [PCB_REG + PCB_UFP + (1 * 64)] %asi, %f16 ldda [PCB_REG + PCB_UFP + (2 * 64)] %asi, %f32 ldda [PCB_REG + PCB_UFP + (3 * 64)] %asi, %f48 membar #Sync done END(tl0_fp_restore) .macro tl1_insn_excptn wrpr %g0, PSTATE_ALT, %pstate wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 /* * XXX in theory, a store to AA_IMMU_SFSR must be immediately * followed by a DONE, FLUSH or RETRY for USIII. In practice, * this triggers a RED state exception though. */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl1_insn_exceptn_trap mov T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2 .align 32 .endm ENTRY(tl1_insn_exceptn_trap) tl1_split clr %o1 set trap, %o2 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov %g2, %o0 END(tl1_insn_exceptn_trap) .macro tl1_fp_disabled ba,a %xcc, tl1_fp_disabled_1 nop .align 32 .endm ENTRY(tl1_fp_disabled_1) rdpr %tpc, %g1 set fpu_fault_begin, %g2 sub %g1, %g2, %g1 cmp %g1, fpu_fault_size bgeu,a,pn %xcc, 1f nop wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi ldda [PCB_REG + PCB_KFP + (0 * 64)] %asi, %f0 ldda [PCB_REG + PCB_KFP + (1 * 64)] %asi, %f16 ldda [PCB_REG + PCB_KFP + (2 * 64)] %asi, %f32 ldda [PCB_REG + PCB_KFP + (3 * 64)] %asi, %f48 membar #Sync retry 1: tl1_split clr %o1 set trap, %o2 ba %xcc, tl1_trap mov T_FP_DISABLED | T_KERNEL, %o0 END(tl1_fp_disabled_1) .macro tl1_data_excptn wrpr %g0, PSTATE_ALT, %pstate ba,a %xcc, tl1_data_excptn_trap nop .align 32 .endm ENTRY(tl1_data_excptn_trap) RESUME_SPILLFILL_MMU_CLR_SFSR ba %xcc, tl1_sfsr_trap mov T_DATA_EXCEPTION | T_KERNEL, %g2 END(tl1_data_excptn_trap) .macro tl1_align wrpr %g0, PSTATE_ALT, %pstate ba,a %xcc, tl1_align_trap nop .align 32 .endm ENTRY(tl1_align_trap) RESUME_SPILLFILL_ALIGN ba %xcc, tl1_sfsr_trap mov T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2 END(tl1_align_trap) ENTRY(tl1_sfsr_trap) wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split clr %o1 set trap, %o2 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov %g2, %o0 END(tl1_sfsr_trap) .macro tl1_intr level, mask tl1_split set \mask, %o1 ba %xcc, tl1_intr mov \level, %o0 .align 32 .endm .macro tl1_intr_level INTR_LEVEL(1) .endm .macro tl1_immu_miss /* * Load the context and the virtual page number from the tag access * register. We ignore the context. */ wr %g0, ASI_IMMU, %asi ldxa [%g0 + AA_IMMU_TAR] %asi, %g5 /* * Compute the address of the TTE. The TSB mask and address of the * TSB are patched at startup. */ .globl tl1_immu_miss_patch_tsb_1 tl1_immu_miss_patch_tsb_1: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_immu_miss_patch_tsb_mask_1 tl1_immu_miss_patch_tsb_mask_1: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 srlx %g5, TAR_VPN_SHIFT, %g5 and %g5, %g6, %g6 sllx %g6, TTE_SHIFT, %g6 add %g6, %g7, %g6 /* * Load the TTE. */ .globl tl1_immu_miss_patch_quad_ldd_1 tl1_immu_miss_patch_quad_ldd_1: ldda [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and executable and that the virtual page * numbers match. */ brgez,pn %g7, tl1_immu_miss_trap andcc %g7, TD_EXEC, %g0 bz,pn %xcc, tl1_immu_miss_trap srlx %g6, TV_SIZE_BITS, %g6 cmp %g5, %g6 bne,pn %xcc, tl1_immu_miss_trap EMPTY /* * Set the reference bit if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pn %xcc, tl1_immu_miss_set_ref nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g7, [%g0] ASI_ITLB_DATA_IN_REG retry .align 128 .endm ENTRY(tl1_immu_miss_set_ref) /* * Recompute the TTE address, which we clobbered loading the TTE. * The TSB mask and address of the TSB are patched at startup. */ .globl tl1_immu_miss_patch_tsb_2 tl1_immu_miss_patch_tsb_2: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_immu_miss_patch_tsb_mask_2 tl1_immu_miss_patch_tsb_mask_2: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 and %g5, %g6, %g5 sllx %g5, TTE_SHIFT, %g5 add %g5, %g7, %g5 /* * Set the reference bit. */ .globl tl1_immu_miss_patch_asi_1 tl1_immu_miss_patch_asi_1: wr %g0, TSB_ASI, %asi TTE_SET_REF(%g5, %g6, %g7, a, %asi) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g6, 1f nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g6, [%g0] ASI_ITLB_DATA_IN_REG 1: retry END(tl1_immu_miss_set_ref) ENTRY(tl1_immu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate ldxa [%g0 + AA_IMMU_TAR] %asi, %g2 tl1_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl1_trap mov T_INSTRUCTION_MISS | T_KERNEL, %o0 END(tl1_immu_miss_trap) .macro tl1_dmmu_miss /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 /* * Extract the context from the contents of the tag access register. * If it's non-zero this is a fault on a user address. Note that the * faulting address is passed in %g1. */ sllx %g5, 64 - TAR_VPN_SHIFT, %g6 brnz,a,pn %g6, tl1_dmmu_miss_user mov %g5, %g1 /* * Check for the direct mapped physical region. These addresses have * the high bit set so they are negative. */ brlz,pn %g5, tl1_dmmu_miss_direct EMPTY /* * Compute the address of the TTE. The TSB mask and address of the * TSB are patched at startup. */ .globl tl1_dmmu_miss_patch_tsb_1 tl1_dmmu_miss_patch_tsb_1: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_miss_patch_tsb_mask_1 tl1_dmmu_miss_patch_tsb_mask_1: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 srlx %g5, TAR_VPN_SHIFT, %g5 and %g5, %g6, %g6 sllx %g6, TTE_SHIFT, %g6 add %g6, %g7, %g6 /* * Load the TTE. */ .globl tl1_dmmu_miss_patch_quad_ldd_1 tl1_dmmu_miss_patch_quad_ldd_1: ldda [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and that the virtual page numbers match. */ brgez,pn %g7, tl1_dmmu_miss_trap srlx %g6, TV_SIZE_BITS, %g6 cmp %g5, %g6 bne,pn %xcc, tl1_dmmu_miss_trap EMPTY /* * Set the reference bit if it's currently clear. */ andcc %g7, TD_REF, %g0 bz,a,pt %xcc, tl1_dmmu_miss_set_ref nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g7, [%g0] ASI_DTLB_DATA_IN_REG retry .align 128 .endm ENTRY(tl1_dmmu_miss_set_ref) /* * Recompute the TTE address, which we clobbered loading the TTE. * The TSB mask and address of the TSB are patched at startup. */ .globl tl1_dmmu_miss_patch_tsb_mask_2 tl1_dmmu_miss_patch_tsb_2: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_miss_patch_tsb_2 tl1_dmmu_miss_patch_tsb_mask_2: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 and %g5, %g6, %g5 sllx %g5, TTE_SHIFT, %g5 add %g5, %g7, %g5 /* * Set the reference bit. */ .globl tl1_dmmu_miss_patch_asi_1 tl1_dmmu_miss_patch_asi_1: wr %g0, TSB_ASI, %asi TTE_SET_REF(%g5, %g6, %g7, a, %asi) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g6, 1f nop /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(tl1_dmmu_miss_set_ref) ENTRY(tl1_dmmu_miss_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 KSTACK_CHECK tl1_split clr %o1 set trap, %o2 mov %g2, %o3 ba %xcc, tl1_trap mov T_DATA_MISS | T_KERNEL, %o0 END(tl1_dmmu_miss_trap) ENTRY(tl1_dmmu_miss_direct) /* * Mask off the high bits of the virtual address to get the physical * address, and or in the TTE bits. The virtual address bits that * correspond to the TTE valid and page size bits are left set, so * they don't have to be included in the TTE bits below. We know they * are set because the virtual address is in the upper va hole. * NB: if we are taking advantage of the ASI_ATOMIC_QUAD_LDD_PHYS * and we get a miss on the directly accessed kernel TSB we must not * set TD_CV in order to access it uniformly bypassing the D$. */ setx TLB_DIRECT_ADDRESS_MASK, %g7, %g4 and %g5, %g4, %g4 setx TLB_DIRECT_TO_TTE_MASK, %g7, %g6 and %g5, %g6, %g5 .globl tl1_dmmu_miss_direct_patch_tsb_phys_1 tl1_dmmu_miss_direct_patch_tsb_phys_1: sethi %uhi(TSB_KERNEL_PHYS), %g3 or %g3, %ulo(TSB_KERNEL_PHYS), %g3 sllx %g3, 32, %g3 sethi %hi(TSB_KERNEL_PHYS), %g3 or %g7, %g3, %g7 cmp %g4, %g7 bl,pt %xcc, 1f or %g5, TD_CP | TD_W, %g5 .globl tl1_dmmu_miss_direct_patch_tsb_phys_end_1 tl1_dmmu_miss_direct_patch_tsb_phys_end_1: sethi %uhi(TSB_KERNEL_PHYS_END), %g3 or %g3, %ulo(TSB_KERNEL_PHYS_END), %g3 sllx %g3, 32, %g3 sethi %hi(TSB_KERNEL_PHYS_END), %g7 or %g7, %g3, %g7 cmp %g4, %g7 bg,a,pt %xcc, 1f nop ba,pt %xcc, 2f nop 1: or %g5, TD_CV, %g5 /* * Load the TTE data into the TLB and retry the instruction. */ 2: stxa %g5, [%g0] ASI_DTLB_DATA_IN_REG retry END(tl1_dmmu_miss_direct) .macro tl1_dmmu_prot ba,a %xcc, tl1_dmmu_prot_1 nop .align 128 .endm ENTRY(tl1_dmmu_prot_1) /* * Load the context and the virtual page number from the tag access * register. */ wr %g0, ASI_DMMU, %asi ldxa [%g0 + AA_DMMU_TAR] %asi, %g5 /* * Extract the context from the contents of the tag access register. * If it's non-zero this is a fault on a user address. Note that the * faulting address is passed in %g1. */ sllx %g5, 64 - TAR_VPN_SHIFT, %g6 brnz,a,pn %g6, tl1_dmmu_prot_user mov %g5, %g1 /* * Compute the address of the TTE. The TSB mask and address of the * TSB are patched at startup. */ .globl tl1_dmmu_prot_patch_tsb_1 tl1_dmmu_prot_patch_tsb_1: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_prot_patch_tsb_mask_1 tl1_dmmu_prot_patch_tsb_mask_1: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 srlx %g5, TAR_VPN_SHIFT, %g5 and %g5, %g6, %g6 sllx %g6, TTE_SHIFT, %g6 add %g6, %g7, %g6 /* * Load the TTE. */ .globl tl1_dmmu_prot_patch_quad_ldd_1 tl1_dmmu_prot_patch_quad_ldd_1: ldda [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */ /* * Check that it's valid and writeable and that the virtual page * numbers match. */ brgez,pn %g7, tl1_dmmu_prot_trap andcc %g7, TD_SW, %g0 bz,pn %xcc, tl1_dmmu_prot_trap srlx %g6, TV_SIZE_BITS, %g6 cmp %g5, %g6 bne,pn %xcc, tl1_dmmu_prot_trap EMPTY /* * Delete the old TLB entry and clear the SFSR. */ sllx %g5, TAR_VPN_SHIFT, %g6 or %g6, TLB_DEMAP_NUCLEUS, %g6 stxa %g0, [%g6] ASI_DMMU_DEMAP stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync /* * Recompute the TTE address, which we clobbered loading the TTE. * The TSB mask and address of the TSB are patched at startup. */ .globl tl1_dmmu_prot_patch_tsb_2 tl1_dmmu_prot_patch_tsb_2: sethi %uhi(TSB_KERNEL), %g6 or %g6, %ulo(TSB_KERNEL), %g6 sllx %g6, 32, %g6 sethi %hi(TSB_KERNEL), %g7 or %g7, %g6, %g7 .globl tl1_dmmu_prot_patch_tsb_mask_2 tl1_dmmu_prot_patch_tsb_mask_2: sethi %hi(TSB_KERNEL_MASK), %g6 or %g6, %lo(TSB_KERNEL_MASK), %g6 and %g5, %g6, %g5 sllx %g5, TTE_SHIFT, %g5 add %g5, %g7, %g5 /* * Set the hardware write bit. */ .globl tl1_dmmu_prot_patch_asi_1 tl1_dmmu_prot_patch_asi_1: wr %g0, TSB_ASI, %asi TTE_SET_W(%g5, %g6, %g7, a, %asi) /* * May have become invalid during casxa, in which case start over. */ brgez,pn %g6, 1f or %g6, TD_W, %g6 /* * Load the TTE data into the TLB and retry the instruction. */ stxa %g6, [%g0] ASI_DTLB_DATA_IN_REG 1: retry END(tl1_dmmu_prot_1) ENTRY(tl1_dmmu_prot_trap) /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Load the SFAR, SFSR and TAR. Clear the SFSR. */ ldxa [%g0 + AA_DMMU_TAR] %asi, %g2 ldxa [%g0 + AA_DMMU_SFAR] %asi, %g3 ldxa [%g0 + AA_DMMU_SFSR] %asi, %g4 stxa %g0, [%g0 + AA_DMMU_SFSR] %asi membar #Sync tl1_split clr %o1 set trap, %o2 mov %g2, %o3 mov %g3, %o4 mov %g4, %o5 ba %xcc, tl1_trap mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl1_dmmu_prot_trap) .macro tl1_spill_0_n SPILL(stx, %sp + SPOFF, 8, EMPTY) saved retry .align 32 RSF_FATAL(T_SPILL) RSF_FATAL(T_SPILL) .endm .macro tl1_spill_2_n wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_3_n wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_7_n btst 1, %sp bnz,a,pn %xcc, tl1_spill_0_n nop srl %sp, 0, %sp SPILL(stw, %sp, 4, EMPTY) saved retry .align 32 RSF_FATAL(T_SPILL) RSF_FATAL(T_SPILL) .endm .macro tl1_spill_0_o wr %g0, ASI_AIUP, %asi SPILL(stxa, %sp + SPOFF, 8, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_1_o wr %g0, ASI_AIUP, %asi SPILL(stwa, %sp, 4, %asi) saved retry .align 32 RSF_SPILL_TOPCB RSF_SPILL_TOPCB .endm .macro tl1_spill_2_o RSF_SPILL_TOPCB .align 128 .endm .macro tl1_fill_0_n FILL(ldx, %sp + SPOFF, 8, EMPTY) restored retry .align 32 RSF_FATAL(T_FILL) RSF_FATAL(T_FILL) .endm .macro tl1_fill_2_n wr %g0, ASI_AIUP, %asi FILL(ldxa, %sp + SPOFF, 8, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm .macro tl1_fill_3_n wr %g0, ASI_AIUP, %asi FILL(lduwa, %sp, 4, %asi) restored retry .align 32 RSF_FILL_MAGIC RSF_FILL_MAGIC .endm .macro tl1_fill_7_n btst 1, %sp bnz,a,pt %xcc, tl1_fill_0_n nop srl %sp, 0, %sp FILL(lduw, %sp, 4, EMPTY) restored retry .align 32 RSF_FATAL(T_FILL) RSF_FATAL(T_FILL) .endm /* * This is used to spill windows that are still occupied with user * data on kernel entry to the pcb. */ ENTRY(tl1_spill_topcb) wrpr %g0, PSTATE_ALT, %pstate /* Free some globals for our use. */ dec 24, ASP_REG stx %g1, [ASP_REG + 0] stx %g2, [ASP_REG + 8] stx %g3, [ASP_REG + 16] ldx [PCB_REG + PCB_NSAVED], %g1 sllx %g1, PTR_SHIFT, %g2 add %g2, PCB_REG, %g2 stx %sp, [%g2 + PCB_RWSP] sllx %g1, RW_SHIFT, %g2 add %g2, PCB_REG, %g2 SPILL(stx, %g2 + PCB_RW, 8, EMPTY) inc %g1 stx %g1, [PCB_REG + PCB_NSAVED] #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" , %g1, %g2, %g3, 7, 8, 9) rdpr %tpc, %g2 stx %g2, [%g1 + KTR_PARM1] rdpr %tnpc, %g2 stx %g2, [%g1 + KTR_PARM2] stx %sp, [%g1 + KTR_PARM3] ldx [PCB_REG + PCB_NSAVED], %g2 stx %g2, [%g1 + KTR_PARM4] 9: #endif saved ldx [ASP_REG + 16], %g3 ldx [ASP_REG + 8], %g2 ldx [ASP_REG + 0], %g1 inc 24, ASP_REG retry END(tl1_spill_topcb) .macro tl1_spill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_fill_bad count .rept \count sir .align 128 .endr .endm .macro tl1_soft count .rept \count tl1_gen T_SOFT | T_KERNEL .endr .endm .sect .trap .globl tl_trap_begin tl_trap_begin: nop .align 0x8000 .globl tl0_base tl0_base: tl0_reserved 8 ! 0x0-0x7 tl0_insn_excptn: tl0_insn_excptn ! 0x8 tl0_reserved 1 ! 0x9 tl0_insn_error: tl0_gen T_INSTRUCTION_ERROR ! 0xa tl0_reserved 5 ! 0xb-0xf tl0_insn_illegal: tl0_gen T_ILLEGAL_INSTRUCTION ! 0x10 tl0_priv_opcode: tl0_gen T_PRIVILEGED_OPCODE ! 0x11 tl0_reserved 14 ! 0x12-0x1f tl0_fp_disabled: tl0_gen T_FP_DISABLED ! 0x20 tl0_fp_ieee: tl0_gen T_FP_EXCEPTION_IEEE_754 ! 0x21 tl0_fp_other: tl0_gen T_FP_EXCEPTION_OTHER ! 0x22 tl0_tag_ovflw: tl0_gen T_TAG_OVERFLOW ! 0x23 tl0_clean_window: clean_window ! 0x24 tl0_divide: tl0_gen T_DIVISION_BY_ZERO ! 0x28 tl0_reserved 7 ! 0x29-0x2f tl0_data_excptn: tl0_data_excptn ! 0x30 tl0_reserved 1 ! 0x31 tl0_data_error: tl0_gen T_DATA_ERROR ! 0x32 tl0_reserved 1 ! 0x33 tl0_align: tl0_align ! 0x34 tl0_align_lddf: tl0_gen T_RESERVED ! 0x35 tl0_align_stdf: tl0_gen T_RESERVED ! 0x36 tl0_priv_action: tl0_gen T_PRIVILEGED_ACTION ! 0x37 tl0_reserved 9 ! 0x38-0x40 tl0_intr_level: tl0_intr_level ! 0x41-0x4f tl0_reserved 16 ! 0x50-0x5f tl0_intr_vector: intr_vector ! 0x60 tl0_watch_phys: tl0_gen T_PA_WATCHPOINT ! 0x61 tl0_watch_virt: tl0_gen T_VA_WATCHPOINT ! 0x62 tl0_ecc: tl0_gen T_CORRECTED_ECC_ERROR ! 0x63 tl0_immu_miss: tl0_immu_miss ! 0x64 tl0_dmmu_miss: tl0_dmmu_miss ! 0x68 tl0_dmmu_prot: tl0_dmmu_prot ! 0x6c tl0_reserved 16 ! 0x70-0x7f tl0_spill_0_n: tl0_spill_0_n ! 0x80 tl0_spill_1_n: tl0_spill_1_n ! 0x84 tl0_spill_bad 14 ! 0x88-0xbf tl0_fill_0_n: tl0_fill_0_n ! 0xc0 tl0_fill_1_n: tl0_fill_1_n ! 0xc4 tl0_fill_bad 14 ! 0xc8-0xff tl0_soft: tl0_gen T_SYSCALL ! 0x100 tl0_gen T_BREAKPOINT ! 0x101 tl0_gen T_DIVISION_BY_ZERO ! 0x102 tl0_reserved 1 ! 0x103 tl0_gen T_CLEAN_WINDOW ! 0x104 tl0_gen T_RANGE_CHECK ! 0x105 tl0_gen T_FIX_ALIGNMENT ! 0x106 tl0_gen T_INTEGER_OVERFLOW ! 0x107 tl0_gen T_SYSCALL ! 0x108 tl0_gen T_SYSCALL ! 0x109 tl0_fp_restore ! 0x10a tl0_reserved 5 ! 0x10b-0x10f tl0_gen T_TRAP_INSTRUCTION_16 ! 0x110 tl0_gen T_TRAP_INSTRUCTION_17 ! 0x111 tl0_gen T_TRAP_INSTRUCTION_18 ! 0x112 tl0_gen T_TRAP_INSTRUCTION_19 ! 0x113 tl0_gen T_TRAP_INSTRUCTION_20 ! 0x114 tl0_gen T_TRAP_INSTRUCTION_21 ! 0x115 tl0_gen T_TRAP_INSTRUCTION_22 ! 0x116 tl0_gen T_TRAP_INSTRUCTION_23 ! 0x117 tl0_gen T_TRAP_INSTRUCTION_24 ! 0x118 tl0_gen T_TRAP_INSTRUCTION_25 ! 0x119 tl0_gen T_TRAP_INSTRUCTION_26 ! 0x11a tl0_gen T_TRAP_INSTRUCTION_27 ! 0x11b tl0_gen T_TRAP_INSTRUCTION_28 ! 0x11c tl0_gen T_TRAP_INSTRUCTION_29 ! 0x11d tl0_gen T_TRAP_INSTRUCTION_30 ! 0x11e tl0_gen T_TRAP_INSTRUCTION_31 ! 0x11f tl0_reserved 32 ! 0x120-0x13f tl0_gen T_SYSCALL ! 0x140 tl0_syscall ! 0x141 tl0_gen T_SYSCALL ! 0x142 tl0_gen T_SYSCALL ! 0x143 tl0_reserved 188 ! 0x144-0x1ff tl1_base: tl1_reserved 8 ! 0x200-0x207 tl1_insn_excptn: tl1_insn_excptn ! 0x208 tl1_reserved 1 ! 0x209 tl1_insn_error: tl1_gen T_INSTRUCTION_ERROR ! 0x20a tl1_reserved 5 ! 0x20b-0x20f tl1_insn_illegal: tl1_gen T_ILLEGAL_INSTRUCTION ! 0x210 tl1_priv_opcode: tl1_gen T_PRIVILEGED_OPCODE ! 0x211 tl1_reserved 14 ! 0x212-0x21f tl1_fp_disabled: tl1_fp_disabled ! 0x220 tl1_fp_ieee: tl1_gen T_FP_EXCEPTION_IEEE_754 ! 0x221 tl1_fp_other: tl1_gen T_FP_EXCEPTION_OTHER ! 0x222 tl1_tag_ovflw: tl1_gen T_TAG_OVERFLOW ! 0x223 tl1_clean_window: clean_window ! 0x224 tl1_divide: tl1_gen T_DIVISION_BY_ZERO ! 0x228 tl1_reserved 7 ! 0x229-0x22f tl1_data_excptn: tl1_data_excptn ! 0x230 tl1_reserved 1 ! 0x231 tl1_data_error: tl1_gen T_DATA_ERROR ! 0x232 tl1_reserved 1 ! 0x233 tl1_align: tl1_align ! 0x234 tl1_align_lddf: tl1_gen T_RESERVED ! 0x235 tl1_align_stdf: tl1_gen T_RESERVED ! 0x236 tl1_priv_action: tl1_gen T_PRIVILEGED_ACTION ! 0x237 tl1_reserved 9 ! 0x238-0x240 tl1_intr_level: tl1_intr_level ! 0x241-0x24f tl1_reserved 16 ! 0x250-0x25f tl1_intr_vector: intr_vector ! 0x260 tl1_watch_phys: tl1_gen T_PA_WATCHPOINT ! 0x261 tl1_watch_virt: tl1_gen T_VA_WATCHPOINT ! 0x262 tl1_ecc: tl1_gen T_CORRECTED_ECC_ERROR ! 0x263 tl1_immu_miss: tl1_immu_miss ! 0x264 tl1_dmmu_miss: tl1_dmmu_miss ! 0x268 tl1_dmmu_prot: tl1_dmmu_prot ! 0x26c tl1_reserved 16 ! 0x270-0x27f tl1_spill_0_n: tl1_spill_0_n ! 0x280 tl1_spill_bad 1 ! 0x284 tl1_spill_2_n: tl1_spill_2_n ! 0x288 tl1_spill_3_n: tl1_spill_3_n ! 0x28c tl1_spill_bad 3 ! 0x290-0x29b tl1_spill_7_n: tl1_spill_7_n ! 0x29c tl1_spill_0_o: tl1_spill_0_o ! 0x2a0 tl1_spill_1_o: tl1_spill_1_o ! 0x2a4 tl1_spill_2_o: tl1_spill_2_o ! 0x2a8 tl1_spill_bad 5 ! 0x2ac-0x2bf tl1_fill_0_n: tl1_fill_0_n ! 0x2c0 tl1_fill_bad 1 ! 0x2c4 tl1_fill_2_n: tl1_fill_2_n ! 0x2c8 tl1_fill_3_n: tl1_fill_3_n ! 0x2cc tl1_fill_bad 3 ! 0x2d0-0x2db tl1_fill_7_n: tl1_fill_7_n ! 0x2dc tl1_fill_bad 8 ! 0x2e0-0x2ff tl1_reserved 1 ! 0x300 tl1_breakpoint: tl1_gen T_BREAKPOINT ! 0x301 tl1_gen T_RSTRWP_PHYS ! 0x302 tl1_gen T_RSTRWP_VIRT ! 0x303 tl1_reserved 252 ! 0x304-0x3ff .globl tl_trap_end tl_trap_end: nop /* * User trap entry point * * void tl0_utrap(u_long type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_long sfsr) * * This handles redirecting a trap back to usermode as a user trap. The user * program must have first registered a trap handler with the kernel using * sysarch(SPARC_UTRAP_INSTALL). The trap handler is passed enough state * for it to return to the trapping code directly, it will not return through * the kernel. The trap type is passed in %o0, all out registers must be * passed through to tl0_trap or to usermode untouched. Note that the * parameters passed in out registers may be used by the user trap handler. * Do not change the registers they are passed in or you will break the ABI. * * If the trap type allows user traps, setup state to execute the user trap * handler and bounce back to usermode, otherwise branch to tl0_trap. */ ENTRY(tl0_utrap) /* * Check if the trap type allows user traps. */ cmp %o0, UT_MAX bge,a,pt %xcc, tl0_trap nop /* * Load the user trap handler from the utrap table. */ ldx [PCPU(CURTHREAD)], %l0 ldx [%l0 + TD_PROC], %l0 ldx [%l0 + P_MD + MD_UTRAP], %l0 brz,pt %l0, tl0_trap sllx %o0, PTR_SHIFT, %l1 ldx [%l0 + %l1], %l0 brz,a,pt %l0, tl0_trap nop /* * If the save we did on entry to the kernel had to spill a window * to the pcb, pretend we took a spill trap instead. Any windows * that are in the pcb must be copied out or the fill handler will * not be able to find them, since the user trap handler returns * directly to the trapping code. Note that we only support precise * user traps, which implies that the condition that caused the trap * in the first place is still valid, so it will occur again when we * re-execute the trapping instruction. */ ldx [PCB_REG + PCB_NSAVED], %l1 brnz,a,pn %l1, tl0_trap mov T_SPILL, %o0 /* * Pass %fsr in %l4, %tstate in %l5, %tpc in %l6 and %tnpc in %l7. * The ABI specifies only %l6 and %l7, but we need to pass %fsr or * it may be clobbered by an interrupt before the user trap code * can read it, and we must pass %tstate in order to restore %ccr * and %asi. The %fsr must be stored to memory, so we use the * temporary stack for that. */ rd %fprs, %l1 or %l1, FPRS_FEF, %l2 wr %l2, 0, %fprs dec 8, ASP_REG stx %fsr, [ASP_REG] ldx [ASP_REG], %l4 inc 8, ASP_REG wr %l1, 0, %fprs rdpr %tstate, %l5 rdpr %tpc, %l6 rdpr %tnpc, %l7 /* * Setup %tnpc to return to. */ wrpr %l0, 0, %tnpc /* * Setup %wstate for return, clear WSTATE_TRANSITION. */ rdpr %wstate, %l1 and %l1, WSTATE_NORMAL_MASK, %l1 wrpr %l1, 0, %wstate /* * Setup %tstate for return, change the saved cwp to point to the * current window instead of the window at the time of the trap. */ andn %l5, TSTATE_CWP_MASK, %l1 rdpr %cwp, %l2 wrpr %l1, %l2, %tstate /* * Setup %sp. Userland processes will crash if this is not setup. */ sub %fp, CCFSZ, %sp /* * Execute the user trap handler. */ done END(tl0_utrap) /* * (Real) User trap entry point * * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfsr, * u_int sfsr) * * The following setup has been performed: * - the windows have been split and the active user window has been saved * (maybe just to the pcb) * - we are on alternate globals and interrupts are disabled * * We switch to the kernel stack, build a trapframe, switch to normal * globals, enable interrupts and call trap. * * NOTE: We must be very careful setting up the per-cpu pointer. We know that * it has been pre-set in alternate globals, so we read it from there and setup * the normal %g7 *before* enabling interrupts. This avoids any possibility * of cpu migration and using the wrong pcpup. */ ENTRY(tl0_trap) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif 1: and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stx %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_Y] stx %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stx %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] rd %gsr, %l6 stx %l6, [%sp + SPOFF + CCFSZ + TF_GSR] wr %g0, 0, %fprs mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] set tl0_ret - 8, %o7 jmpl %o2, %g0 add %sp, CCFSZ + SPOFF, %o0 END(tl0_trap) /* * void tl0_intr(u_int level, u_int mask) */ ENTRY(tl0_intr) /* * Force kernel store order. */ wrpr %g0, PSTATE_ALT, %pstate rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rd %y, %l3 rd %fprs, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] rdpr %pil, %g2 stx %g2, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %l2, [%g1 + KTR_PARM5] stx %i6, [%g1 + KTR_PARM6] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %clear_softint and %l5, WSTATE_NORMAL_MASK, %l5 sllx %l5, WSTATE_OTHER_SHIFT, %l5 wrpr %l5, WSTATE_KERNEL, %wstate rdpr %canrestore, %l6 wrpr %l6, 0, %otherwin wrpr %g0, 0, %canrestore sub PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_Y] stx %l4, [%sp + SPOFF + CCFSZ + TF_FPRS] stx %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE] wr %g0, FPRS_FEF, %fprs stx %fsr, [%sp + SPOFF + CCFSZ + TF_FSR] rd %gsr, %l6 stx %l6, [%sp + SPOFF + CCFSZ + TF_GSR] wr %g0, 0, %fprs mov %o0, %l3 mov T_INTERRUPT, %o1 stx %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] SET(intr_handlers, %l1, %l0) sllx %l3, IH_SHIFT, %l1 ldx [%l0 + %l1], %l1 KASSERT(%l1, "tl0_intr: ih null") call %l1 add %sp, CCFSZ + SPOFF, %o0 /* %l3 contains PIL */ SET(intrcnt, %l1, %l2) prefetcha [%l2] ASI_N, 1 SET(pil_countp, %l1, %l0) sllx %l3, 1, %l1 lduh [%l0 + %l1], %l0 sllx %l0, 3, %l0 add %l0, %l2, %l0 ldx [%l0], %l1 inc %l1 stx %l1, [%l0] call counter_intr_inc nop ba,a %xcc, tl0_ret nop END(tl0_intr) /* * Initiate return to usermode. * * Called with a trapframe on the stack. The window that was setup in * tl0_trap may have been used by "fast" trap handlers that pretend to be * leaf functions, so all ins and locals may have been clobbered since * then. * * This code is rather long and complicated. */ ENTRY(tl0_ret) /* * Check for pending asts atomically with returning. We must raise * the PIL before checking, and if no asts are found the PIL must * remain raised until the retry is executed, or we risk missing asts * caused by interrupts occurring after the test. If the PIL is * lowered, as it is when we call ast, the check must be re-executed. */ wrpr %g0, PIL_TICK, %pil ldx [PCPU(CURTHREAD)], %l0 lduw [%l0 + TD_FLAGS], %l1 set TDF_ASTPENDING | TDF_NEEDRESCHED, %l2 and %l1, %l2, %l1 brz,a,pt %l1, 1f nop /* * We have an AST. Re-enable interrupts and handle it, then restart * the return sequence. */ wrpr %g0, 0, %pil call ast add %sp, CCFSZ + SPOFF, %o0 ba,a %xcc, tl0_ret nop /* * Check for windows that were spilled to the pcb and need to be * copied out. This must be the last thing that is done before the * return to usermode. If there are still user windows in the cpu * and we call a nested function after this, which causes them to be * spilled to the pcb, they will not be copied out and the stack will * be inconsistent. */ 1: ldx [PCB_REG + PCB_NSAVED], %l1 brz,a,pt %l1, 2f nop wrpr %g0, 0, %pil mov T_SPILL, %o0 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] call trap add %sp, SPOFF + CCFSZ, %o0 ba,a %xcc, tl0_ret nop /* * Restore the out and most global registers from the trapframe. * The ins will become the outs when we restore below. */ 2: ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 /* * Load everything we need to restore below before disabling * interrupts. */ ldx [%sp + SPOFF + CCFSZ + TF_FPRS], %l0 ldx [%sp + SPOFF + CCFSZ + TF_GSR], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l3 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l4 ldx [%sp + SPOFF + CCFSZ + TF_Y], %l5 ldx [%sp + SPOFF + CCFSZ + TF_WSTATE], %l6 /* * Disable interrupts to restore the special globals. They are not * saved and restored for all kernel traps, so an interrupt at the * wrong time would clobber them. */ wrpr %g0, PSTATE_NORMAL, %pstate ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 /* * Switch to alternate globals. This frees up some registers we * can use after the restore changes our window. */ wrpr %g0, PSTATE_ALT, %pstate /* * Drop %pil to zero. It must have been zero at the time of the * trap, since we were in usermode, but it was raised above in * order to check for asts atomically. We have interrupts disabled * so any interrupts will not be serviced until we complete the * return to usermode. */ wrpr %g0, 0, %pil /* * Save %fprs in an alternate global so it can be restored after the * restore instruction below. If we restore it before the restore, * and the restore traps we may run for a while with floating point * enabled in the kernel, which we want to avoid. */ mov %l0, %g1 /* * Restore %fsr and %gsr. These need floating point enabled in %fprs, * so we set it temporarily and then clear it. */ wr %g0, FPRS_FEF, %fprs ldx [%sp + SPOFF + CCFSZ + TF_FSR], %fsr wr %l1, 0, %gsr wr %g0, 0, %fprs /* * Restore program counters. This could be done after the restore * but we're out of alternate globals to store them in... */ wrpr %l2, 0, %tnpc wrpr %l3, 0, %tpc /* * Save %tstate in an alternate global and clear the %cwp field. %cwp * will be affected by the restore below and we need to make sure it * points to the current window at that time, not the window that was * active at the time of the trap. */ andn %l4, TSTATE_CWP_MASK, %g2 /* * Restore %y. Could also be below if we had more alternate globals. */ wr %l5, 0, %y /* * Setup %wstate for return. We need to restore the user window state * which we saved in wstate.other when we trapped. We also need to * set the transition bit so the restore will be handled specially * if it traps, use the xor feature of wrpr to do that. */ srlx %l6, WSTATE_OTHER_SHIFT, %g3 wrpr %g3, WSTATE_TRANSITION, %wstate /* * Setup window management registers for return. If not all user * windows were spilled in the kernel %otherwin will be non-zero, * so we need to transfer it to %canrestore to correctly restore * those windows. Otherwise everything gets set to zero and the * restore below will fill a window directly from the user stack. */ rdpr %otherwin, %o0 wrpr %o0, 0, %canrestore wrpr %g0, 0, %otherwin wrpr %o0, 0, %cleanwin /* * Now do the restore. If this instruction causes a fill trap which * fails to fill a window from the user stack, we will resume at * tl0_ret_fill_end and call back into the kernel. */ restore tl0_ret_fill: /* * We made it. We're back in the window that was active at the time * of the trap, and ready to return to usermode. */ /* * Restore %frps. This was saved in an alternate global above. */ wr %g1, 0, %fprs /* * Fixup %tstate so the saved %cwp points to the current window and * restore it. */ rdpr %cwp, %g4 wrpr %g2, %g4, %tstate /* * Restore the user window state. The transition bit was set above * for special handling of the restore, this clears it. */ wrpr %g3, 0, %wstate #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tnpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif /* * Return to usermode. */ retry tl0_ret_fill_end: #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx" , %l0, %l1, %l2, 7, 8, 9) rdpr %pstate, %l1 stx %l1, [%l0 + KTR_PARM1] stx %l6, [%l0 + KTR_PARM2] stx %sp, [%l0 + KTR_PARM3] 9: #endif /* * The restore above caused a fill trap and the fill handler was * unable to fill a window from the user stack. The special fill * handler recognized this and punted, sending us here. We need * to carefully undo any state that was restored before the restore * was executed and call trap again. Trap will copyin a window * from the user stack which will fault in the page we need so the * restore above will succeed when we try again. If this fails * the process has trashed its stack, so we kill it. */ /* * Restore the kernel window state. This was saved in %l6 above, and * since the restore failed we're back in the same window. */ wrpr %l6, 0, %wstate /* * Restore the normal globals which have predefined values in the * kernel. We clobbered them above restoring the user's globals * so this is very important. * XXX PSTATE_ALT must already be set. */ wrpr %g0, PSTATE_ALT, %pstate mov PCB_REG, %o0 mov PCPU_REG, %o1 wrpr %g0, PSTATE_NORMAL, %pstate mov %o0, PCB_REG mov %o1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate /* * Simulate a fill trap and then start the whole return sequence over * again. This is special because it only copies in 1 window, not 2 * as we would for a normal failed fill. This may be the first time * the process has been run, so there may not be 2 windows worth of * stack to copyin. */ mov T_FILL_RET, %o0 stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] call trap add %sp, SPOFF + CCFSZ, %o0 ba,a %xcc, tl0_ret nop END(tl0_ret) /* * Kernel trap entry point * * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar, * u_int sfsr) * * This is easy because the stack is already setup and the windows don't need * to be split. We build a trapframe and call trap(), the same as above, but * the outs don't need to be saved. */ ENTRY(tl1_trap) rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %o0, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o3, [%sp + SPOFF + CCFSZ + TF_TAR] stx %o4, [%sp + SPOFF + CCFSZ + TF_SFAR] stx %o5, [%sp + SPOFF + CCFSZ + TF_SFSR] stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stx %l4, [%sp + SPOFF + CCFSZ + TF_Y] mov PCB_REG, %l0 mov PCPU_REG, %l1 wrpr %g0, PSTATE_NORMAL, %pstate stx %g6, [%sp + SPOFF + CCFSZ + TF_G6] stx %g7, [%sp + SPOFF + CCFSZ + TF_G7] mov %l0, PCB_REG mov %l1, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate stx %i0, [%sp + SPOFF + CCFSZ + TF_O0] stx %i1, [%sp + SPOFF + CCFSZ + TF_O1] stx %i2, [%sp + SPOFF + CCFSZ + TF_O2] stx %i3, [%sp + SPOFF + CCFSZ + TF_O3] stx %i4, [%sp + SPOFF + CCFSZ + TF_O4] stx %i5, [%sp + SPOFF + CCFSZ + TF_O5] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] set tl1_ret - 8, %o7 jmpl %o2, %g0 add %sp, CCFSZ + SPOFF, %o0 END(tl1_trap) ENTRY(tl1_ret) ldx [%sp + SPOFF + CCFSZ + TF_O0], %i0 ldx [%sp + SPOFF + CCFSZ + TF_O1], %i1 ldx [%sp + SPOFF + CCFSZ + TF_O2], %i2 ldx [%sp + SPOFF + CCFSZ + TF_O3], %i3 ldx [%sp + SPOFF + CCFSZ + TF_O4], %i4 ldx [%sp + SPOFF + CCFSZ + TF_O5], %i5 ldx [%sp + SPOFF + CCFSZ + TF_O6], %i6 ldx [%sp + SPOFF + CCFSZ + TF_O7], %i7 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 ldx [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0 ldx [%sp + SPOFF + CCFSZ + TF_TPC], %l1 ldx [%sp + SPOFF + CCFSZ + TF_TNPC], %l2 ldx [%sp + SPOFF + CCFSZ + TF_PIL], %l3 ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 set VM_MIN_PROM_ADDRESS, %l5 cmp %l1, %l5 bl,a,pt %xcc, 1f nop set VM_MAX_PROM_ADDRESS, %l5 cmp %l1, %l5 bg,a,pt %xcc, 1f nop wrpr %g0, PSTATE_NORMAL, %pstate ldx [%sp + SPOFF + CCFSZ + TF_G6], %g6 ldx [%sp + SPOFF + CCFSZ + TF_G7], %g7 1: wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 wrpr %l3, 0, %pil wr %l4, 0, %y restore wrpr %g0, 2, %tl rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "tl1_ret: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tstate, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry END(tl1_ret) /* * void tl1_intr(u_int level, u_int mask) */ ENTRY(tl1_intr) rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %pil, %l3 rd %y, %l4 rdpr %wstate, %l5 #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%p level=%#x pil=%#lx pc=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] stx %o0, [%g1 + KTR_PARM2] stx %l3, [%g1 + KTR_PARM3] stx %l1, [%g1 + KTR_PARM4] stx %i6, [%g1 + KTR_PARM5] 9: #endif wrpr %o0, 0, %pil wr %o1, 0, %clear_softint wrpr %g0, 1, %tl and %l5, WSTATE_OTHER_MASK, %l5 wrpr %l5, WSTATE_KERNEL, %wstate stx %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE] stx %l1, [%sp + SPOFF + CCFSZ + TF_TPC] stx %l2, [%sp + SPOFF + CCFSZ + TF_TNPC] stx %l3, [%sp + SPOFF + CCFSZ + TF_PIL] stx %l4, [%sp + SPOFF + CCFSZ + TF_Y] mov %o0, %l7 mov T_INTERRUPT | T_KERNEL, %o1 stx %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL] stx %o1, [%sp + SPOFF + CCFSZ + TF_TYPE] stx %i6, [%sp + SPOFF + CCFSZ + TF_O6] stx %i7, [%sp + SPOFF + CCFSZ + TF_O7] mov PCB_REG, %l4 mov PCPU_REG, %l5 wrpr %g0, PSTATE_NORMAL, %pstate stx %g1, [%sp + SPOFF + CCFSZ + TF_G1] stx %g2, [%sp + SPOFF + CCFSZ + TF_G2] stx %g3, [%sp + SPOFF + CCFSZ + TF_G3] stx %g4, [%sp + SPOFF + CCFSZ + TF_G4] stx %g5, [%sp + SPOFF + CCFSZ + TF_G5] mov %l4, PCB_REG mov %l5, PCPU_REG wrpr %g0, PSTATE_KERNEL, %pstate SET(intr_handlers, %l5, %l4) sllx %l7, IH_SHIFT, %l5 ldx [%l4 + %l5], %l5 KASSERT(%l5, "tl1_intr: ih null") call %l5 add %sp, CCFSZ + SPOFF, %o0 /* %l7 contains PIL */ SET(intrcnt, %l5, %l4) prefetcha [%l4] ASI_N, 1 SET(pil_countp, %l5, %l6) sllx %l7, 1, %l5 lduh [%l5 + %l6], %l5 sllx %l5, 3, %l5 add %l5, %l4, %l4 ldx [%l4], %l5 inc %l5 call counter_intr_inc stx %l5, [%l4] ldx [%sp + SPOFF + CCFSZ + TF_Y], %l4 ldx [%sp + SPOFF + CCFSZ + TF_G1], %g1 ldx [%sp + SPOFF + CCFSZ + TF_G2], %g2 ldx [%sp + SPOFF + CCFSZ + TF_G3], %g3 ldx [%sp + SPOFF + CCFSZ + TF_G4], %g4 ldx [%sp + SPOFF + CCFSZ + TF_G5], %g5 wrpr %g0, PSTATE_ALT, %pstate andn %l0, TSTATE_CWP_MASK, %g1 mov %l1, %g2 mov %l2, %g3 wrpr %l3, 0, %pil wr %l4, 0, %y restore wrpr %g0, 2, %tl rdpr %cwp, %g4 wrpr %g1, %g4, %tstate wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc #if KTR_COMPILE & KTR_INTR CATR(KTR_INTR, "tl1_intr: td=%#x pil=%#lx ts=%#lx pc=%#lx sp=%#lx" , %g2, %g3, %g4, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g3 stx %g3, [%g2 + KTR_PARM1] rdpr %pil, %g3 stx %g3, [%g2 + KTR_PARM2] rdpr %tstate, %g3 stx %g3, [%g2 + KTR_PARM3] rdpr %tpc, %g3 stx %g3, [%g2 + KTR_PARM4] stx %sp, [%g2 + KTR_PARM5] 9: #endif retry END(tl1_intr) .globl tl_text_end tl_text_end: nop /* * Freshly forked processes come here when switched to for the first time. * The arguments to fork_exit() have been setup in the locals, we must move * them to the outs. */ ENTRY(fork_trampoline) #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [PCPU(CURTHREAD)], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g2 + TD_PROC], %g2 add %g2, P_COMM, %g2 stx %g2, [%g1 + KTR_PARM2] rdpr %cwp, %g2 stx %g2, [%g1 + KTR_PARM3] 9: #endif mov %l0, %o0 mov %l1, %o1 call fork_exit mov %l2, %o2 ba,a %xcc, tl0_ret nop END(fork_trampoline) Index: head/sys/sparc64/sparc64/interrupt.S =================================================================== --- head/sys/sparc64/sparc64/interrupt.S (revision 331253) +++ head/sys/sparc64/sparc64/interrupt.S (revision 331254) @@ -1,231 +1,231 @@ /*- * Copyright (c) 2002 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include -#include "assym.s" +#include "assym.inc" /* * Handle a vectored interrupt. * * This is either a data bearing mondo vector interrupt, or a cross trap * request from another cpu. In either case the hardware supplies an * interrupt packet, in the form of 3 data words which are read from internal * registers. A data bearing mondo vector packet consists of an interrupt * number in the first data word, and zero in 2nd and 3rd. We use the * interrupt number to find the function, argument and priority from the * intr_vector table, allocate and fill in an intr_request from the per-cpu * free list, link it onto the per-cpu active list and finally post a softint * at the desired priority. Cross trap requests come in 2 forms, direct * and queued. Direct requests are distinguished by the first data word * being zero. The 2nd data word carries a function to call and the 3rd * an argument to pass. The function is jumped to directly. It executes * in nucleus context on interrupt globals and with all interrupts disabled, * therefore it must be fast, and the things that it can do are limited. * Queued cross trap requests are handled much like mondo vectors, except * that the function, argument and priority are contained in the interrupt * packet itself. They are distinguished by the upper 4 bits of the data * word being non-zero, which specifies the priority of the softint to * deliver. * * Register usage: * %g1 - pointer to intr_request * %g2 - pointer to intr_vector, temp once required data is loaded * %g3 - interrupt number for mondo vectors, unused otherwise * %g4 - function, from the interrupt packet for cross traps, or * loaded from the interrupt registers for mondo vecors * %g5 - argument, as above for %g4 * %g6 - softint priority */ ENTRY(intr_vector) /* * Load the interrupt packet from the hardware. */ wr %g0, ASI_SDB_INTR_R, %asi ldxa [%g0 + AA_SDB_INTR_D0] %asi, %g3 ldxa [%g0 + AA_SDB_INTR_D1] %asi, %g4 ldxa [%g0 + AA_SDB_INTR_D2] %asi, %g5 stxa %g0, [%g0] ASI_INTR_RECEIVE membar #Sync /* * If the first data word is zero this is a direct cross trap request. * The 2nd word points to code to execute and the 3rd is an argument * to pass. Jump to it. */ brnz,pt %g3, 1f /* * NB: Zeus CPUs set some undocumented bits in the first data word. */ and %g3, IV_MAX - 1, %g3 jmpl %g4, %g0 nop /* NOTREACHED */ /* * If the high 4 bits of the 1st data word are non-zero, this is a * queued cross trap request to be delivered as a softint. The high * 4 bits of the 1st data word specify a priority, and the 2nd and * 3rd a function and argument. */ 1: srlx %g3, 60, %g6 brnz,a,pn %g6, 2f clr %g3 /* * Find the function, argument and desired priority from the * intr_vector table. */ SET(intr_vectors, %g4, %g2) sllx %g3, IV_SHIFT, %g4 add %g2, %g4, %g2 ldx [%g2 + IV_FUNC], %g4 ldx [%g2 + IV_ARG], %g5 lduw [%g2 + IV_PRI], %g6 /* * Get an intr_request from the free list. There should always be one * unless we are getting an interrupt storm from stray interrupts, in * which case the we will deference a NULL pointer and panic. */ 2: ldx [PCPU(IRFREE)], %g1 ldx [%g1 + IR_NEXT], %g2 stx %g2, [PCPU(IRFREE)] /* * Store the vector number, function, argument and priority. */ stw %g3, [%g1 + IR_VEC] stx %g4, [%g1 + IR_FUNC] stx %g5, [%g1 + IR_ARG] stw %g6, [%g1 + IR_PRI] /* * Link it onto the end of the active list. */ stx %g0, [%g1 + IR_NEXT] ldx [PCPU(IRTAIL)], %g4 stx %g1, [%g4] add %g1, IR_NEXT, %g1 stx %g1, [PCPU(IRTAIL)] /* * Trigger a softint at the level indicated by the priority. */ mov 1, %g1 sllx %g1, %g6, %g1 wr %g1, 0, %set_softint /* * Done, retry the instruction. */ retry END(intr_vector) ENTRY(intr_vector_stray) /* * SPARC64-VI trigger stray vector interrupts in order to indicate * uncorrectable errors in interrupt packets, which still need to be * acknowledged though. * US-IV occasionally trigger stray vector interrupts for reasons * unknown accompanied by a state in which they even fault on locked * TLB entries so we can't even log these here. Just retrying the * instruction in that case gets the CPU back on track. */ rdpr %ver, %g1 srlx %g1, VER_IMPL_SHIFT, %g1 sll %g1, VER_IMPL_SIZE, %g1 srl %g1, VER_IMPL_SIZE, %g1 cmp %g1, CPU_IMPL_SPARC64VI bne,a,pn %icc, 1f nop stxa %g0, [%g0] ASI_INTR_RECEIVE membar #Sync 1: retry END(intr_vector_stray) ENTRY(intr_fast) save %sp, -CCFSZ, %sp /* * Disable interrupts while we fiddle with the interrupt request lists * as interrupts at levels higher than what got us here aren't blocked. */ 1: wrpr %g0, PSTATE_NORMAL, %pstate ldx [PCPU(IRHEAD)], %l0 brnz,a,pt %l0, 2f nop wrpr %g0, PSTATE_KERNEL, %pstate ret restore 2: ldx [%l0 + IR_NEXT], %l1 brnz,pt %l1, 3f stx %l1, [PCPU(IRHEAD)] PCPU_ADDR(IRHEAD, %l1) stx %l1, [PCPU(IRTAIL)] 3: ldx [%l0 + IR_FUNC], %o0 ldx [%l0 + IR_ARG], %o1 lduw [%l0 + IR_VEC], %l2 ldx [PCPU(IRFREE)], %l1 stx %l1, [%l0 + IR_NEXT] stx %l0, [PCPU(IRFREE)] wrpr %g0, PSTATE_KERNEL, %pstate KASSERT(%o0, "intr_fast: ir_func null") call %o0 mov %o1, %o0 /* intrcnt[intr_countp[%l2]]++ */ SET(intrcnt, %l7, %l3) /* %l3 = intrcnt */ prefetcha [%l3] ASI_N, 1 SET(intr_countp, %l7, %l4) /* %l4 = intr_countp */ sllx %l2, 1, %l2 /* %l2 = vec << 1 */ lduh [%l4 + %l2], %l4 /* %l4 = intr_countp[%l2] */ sllx %l4, 3, %l4 /* %l4 = intr_countp[%l2] << 3 */ add %l4, %l3, %l4 /* %l4 = intrcnt[intr_countp[%l2]] */ ldx [%l4], %l2 inc %l2 stx %l2, [%l4] ba,a %xcc, 1b nop END(intr_fast) Index: head/sys/sparc64/sparc64/locore.S =================================================================== --- head/sys/sparc64/sparc64/locore.S (revision 331253) +++ head/sys/sparc64/sparc64/locore.S (revision 331254) @@ -1,148 +1,148 @@ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include -#include "assym.s" +#include "assym.inc" .register %g2,#ignore .globl kernbase .set kernbase, KERNBASE /* * void _start(caddr_t metadata, u_long o1, u_long o2, u_long o3, * u_long ofw_vec) */ ENTRY(btext) ENTRY(_start) /* * Initialize misc. state to known values: interrupts disabled, normal * globals, windows flushed (cr = 0, cs = nwindows - 1), PIL_TICK and * floating point disabled. * Note that some firmware versions don't implement a clean window * trap handler so we unfortunately can't clear the windows by setting * %cleanwin to zero here. */ wrpr %g0, PSTATE_NORMAL, %pstate flushw wrpr %g0, PIL_TICK, %pil wr %g0, 0, %fprs /* * Get onto our per-CPU panic stack, which precedes the struct pcpu in * the per-CPU page. */ SET(pcpu0 + (PCPU_PAGES * PAGE_SIZE) - PC_SIZEOF, %l1, %l0) sub %l0, SPOFF + CCFSZ, %sp /* * Do initial bootstrap to setup pmap and thread0. */ call sparc64_init nop /* * Get onto thread0's kstack. */ sub PCB_REG, SPOFF + CCFSZ, %sp /* * And away we go. This doesn't return. */ call mi_startup nop sir ! NOTREACHED END(_start) /* * void cpu_setregs(struct pcpu *pc) */ ENTRY(cpu_setregs) ldx [%o0 + PC_CURPCB], %o1 /* * Ensure we are on normal globals. */ wrpr %g0, PSTATE_NORMAL, %pstate /* * Normal %g6 points to the current thread's PCB, and %g7 points to * the per-CPU data structure. */ mov %o1, PCB_REG mov %o0, PCPU_REG /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Alternate %g5 points to a per-CPU panic stack, %g6 points to the * current thread's PCB, and %g7 points to the per-CPU data structure. */ mov %o0, ASP_REG mov %o1, PCB_REG mov %o0, PCPU_REG /* * Switch to interrupt globals. */ wrpr %g0, PSTATE_INTR, %pstate /* * Interrupt %g7 points to the per-CPU data structure. */ mov %o0, PCPU_REG /* * Switch to normal globals again. */ wrpr %g0, PSTATE_NORMAL, %pstate /* * Force trap level 1 and take over the trap table. */ SET(tl0_base, %o2, %o1) SET(tba_taken_over, %o3, %o2) mov 1, %o3 wrpr %g0, WSTATE_KERNEL, %wstate wrpr %g0, 1, %tl wrpr %o1, 0, %tba stw %o3, [%o2] retl nop END(cpu_setregs) Index: head/sys/sparc64/sparc64/mp_exception.S =================================================================== --- head/sys/sparc64/sparc64/mp_exception.S (revision 331253) +++ head/sys/sparc64/sparc64/mp_exception.S (revision 331254) @@ -1,309 +1,309 @@ /*- * Copyright (c) 2002 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include -#include "assym.s" +#include "assym.inc" .register %g2, #ignore .register %g3, #ignore #define IPI_DONE(r1, r2, r3, r4, r5) \ lduw [PCPU(CPUID)], r2 ; \ mov _NCPUBITS, r3 ; \ udivx r2, r3, r4 ; \ srl r4, 0, r5 ; \ sllx r5, PTR_SHIFT, r5 ; \ add r1, r5, r1 ; \ smul r4, r3, r3 ; \ sub r2, r3, r3 ; \ mov 1, r4 ; \ sllx r4, r3, r4 ; \ ATOMIC_CLEAR_LONG(r1, r2, r3, r4) /* * Invalidate a physical page in the data cache. For UltraSPARC I and II. */ ENTRY(tl_ipi_spitfire_dcache_page_inval) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_spitfire_dcache_page_inval: pa=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ICA_PA], %g2 stx %g2, [%g1 + KTR_PARM1] 9: #endif ldx [%g5 + ICA_PA], %g6 srlx %g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6 lduw [PCPU(CACHE) + DC_SIZE], %g3 lduw [PCPU(CACHE) + DC_LINESIZE], %g4 sub %g3, %g4, %g2 1: ldxa [%g2] ASI_DCACHE_TAG, %g1 srlx %g1, DC_VALID_SHIFT, %g3 andcc %g3, DC_VALID_MASK, %g0 bz,pt %xcc, 2f set DC_TAG_MASK, %g3 sllx %g3, DC_TAG_SHIFT, %g3 and %g1, %g3, %g1 cmp %g1, %g6 bne,a,pt %xcc, 2f nop stxa %g1, [%g2] ASI_DCACHE_TAG membar #Sync 2: brgz,pt %g2, 1b sub %g2, %g4, %g2 IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_spitfire_dcache_page_inval) /* * Invalidate a physical page in the instruction cache. For UltraSPARC I and * II. */ ENTRY(tl_ipi_spitfire_icache_page_inval) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_spitfire_icache_page_inval: pa=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ICA_PA], %g2 stx %g2, [%g1 + KTR_PARM1] 9: #endif ldx [%g5 + ICA_PA], %g6 srlx %g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6 lduw [PCPU(CACHE) + IC_SIZE], %g3 lduw [PCPU(CACHE) + IC_LINESIZE], %g4 sub %g3, %g4, %g2 1: ldda [%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */ srlx %g1, IC_VALID_SHIFT, %g3 andcc %g3, IC_VALID_MASK, %g0 bz,pt %xcc, 2f set IC_TAG_MASK, %g3 sllx %g3, IC_TAG_SHIFT, %g3 and %g1, %g3, %g1 cmp %g1, %g6 bne,a,pt %xcc, 2f nop stxa %g1, [%g2] ASI_ICACHE_TAG membar #Sync 2: brgz,pt %g2, 1b sub %g2, %g4, %g2 IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_spitfire_icache_page_inval) /* * Invalidate a physical page in the data cache. For UltraSPARC III. */ ENTRY(tl_ipi_cheetah_dcache_page_inval) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_cheetah_dcache_page_inval: pa=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ICA_PA], %g2 stx %g2, [%g1 + KTR_PARM1] 9: #endif ldx [%g5 + ICA_PA], %g1 set PAGE_SIZE, %g2 add %g1, %g2, %g3 lduw [PCPU(CACHE) + DC_LINESIZE], %g2 1: stxa %g0, [%g1] ASI_DCACHE_INVALIDATE membar #Sync add %g1, %g2, %g1 cmp %g1, %g3 blt,a,pt %xcc, 1b nop IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_cheetah_dcache_page_inval) /* * Trigger a softint at the desired level. */ ENTRY(tl_ipi_level) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx" , %g1, %g2, %g3, 7, 8, 9) lduw [PCPU(CPUID)], %g2 stx %g2, [%g1 + KTR_PARM1] lduw [PCPU(MID)], %g2 stx %g2, [%g1 + KTR_PARM2] stx %g4, [%g1 + KTR_PARM3] stx %g5, [%g1 + KTR_PARM4] 9: #endif mov 1, %g1 sllx %g1, %g5, %g1 wr %g1, 0, %set_softint retry END(tl_ipi_level) /* * Demap a page from the dtlb and/or itlb. */ ENTRY(tl_ipi_tlb_page_demap) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ITA_PMAP], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g5 + ITA_VA], %g2 stx %g2, [%g1 + KTR_PARM2] 9: #endif ldx [%g5 + ITA_PMAP], %g1 SET(kernel_pmap_store, %g3, %g2) mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 cmp %g1, %g2 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 ldx [%g5 + ITA_VA], %g2 or %g2, %g3, %g2 sethi %hi(KERNBASE), %g3 stxa %g0, [%g2] ASI_DMMU_DEMAP stxa %g0, [%g2] ASI_IMMU_DEMAP flush %g3 IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tlb_page_demap) /* * Demap a range of pages from the dtlb and itlb. */ ENTRY(tl_ipi_tlb_range_demap) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ITA_PMAP], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g5 + ITA_START], %g2 stx %g2, [%g1 + KTR_PARM2] ldx [%g5 + ITA_END], %g2 stx %g2, [%g1 + KTR_PARM3] 9: #endif ldx [%g5 + ITA_PMAP], %g1 SET(kernel_pmap_store, %g3, %g2) mov TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3 cmp %g1, %g2 movne %xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3 ldx [%g5 + ITA_START], %g1 ldx [%g5 + ITA_END], %g2 sethi %hi(KERNBASE), %g6 1: or %g1, %g3, %g4 stxa %g0, [%g4] ASI_DMMU_DEMAP stxa %g0, [%g4] ASI_IMMU_DEMAP flush %g6 set PAGE_SIZE, %g6 add %g1, %g6, %g1 cmp %g1, %g2 blt,a,pt %xcc, 1b sethi %hi(KERNBASE), %g6 IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tlb_range_demap) /* * Demap the primary context from the dtlb and itlb. */ ENTRY(tl_ipi_tlb_context_demap) #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "tl_ipi_tlb_context_demap: pm=%p va=%#lx" , %g1, %g2, %g3, 7, 8, 9) ldx [%g5 + ITA_PMAP], %g2 stx %g2, [%g1 + KTR_PARM1] ldx [%g5 + ITA_VA], %g2 stx %g2, [%g1 + KTR_PARM2] 9: #endif mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1 sethi %hi(KERNBASE), %g3 stxa %g0, [%g1] ASI_DMMU_DEMAP stxa %g0, [%g1] ASI_IMMU_DEMAP flush %g3 IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tlb_context_demap) /* * Read %stick. */ ENTRY(tl_ipi_stick_rd) ldx [%g5 + IRA_VAL], %g1 rd %asr24, %g2 stx %g2, [%g1] IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_stick_rd) /* * Read %tick. */ ENTRY(tl_ipi_tick_rd) ldx [%g5 + IRA_VAL], %g1 rd %tick, %g2 stx %g2, [%g1] IPI_DONE(%g5, %g1, %g2, %g3, %g4) retry END(tl_ipi_tick_rd) Index: head/sys/sparc64/sparc64/mp_locore.S =================================================================== --- head/sys/sparc64/sparc64/mp_locore.S (revision 331253) +++ head/sys/sparc64/sparc64/mp_locore.S (revision 331254) @@ -1,296 +1,296 @@ /*- * Copyright (c) 2002 Jake Burkholder. * Copyright (c) 2008 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" .register %g2, #ignore .register %g3, #ignore .text _ALIGN_TEXT /* * Initialize misc. state to known values: interrupts disabled, normal * globals, windows flushed (cr = 0, cs = nwindows - 1), PIL_TICK and * floating point disabled. * Note that some firmware versions don't implement a clean window * trap handler so we unfortunately can't clear the windows by setting * %cleanwin to zero here. */ 1: wrpr %g0, PSTATE_NORMAL, %pstate wrpr %g0, PIL_TICK, %pil wr %g0, 0, %fprs rdpr %ver, %l7 srlx %l7, VER_IMPL_SHIFT, %l7 sll %l7, VER_IMPL_SIZE, %l7 srl %l7, VER_IMPL_SIZE, %l7 cmp %l7, CPU_IMPL_ULTRASPARCIIIp bne %icc, 3f nop /* * Relocate the locked entry in it16 slot 0 (if existent) * as part of working around Cheetah+ erratum 34. */ setx TD_V | TD_L, %l1, %l0 /* * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ ldxa [%g0] ASI_ITLB_DATA_ACCESS_REG, %g0 ldxa [%g0] ASI_ITLB_DATA_ACCESS_REG, %l6 and %l6, %l0, %l1 cmp %l0, %l1 bne %xcc, 3f nop /* Flush the mapping of slot 0. */ ldxa [%g0] ASI_ITLB_TAG_READ_REG, %l5 srlx %l5, TAR_VPN_SHIFT, %l0 sllx %l0, TAR_VPN_SHIFT, %l0 or %l0, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %l0 stxa %g0, [%l0] ASI_IMMU_DEMAP /* The USIII-family ignores the address. */ flush %g0 /* * Search a replacement slot != 0 and enter the data and tag * that formerly were in slot 0. */ mov (1 << TLB_DAR_SLOT_SHIFT), %l4 setx TD_V, %l1, %l0 /* * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ 2: ldxa [%l4] ASI_ITLB_DATA_ACCESS_REG, %g0 ldxa [%l4] ASI_ITLB_DATA_ACCESS_REG, %l1 and %l1, %l0, %l1 cmp %l0, %l1 be,a %xcc, 2b add %l4, (1 << TLB_DAR_SLOT_SHIFT), %l4 wr %g0, ASI_IMMU, %asi stxa %l5, [%g0 + AA_IMMU_TAR] %asi stxa %l6, [%l4] ASI_ITLB_DATA_ACCESS_REG /* The USIII-family ignores the address. */ flush %g0 3: rd %pc, %l6 ldx [%l6 + (9f-3b)], %l1 add %l6, (11f-3b), %l2 clr %l3 4: cmp %l3, %l1 be %xcc, 8f nop ldx [%l2 + TTE_VPN], %l4 ldx [%l2 + TTE_DATA], %l5 srlx %l4, TV_SIZE_BITS, %l4 sllx %l4, PAGE_SHIFT_4M, %l4 wr %g0, ASI_DMMU, %asi stxa %l4, [%g0 + AA_DMMU_TAR] %asi stxa %l5, [%g0] ASI_DTLB_DATA_IN_REG membar #Sync cmp %l7, CPU_IMPL_ULTRASPARCIIIp bne %icc, 6f wr %g0, ASI_IMMU, %asi /* * Search an unused slot != 0 and explicitly enter the data * and tag there in order to avoid Cheetah+ erratum 34. */ mov (1 << TLB_DAR_SLOT_SHIFT), %l0 setx TD_V, %o1, %o0 /* * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ 5: ldxa [%l0] ASI_ITLB_DATA_ACCESS_REG, %g0 ldxa [%l0] ASI_ITLB_DATA_ACCESS_REG, %o1 and %o1, %o0, %o1 cmp %o0, %o1 be,a %xcc, 5b add %l0, (1 << TLB_DAR_SLOT_SHIFT), %l0 sethi %hi(KERNBASE), %o0 stxa %l4, [%g0 + AA_IMMU_TAR] %asi stxa %l5, [%l0] ASI_ITLB_DATA_ACCESS_REG flush %o0 ba %xcc, 7f nop 6: sethi %hi(KERNBASE), %l0 stxa %l4, [%g0 + AA_IMMU_TAR] %asi stxa %l5, [%g0] ASI_ITLB_DATA_IN_REG flush %l0 7: add %l2, 1 << TTE_SHIFT, %l2 add %l3, 1, %l3 ba %xcc, 4b nop 8: ldx [%l6 + (10f-3b)], %l1 jmpl %l1, %g0 nop _ALIGN_DATA 9: .xword 0x0 10: .xword 0x0 11: DATA(mp_tramp_code) .xword 1b DATA(mp_tramp_code_len) .xword 11b-1b DATA(mp_tramp_tlb_slots) .xword 9b-1b DATA(mp_tramp_func) .xword 10b-1b /* * void mp_startup(void) */ ENTRY(mp_startup) SET(cpu_start_args, %l1, %l0) mov CPU_TICKSYNC, %l1 membar #StoreLoad stw %l1, [%l0 + CSA_STATE] 1: ldx [%l0 + CSA_TICK], %l1 brz %l1, 1b nop wrpr %l1, 0, %tick rdpr %ver, %l1 stx %l1, [%l0 + CSA_VER] srlx %l1, VER_IMPL_SHIFT, %l1 sll %l1, VER_IMPL_SIZE, %l1 srl %l1, VER_IMPL_SIZE, %l1 cmp %l1, CPU_IMPL_SPARC64V bl %icc, 4f nop cmp %l1, CPU_IMPL_ULTRASPARCI bl %icc, 2f nop cmp %l1, CPU_IMPL_ULTRASPARCIII bl %icc, 4f nop 2: mov CPU_STICKSYNC, %l2 membar #StoreLoad stw %l2, [%l0 + CSA_STATE] 3: ldx [%l0 + CSA_STICK], %l2 brz %l2, 3b nop wr %l2, 0, %asr24 4: call cpu_get_mid mov %l1, %o0 /* * Inform the boot processor we have inited. */ mov CPU_INIT, %l1 membar #LoadStore stw %l1, [%l0 + CSA_STATE] /* * Wait till its our turn to bootstrap. */ 5: lduw [%l0 + CSA_MID], %l1 cmp %l1, %o0 bne %xcc, 5b nop add %l0, CSA_TTES, %l1 clr %l2 /* * Map the per-CPU pages. */ 6: sllx %l2, TTE_SHIFT, %l3 add %l1, %l3, %l3 ldx [%l3 + TTE_VPN], %l4 ldx [%l3 + TTE_DATA], %l5 wr %g0, ASI_DMMU, %asi srlx %l4, TV_SIZE_BITS, %l4 sllx %l4, PAGE_SHIFT_8K, %l4 stxa %l4, [%g0 + AA_DMMU_TAR] %asi stxa %l5, [%g0] ASI_DTLB_DATA_IN_REG membar #Sync add %l2, 1, %l2 cmp %l2, PCPU_PAGES bne %xcc, 6b nop /* * Get onto our per-CPU panic stack, which precedes the struct pcpu * in the per-CPU page. */ ldx [%l0 + CSA_PCPU], %l1 set PCPU_PAGES * PAGE_SIZE - PC_SIZEOF, %l2 add %l1, %l2, %l1 sub %l1, SPOFF + CCFSZ, %sp /* Initialize global registers. */ call cpu_setregs mov %l1, %o0 #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, "mp_startup: bootstrap cpuid=%d mid=%d pcpu=%#lx data=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) lduw [PCPU(CPUID)], %g2 stx %g2, [%g1 + KTR_PARM1] lduw [PCPU(MID)], %g2 stx %g2, [%g1 + KTR_PARM2] stx %l1, [%g1 + KTR_PARM3] stx %sp, [%g1 + KTR_PARM5] 9: #endif /* * And away we go. This doesn't return. */ call cpu_mp_bootstrap mov %l1, %o0 sir ! NOTREACHED END(mp_startup) Index: head/sys/sparc64/sparc64/support.S =================================================================== --- head/sys/sparc64/sparc64/support.S (revision 331253) +++ head/sys/sparc64/sparc64/support.S (revision 331254) @@ -1,962 +1,962 @@ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" .register %g2, #ignore .register %g3, #ignore .register %g6, #ignore /* * Common code for copy routines. * * We use large macros to generate functions for each of the copy routines. * This allows the load and store instructions to be generated for the right * operation, asi or not. It is possible to write an asi independent function * but this would require 2 expensive wrs in the main loop to switch %asi. * It would also screw up profiling (if we ever get it), but may save some I$. * We assume that either one of dasi and sasi is empty, or that they are both * the same (empty or non-empty). It is up to the caller to set %asi. */ /* * ASI independent implementation of copystr(9). * Used to implement copyinstr() and copystr(). * * Return value is in %g1. */ #define _COPYSTR(src, dst, len, done, sa, sasi, da, dasi) \ brz len, 4f ; \ mov src, %g2 ; \ 1: deccc 1, len ; \ bl,a,pn %xcc, 3f ; \ nop ; \ LD(ub, sa) [src] sasi, %g1 ; \ ST(b, da) %g1, [dst] dasi ; \ brz,pn %g1, 3f ; \ inc src ; \ ba %xcc, 1b ; \ inc dst ; \ 2: mov ENAMETOOLONG, %g1 ; \ 3: sub src, %g2, %g2 ; \ brnz,a done, 4f ; \ stx %g2, [done] ; \ 4: /* * ASI independent implementation of memset(3). * Used to implement bzero(), memset() and aszero(). * * If the pattern is non-zero, duplicate it to fill 64 bits. * Store bytes until dst is 8-byte aligned, then store 8 bytes. * It has yet to be determined how much unrolling is beneficial. * Could also read and compare before writing to minimize snoop traffic. * * XXX bzero() should be implemented as * #define bzero(dst, len) (void)memset((dst), 0, (len)) * if at all. */ #define _MEMSET(dst, pat, len, da, dasi) \ brlez,pn len, 5f ; \ and pat, 0xff, pat ; \ brz,pt pat, 1f ; \ sllx pat, 8, %g1 ; \ or pat, %g1, pat ; \ sllx pat, 16, %g1 ; \ or pat, %g1, pat ; \ sllx pat, 32, %g1 ; \ or pat, %g1, pat ; \ .align 16 ; \ 1: deccc 1, len ; \ bl,pn %xcc, 5f ; \ btst 7, dst ; \ bz,a,pt %xcc, 2f ; \ inc 1, len ; \ ST(b, da) pat, [dst] dasi ; \ ba %xcc, 1b ; \ inc dst ; \ .align 16 ; \ 2: deccc 32, len ; \ bl,a,pn %xcc, 3f ; \ inc 32, len ; \ ST(x, da) pat, [dst] dasi ; \ ST(x, da) pat, [dst + 8] dasi ; \ ST(x, da) pat, [dst + 16] dasi ; \ ST(x, da) pat, [dst + 24] dasi ; \ ba %xcc, 2b ; \ inc 32, dst ; \ .align 16 ; \ 3: deccc 8, len ; \ bl,a,pn %xcc, 4f ; \ inc 8, len ; \ ST(x, da) pat, [dst] dasi ; \ ba %xcc, 3b ; \ inc 8, dst ; \ .align 16 ; \ 4: deccc 1, len ; \ bl,a,pn %xcc, 5f ; \ nop ; \ ST(b, da) pat, [dst] dasi ; \ ba %xcc, 4b ; \ inc 1, dst ; \ 5: /* * ASI independent implementation of memcpy(3). * Used to implement bcopy(), copyin(), copyout(), memcpy(), ascopy(), * ascopyfrom() and ascopyto(). * * Transfer bytes until dst is 8-byte aligned. If src is then also 8 byte * aligned, transfer 8 bytes, otherwise finish with bytes. The unaligned * case could be optimized, but it is expected that this is the uncommon * case and of questionable value. The code to do so is also rather large * and ugly. It has yet to be determined how much unrolling is beneficial. * * XXX bcopy() must also check for overlap. This is stupid. * XXX bcopy() should be implemented as * #define bcopy(src, dst, len) (void)memcpy((dst), (src), (len)) * if at all. */ #define _MEMCPY(dst, src, len, da, dasi, sa, sasi) \ 1: deccc 1, len ; \ bl,pn %xcc, 6f ; \ btst 7, dst ; \ bz,a,pt %xcc, 2f ; \ inc 1, len ; \ LD(ub, sa) [src] sasi, %g1 ; \ ST(b, da) %g1, [dst] dasi ; \ inc 1, src ; \ ba %xcc, 1b ; \ inc 1, dst ; \ .align 16 ; \ 2: btst 7, src ; \ bz,a,pt %xcc, 3f ; \ nop ; \ ba,a %xcc, 5f ; \ .align 16 ; \ 3: deccc 32, len ; \ bl,a,pn %xcc, 4f ; \ inc 32, len ; \ LD(x, sa) [src] sasi, %g1 ; \ LD(x, sa) [src + 8] sasi, %g2 ; \ LD(x, sa) [src + 16] sasi, %g3 ; \ LD(x, sa) [src + 24] sasi, %g4 ; \ ST(x, da) %g1, [dst] dasi ; \ ST(x, da) %g2, [dst + 8] dasi ; \ ST(x, da) %g3, [dst + 16] dasi ; \ ST(x, da) %g4, [dst + 24] dasi ; \ inc 32, src ; \ ba %xcc, 3b ; \ inc 32, dst ; \ .align 16 ; \ 4: deccc 8, len ; \ bl,a,pn %xcc, 5f ; \ inc 8, len ; \ LD(x, sa) [src] sasi, %g1 ; \ ST(x, da) %g1, [dst] dasi ; \ inc 8, src ; \ ba %xcc, 4b ; \ inc 8, dst ; \ .align 16 ; \ 5: deccc 1, len ; \ bl,a,pn %xcc, 6f ; \ nop ; \ LD(ub, sa) [src] sasi, %g1 ; \ ST(b, da) %g1, [dst] dasi ; \ inc src ; \ ba %xcc, 5b ; \ inc dst ; \ 6: /* * void ascopy(u_long asi, vm_offset_t src, vm_offset_t dst, size_t len) */ ENTRY(ascopy) wr %o0, 0, %asi _MEMCPY(%o2, %o1, %o3, a, %asi, a, %asi) retl nop END(ascopy) /* * void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len) */ ENTRY(ascopyfrom) wr %o0, 0, %asi _MEMCPY(%o2, %o1, %o3, EMPTY, EMPTY, a, %asi) retl nop END(ascopyfrom) /* * void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len) */ ENTRY(ascopyto) wr %o1, 0, %asi _MEMCPY(%o2, %o0, %o3, a, %asi, EMPTY, EMPTY) retl nop END(ascopyto) /* * void aszero(u_long asi, vm_offset_t pa, size_t len) */ ENTRY(aszero) wr %o0, 0, %asi _MEMSET(%o1, %g0, %o2, a, %asi) retl nop END(aszero) /* * int bcmp(const void *b1, const void *b2, size_t len) */ ENTRY(bcmp) brz,pn %o2, 2f clr %o3 1: ldub [%o0 + %o3], %o4 ldub [%o1 + %o3], %o5 cmp %o4, %o5 bne,pn %xcc, 2f inc %o3 deccc %o2 bne,pt %xcc, 1b nop 2: retl mov %o2, %o0 END(bcmp) /* * void bcopy(const void *src, void *dst, size_t len) */ ENTRY(bcopy) /* * Check for overlap, and copy backwards if so. */ sub %o1, %o0, %g1 cmp %g1, %o2 bgeu,a,pt %xcc, 3f nop /* * Copy backwards. */ add %o0, %o2, %o0 add %o1, %o2, %o1 1: deccc 1, %o2 bl,a,pn %xcc, 2f nop dec 1, %o0 ldub [%o0], %g1 dec 1, %o1 ba %xcc, 1b stb %g1, [%o1] 2: retl nop /* * Do the fast version. */ 3: _MEMCPY(%o1, %o0, %o2, EMPTY, EMPTY, EMPTY, EMPTY) retl nop END(bcopy) /* * void bzero(void *b, size_t len) */ ENTRY(bzero) _MEMSET(%o0, %g0, %o1, EMPTY, EMPTY) retl nop END(bzero) /* * int copystr(const void *src, void *dst, size_t len, size_t *done) */ ENTRY(copystr) _COPYSTR(%o0, %o1, %o2, %o3, EMPTY, EMPTY, EMPTY, EMPTY) retl mov %g1, %o0 END(copystr) /* * void *memcpy(void *dst, const void *src, size_t len) */ ENTRY(memcpy) mov %o0, %o3 _MEMCPY(%o3, %o1, %o2, EMPTY, EMPTY, EMPTY, EMPTY) retl nop END(memcpy) /* * void *memset(void *b, int c, size_t len) */ ENTRY(memset) mov %o0, %o3 _MEMSET(%o3, %o1, %o2, EMPTY, EMPTY) retl nop END(memset) .globl copy_nofault_begin copy_nofault_begin: nop /* * int copyin(const void *uaddr, void *kaddr, size_t len) */ ENTRY(copyin) wr %g0, ASI_AIUP, %asi _MEMCPY(%o1, %o0, %o2, EMPTY, EMPTY, a, %asi) retl clr %o0 END(copyin) /* * int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) */ ENTRY(copyinstr) wr %g0, ASI_AIUP, %asi _COPYSTR(%o0, %o1, %o2, %o3, a, %asi, EMPTY, EMPTY) retl mov %g1, %o0 END(copyinstr) /* * int copyout(const void *kaddr, void *uaddr, size_t len) */ ENTRY(copyout) wr %g0, ASI_AIUP, %asi _MEMCPY(%o1, %o0, %o2, a, %asi, EMPTY, EMPTY) retl clr %o0 END(copyout) .globl copy_nofault_end copy_nofault_end: nop ENTRY(copy_fault) retl mov EFAULT, %o0 END(copy_fault) .globl fs_nofault_begin fs_nofault_begin: nop /* * Chatty aliases for fetch, store functions. */ .globl fubyte, fusword, fuword, subyte, susword, suword .set fubyte, fuword8 .set fusword, fuword16 .set fuword, fuword64 .set subyte, suword8 .set susword, suword16 .set suword, suword64 .globl casuword32, casuword, fuptr, suptr .set casuword, casuword64 .set fuptr, fuword64 .set suptr, suword64 /* * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s) */ ENTRY(casuword32) casa [%o0] ASI_AIUP, %o1, %o2 retl mov %o2, %o0 END(casuword32) /* * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s) */ ENTRY(casuword64) casxa [%o0] ASI_AIUP, %o1, %o2 retl mov %o2, %o0 END(casuword64) /* * int fuword8(const void *base) */ ENTRY(fuword8) retl lduba [%o0] ASI_AIUP, %o0 END(fuword8) /* * int fuword16(const void *base) */ ENTRY(fuword16) retl lduha [%o0] ASI_AIUP, %o0 END(fuword16) /* * int32_t fuword32(const void *base) */ ENTRY(fuword32) retl lduwa [%o0] ASI_AIUP, %o0 END(fuword32) /* * int64_t fuword64(const void *base) */ ENTRY(fuword64) retl ldxa [%o0] ASI_AIUP, %o0 END(fuword64) /* * int suword8(const void *base, int word) */ ENTRY(suword8) stba %o1, [%o0] ASI_AIUP retl clr %o0 END(suword8) /* * int suword16(const void *base, int word) */ ENTRY(suword16) stha %o1, [%o0] ASI_AIUP retl clr %o0 END(suword16) /* * int suword32(const void *base, int32_t word) */ ENTRY(suword32) stwa %o1, [%o0] ASI_AIUP retl clr %o0 END(suword32) /* * int suword64(const void *base, int64_t word) */ ENTRY(suword64) stxa %o1, [%o0] ASI_AIUP retl clr %o0 END(suword64) .globl fs_nofault_intr_begin fs_nofault_intr_begin: nop /* * int fuswintr(const void *base) */ ENTRY(fuswintr) retl lduha [%o0] ASI_AIUP, %o0 END(fuswintr) /* * int suswintr(const void *base, int word) */ ENTRY(suswintr) stha %o1, [%o0] ASI_AIUP retl clr %o0 END(suswintr) .globl fs_nofault_intr_end fs_nofault_intr_end: nop .globl fs_nofault_end fs_nofault_end: nop ENTRY(fs_fault) retl mov -1, %o0 END(fs_fault) .globl fas_nofault_begin fas_nofault_begin: /* * int fasword8(u_long asi, uint64_t addr, uint8_t *val) */ ENTRY(fasword8) wr %o0, 0, %asi membar #Sync lduba [%o1] %asi, %o3 membar #Sync stb %o3, [%o2] retl clr %o0 END(fasword8) /* * int fasword16(u_long asi, uint64_t addr, uint16_t *val) */ ENTRY(fasword16) wr %o0, 0, %asi membar #Sync lduha [%o1] %asi, %o3 membar #Sync sth %o3, [%o2] retl clr %o0 END(fasword16) /* * int fasword32(u_long asi, uint64_t addr, uint32_t *val) */ ENTRY(fasword32) wr %o0, 0, %asi membar #Sync lduwa [%o1] %asi, %o3 membar #Sync stw %o3, [%o2] retl clr %o0 END(fasword32) .globl fas_nofault_end fas_nofault_end: nop .globl fas_fault ENTRY(fas_fault) retl mov -1, %o0 END(fas_fault) .globl fpu_fault_begin fpu_fault_begin: nop /* * void spitfire_block_copy(void *src, void *dst, size_t len) */ ENTRY(spitfire_block_copy) rdpr %pstate, %o3 wrpr %g0, PSTATE_NORMAL, %pstate wr %g0, ASI_BLK_S, %asi wr %g0, FPRS_FEF, %fprs sub PCB_REG, TF_SIZEOF, %o4 ldx [%o4 + TF_FPRS], %o5 andcc %o5, FPRS_FEF, %g0 bz,a,pt %xcc, 1f nop stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi membar #Sync andn %o5, FPRS_FEF, %o5 stx %o5, [%o4 + TF_FPRS] ldx [PCB_REG + PCB_FLAGS], %o4 or %o4, PCB_FEF, %o4 stx %o4, [PCB_REG + PCB_FLAGS] 1: wrpr %o3, 0, %pstate ldda [%o0] %asi, %f0 add %o0, VIS_BLOCKSIZE, %o0 sub %o2, VIS_BLOCKSIZE, %o2 2: ldda [%o0] %asi, %f16 fsrc1 %f0, %f32 fsrc1 %f2, %f34 fsrc1 %f4, %f36 fsrc1 %f6, %f38 fsrc1 %f8, %f40 fsrc1 %f10, %f42 fsrc1 %f12, %f44 fsrc1 %f14, %f46 stda %f32, [%o1] %asi add %o0, VIS_BLOCKSIZE, %o0 subcc %o2, VIS_BLOCKSIZE, %o2 bz,pn %xcc, 3f add %o1, VIS_BLOCKSIZE, %o1 ldda [%o0] %asi, %f0 fsrc1 %f16, %f32 fsrc1 %f18, %f34 fsrc1 %f20, %f36 fsrc1 %f22, %f38 fsrc1 %f24, %f40 fsrc1 %f26, %f42 fsrc1 %f28, %f44 fsrc1 %f30, %f46 stda %f32, [%o1] %asi add %o0, VIS_BLOCKSIZE, %o0 sub %o2, VIS_BLOCKSIZE, %o2 ba,pt %xcc, 2b add %o1, VIS_BLOCKSIZE, %o1 3: membar #Sync stda %f16, [%o1] %asi membar #Sync retl wr %g0, 0, %fprs END(spitfire_block_copy) /* * void zeus_block_copy(void *src, void *dst, size_t len) */ ENTRY(zeus_block_copy) prefetch [%o0 + (0 * VIS_BLOCKSIZE)], 0 rdpr %pstate, %o3 wrpr %g0, PSTATE_NORMAL, %pstate wr %g0, ASI_BLK_S, %asi wr %g0, FPRS_FEF, %fprs sub PCB_REG, TF_SIZEOF, %o4 ldx [%o4 + TF_FPRS], %o5 andcc %o5, FPRS_FEF, %g0 bz,a,pt %xcc, 1f nop stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi membar #Sync andn %o5, FPRS_FEF, %o5 stx %o5, [%o4 + TF_FPRS] ldx [PCB_REG + PCB_FLAGS], %o4 or %o4, PCB_FEF, %o4 stx %o4, [PCB_REG + PCB_FLAGS] 1: wrpr %o3, 0, %pstate ldd [%o0 + (0 * 8)], %f0 prefetch [%o0 + (1 * VIS_BLOCKSIZE)], 0 ldd [%o0 + (1 * 8)], %f2 prefetch [%o0 + (2 * VIS_BLOCKSIZE)], 0 fmovd %f0, %f32 ldd [%o0 + (2 * 8)], %f4 prefetch [%o0 + (3 * VIS_BLOCKSIZE)], 0 fmovd %f2, %f34 ldd [%o0 + (3 * 8)], %f6 prefetch [%o0 + (4 * VIS_BLOCKSIZE)], 1 fmovd %f4, %f36 ldd [%o0 + (4 * 8)], %f8 prefetch [%o0 + (8 * VIS_BLOCKSIZE)], 1 fmovd %f6, %f38 ldd [%o0 + (5 * 8)], %f10 prefetch [%o0 + (12 * VIS_BLOCKSIZE)], 1 fmovd %f8, %f40 ldd [%o0 + (6 * 8)], %f12 prefetch [%o0 + (16 * VIS_BLOCKSIZE)], 1 fmovd %f10, %f42 ldd [%o0 + (7 * 8)], %f14 ldd [%o0 + (8 * 8)], %f0 sub %o2, VIS_BLOCKSIZE, %o2 add %o0, VIS_BLOCKSIZE, %o0 prefetch [%o0 + (19 * VIS_BLOCKSIZE)], 1 ba,pt %xcc, 2f prefetch [%o0 + (23 * VIS_BLOCKSIZE)], 1 .align 32 2: ldd [%o0 + (1 * 8)], %f2 fmovd %f12, %f44 ldd [%o0 + (2 * 8)], %f4 fmovd %f14, %f46 stda %f32, [%o1] %asi ldd [%o0 + (3 * 8)], %f6 fmovd %f0, %f32 ldd [%o0 + (4 * 8)], %f8 fmovd %f2, %f34 ldd [%o0 + (5 * 8)], %f10 fmovd %f4, %f36 ldd [%o0 + (6 * 8)], %f12 fmovd %f6, %f38 ldd [%o0 + (7 * 8)], %f14 fmovd %f8, %f40 ldd [%o0 + (8 * 8)], %f0 fmovd %f10, %f42 sub %o2, VIS_BLOCKSIZE, %o2 prefetch [%o0 + (3 * VIS_BLOCKSIZE)], 0 add %o1, VIS_BLOCKSIZE, %o1 prefetch [%o0 + (24 * VIS_BLOCKSIZE)], 1 add %o0, VIS_BLOCKSIZE, %o0 cmp %o2, VIS_BLOCKSIZE + 8 bgu,pt %xcc, 2b prefetch [%o0 + (12 * VIS_BLOCKSIZE)], 1 ldd [%o0 + (1 * 8)], %f2 fsrc1 %f12, %f44 ldd [%o0 + (2 * 8)], %f4 fsrc1 %f14, %f46 stda %f32, [%o1] %asi ldd [%o0 + (3 * 8)], %f6 fsrc1 %f0, %f32 ldd [%o0 + (4 * 8)], %f8 fsrc1 %f2, %f34 ldd [%o0 + (5 * 8)], %f10 fsrc1 %f4, %f36 ldd [%o0 + (6 * 8)], %f12 fsrc1 %f6, %f38 ldd [%o0 + (7 * 8)], %f14 fsrc1 %f8, %f40 add %o1, VIS_BLOCKSIZE, %o1 fsrc1 %f10, %f42 fsrc1 %f12, %f44 fsrc1 %f14, %f46 stda %f32, [%o1] %asi membar #Sync retl wr %g0, 0, %fprs END(zeus_block_copy) /* * void spitfire_block_zero(void *dst, size_t len) * void zeus_block_zero(void *dst, size_t len) */ ALTENTRY(zeus_block_zero) ENTRY(spitfire_block_zero) rdpr %pstate, %o3 wrpr %g0, PSTATE_NORMAL, %pstate wr %g0, ASI_BLK_S, %asi wr %g0, FPRS_FEF, %fprs sub PCB_REG, TF_SIZEOF, %o4 ldx [%o4 + TF_FPRS], %o5 andcc %o5, FPRS_FEF, %g0 bz,a,pt %xcc, 1f nop stda %f0, [PCB_REG + PCB_UFP + (0 * VIS_BLOCKSIZE)] %asi stda %f16, [PCB_REG + PCB_UFP + (1 * VIS_BLOCKSIZE)] %asi stda %f32, [PCB_REG + PCB_UFP + (2 * VIS_BLOCKSIZE)] %asi stda %f48, [PCB_REG + PCB_UFP + (3 * VIS_BLOCKSIZE)] %asi membar #Sync andn %o5, FPRS_FEF, %o5 stx %o5, [%o4 + TF_FPRS] ldx [PCB_REG + PCB_FLAGS], %o4 or %o4, PCB_FEF, %o4 stx %o4, [PCB_REG + PCB_FLAGS] 1: wrpr %o3, 0, %pstate fzero %f0 fzero %f2 fzero %f4 fzero %f6 fzero %f8 fzero %f10 fzero %f12 fzero %f14 1: stda %f0, [%o0 + (0 * VIS_BLOCKSIZE)] %asi stda %f0, [%o0 + (1 * VIS_BLOCKSIZE)] %asi stda %f0, [%o0 + (2 * VIS_BLOCKSIZE)] %asi stda %f0, [%o0 + (3 * VIS_BLOCKSIZE)] %asi sub %o1, (4 * VIS_BLOCKSIZE), %o1 brnz,pt %o1, 1b add %o0, (4 * VIS_BLOCKSIZE), %o0 membar #Sync retl wr %g0, 0, %fprs END(spitfire_block_zero) .globl fpu_fault_end fpu_fault_end: nop .globl fpu_fault_size .set fpu_fault_size, fpu_fault_end - fpu_fault_begin ENTRY(longjmp) set 1, %g3 movrz %o1, %o1, %g3 mov %o0, %g1 ldx [%g1 + _JB_FP], %g2 1: cmp %fp, %g2 bl,a,pt %xcc, 1b restore bne,pn %xcc, 2f ldx [%g1 + _JB_SP], %o2 cmp %o2, %sp blt,pn %xcc, 2f movge %xcc, %o2, %sp ldx [%g1 + _JB_PC], %o7 retl mov %g3, %o0 2: PANIC("longjmp botch", %l1) END(longjmp) ENTRY(setjmp) stx %sp, [%o0 + _JB_SP] stx %o7, [%o0 + _JB_PC] stx %fp, [%o0 + _JB_FP] retl clr %o0 END(setjmp) /* * void ofw_entry(cell_t args[]) */ ENTRY(ofw_entry) save %sp, -CCFSZ, %sp SET(ofw_vec, %l7, %l6) ldx [%l6], %l6 rdpr %pstate, %l7 andn %l7, PSTATE_AM | PSTATE_IE, %l5 wrpr %l5, 0, %pstate SET(tba_taken_over, %l5, %l4) brz,pn %l4, 1f rdpr %wstate, %l5 andn %l5, WSTATE_PROM_MASK, %l3 wrpr %l3, WSTATE_PROM_KMIX, %wstate 1: call %l6 mov %i0, %o0 brz,pn %l4, 1f nop wrpr %g0, %l5, %wstate 1: wrpr %l7, 0, %pstate ret restore %o0, %g0, %o0 END(ofw_entry) /* * void ofw_exit(cell_t args[]) */ ENTRY(ofw_exit) save %sp, -CCFSZ, %sp flushw SET(ofw_tba, %l7, %l5) ldx [%l5], %l5 rdpr %pstate, %l7 andn %l7, PSTATE_AM | PSTATE_IE, %l7 wrpr %l7, 0, %pstate rdpr %wstate, %l7 andn %l7, WSTATE_PROM_MASK, %l7 wrpr %l7, WSTATE_PROM_KMIX, %wstate wrpr %l5, 0, %tba ! restore the OFW trap table SET(ofw_vec, %l7, %l6) ldx [%l6], %l6 SET(kstack0 + KSTACK_PAGES * PAGE_SIZE - PCB_SIZEOF, %l7, %l0) sub %l0, SPOFF, %fp ! setup a stack in a locked page sub %l0, SPOFF + CCFSZ, %sp mov AA_DMMU_PCXR, %l3 ! force primary DMMU context 0 sethi %hi(KERNBASE), %l5 stxa %g0, [%l3] ASI_DMMU flush %l5 wrpr %g0, 0, %tl ! force trap level 0 call %l6 mov %i0, %o0 ! never to return END(ofw_exit) #ifdef GPROF ENTRY(user) nop ENTRY(btrap) nop ENTRY(etrap) nop ENTRY(bintr) nop ENTRY(eintr) nop /* * XXX including sys/gmon.h in genassym.c is not possible due to uintfptr_t * badness. */ #define GM_STATE 0x0 #define GMON_PROF_OFF 3 #define GMON_PROF_HIRES 4 .globl _mcount .set _mcount, __cyg_profile_func_enter ENTRY(__cyg_profile_func_enter) SET(_gmonparam, %o3, %o2) lduw [%o2 + GM_STATE], %o3 cmp %o3, GMON_PROF_OFF be,a,pn %icc, 1f nop SET(mcount, %o3, %o2) jmpl %o2, %g0 nop 1: retl nop END(__cyg_profile_func_enter) #ifdef GUPROF ENTRY(__cyg_profile_func_exit) SET(_gmonparam, %o3, %o2) lduw [%o2 + GM_STATE], %o3 cmp %o3, GMON_PROF_HIRES be,a,pn %icc, 1f nop SET(mexitcount, %o3, %o2) jmpl %o2, %g0 nop 1: retl nop END(__cyg_profile_func_exit) #endif /* GUPROF */ #endif /* GPROF */ Index: head/sys/sparc64/sparc64/swtch.S =================================================================== --- head/sys/sparc64/sparc64/swtch.S (revision 331253) +++ head/sys/sparc64/sparc64/swtch.S (revision 331254) @@ -1,344 +1,344 @@ /*- * Copyright (c) 2001 Jake Burkholder. * Copyright (c) 2011 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include -#include "assym.s" +#include "assym.inc" #include "opt_sched.h" .register %g2, #ignore .register %g3, #ignore /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) save %sp, -CCFSZ, %sp flushw ba %xcc, .Lsw1 mov %g0, %i2 END(cpu_throw) /* * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) */ ENTRY(cpu_switch) save %sp, -CCFSZ, %sp /* * If the current thread was using floating point in the kernel, save * its context. The userland floating point context has already been * saved in that case. */ rd %fprs, %l2 andcc %l2, FPRS_FEF, %g0 bz,a,pt %xcc, 1f nop call savefpctx add PCB_REG, PCB_KFP, %o0 ba,a,pt %xcc, 2f nop /* * If the current thread was using floating point in userland, save * its context. */ 1: sub PCB_REG, TF_SIZEOF, %l2 ldx [%l2 + TF_FPRS], %l3 andcc %l3, FPRS_FEF, %g0 bz,a,pt %xcc, 2f nop call savefpctx add PCB_REG, PCB_UFP, %o0 andn %l3, FPRS_FEF, %l3 stx %l3, [%l2 + TF_FPRS] ldx [PCB_REG + PCB_FLAGS], %l3 or %l3, PCB_FEF, %l3 stx %l3, [PCB_REG + PCB_FLAGS] /* * Flush the windows out to the stack and save the current frame * pointer and program counter. */ 2: flushw wrpr %g0, 0, %cleanwin stx %fp, [PCB_REG + PCB_SP] stx %i7, [PCB_REG + PCB_PC] /* * Load the new thread's frame pointer and program counter, and set * the current thread and pcb. */ .Lsw1: #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "cpu_switch: new td=%p pc=%#lx fp=%#lx" , %g1, %g2, %g3, 8, 9, 10) stx %i1, [%g1 + KTR_PARM1] ldx [%i1 + TD_PCB], %g2 ldx [%g2 + PCB_PC], %g3 stx %g3, [%g1 + KTR_PARM2] ldx [%g2 + PCB_SP], %g3 stx %g3, [%g1 + KTR_PARM3] 10: #endif ldx [%i1 + TD_PCB], %l0 stx %i1, [PCPU(CURTHREAD)] stx %l0, [PCPU(CURPCB)] wrpr %g0, PSTATE_NORMAL, %pstate mov %l0, PCB_REG wrpr %g0, PSTATE_ALT, %pstate mov %l0, PCB_REG wrpr %g0, PSTATE_KERNEL, %pstate ldx [PCB_REG + PCB_SP], %fp ldx [PCB_REG + PCB_PC], %i7 sub %fp, CCFSZ, %sp /* * Point to the pmaps of the new process, and of the last non-kernel * process to run. */ ldx [%i1 + TD_PROC], %l1 ldx [PCPU(PMAP)], %l2 ldx [%l1 + P_VMSPACE], %i5 add %i5, VM_PMAP, %l1 #if KTR_COMPILE & KTR_PROC CATR(KTR_PROC, "cpu_switch: new pmap=%p old pmap=%p" , %g1, %g2, %g3, 8, 9, 10) stx %l1, [%g1 + KTR_PARM1] stx %l2, [%g1 + KTR_PARM2] 10: #endif /* * If they are the same we are done. */ cmp %l2, %l1 be,a,pn %xcc, 8f nop /* * If the new process is a kernel thread we can just leave the old * context active and avoid recycling its context number. */ SET(vmspace0, %i4, %i3) cmp %i5, %i3 be,a,pn %xcc, 8f nop /* * If there was no non-kernel pmap, don't try to deactivate it. */ brz,pn %l2, 3f lduw [PCPU(CPUID)], %l3 /* * Mark the pmap of the last non-kernel vmspace to run as no longer * active on this CPU. */ mov _NCPUBITS, %l5 udivx %l3, %l5, %l6 srl %l6, 0, %l4 sllx %l4, PTR_SHIFT, %l4 add %l4, PM_ACTIVE, %l4 smul %l6, %l5, %l5 sub %l3, %l5, %l5 mov 1, %l6 sllx %l6, %l5, %l5 #ifdef SMP add %l2, %l4, %l4 membar #LoadStore | #StoreStore ATOMIC_CLEAR_LONG(%l4, %l6, %l7, %l5) #else ldx [%l2 + %l4], %l6 andn %l6, %l5, %l6 stx %l6, [%l2 + %l4] #endif /* * Take away its context number. */ sllx %l3, INT_SHIFT, %l3 add %l2, PM_CONTEXT, %l4 mov -1, %l5 stw %l5, [%l3 + %l4] 3: cmp %i2, %g0 be,pn %xcc, 4f add %i0, TD_LOCK, %l4 #if defined(SCHED_ULE) && defined(SMP) membar #LoadStore | #StoreStore ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) #else stx %i2, [%l4] #endif /* * Find a new TLB context. If we've run out we have to flush all * user mappings from the TLB and reset the context numbers. */ 4: lduw [PCPU(TLB_CTX)], %i3 lduw [PCPU(TLB_CTX_MAX)], %i4 cmp %i3, %i4 bne,a,pt %xcc, 5f nop SET(tlb_flush_user, %i5, %i4) ldx [%i4], %i5 call %i5 lduw [PCPU(TLB_CTX_MIN)], %i3 /* * Advance next free context. */ 5: add %i3, 1, %i4 stw %i4, [PCPU(TLB_CTX)] /* * Set the new context number in the pmap. */ lduw [PCPU(CPUID)], %l3 sllx %l3, INT_SHIFT, %i4 add %l1, PM_CONTEXT, %i5 stw %i3, [%i4 + %i5] /* * Mark the pmap as active on this CPU. */ mov _NCPUBITS, %l5 udivx %l3, %l5, %l6 srl %l6, 0, %l4 sllx %l4, PTR_SHIFT, %l4 add %l4, PM_ACTIVE, %l4 smul %l6, %l5, %l5 sub %l3, %l5, %l5 mov 1, %l6 sllx %l6, %l5, %l5 #ifdef SMP add %l1, %l4, %l4 ATOMIC_SET_LONG(%l4, %l6, %l7, %l5) #else ldx [%l1 + %l4], %l6 or %l6, %l5, %l6 stx %l6, [%l1 + %l4] #endif /* * Make note of the change in pmap. */ #ifdef SMP PCPU_ADDR(PMAP, %l4) ATOMIC_STORE_LONG(%l4, %l5, %l6, %l1) #else stx %l1, [PCPU(PMAP)] #endif /* * Fiddle the hardware bits. Set the TSB registers and install the * new context number in the CPU. */ ldx [%l1 + PM_TSB], %i4 mov AA_DMMU_TSB, %i5 stxa %i4, [%i5] ASI_DMMU mov AA_IMMU_TSB, %i5 stxa %i4, [%i5] ASI_IMMU setx TLB_CXR_PGSZ_MASK, %i5, %i4 mov AA_DMMU_PCXR, %i5 ldxa [%i5] ASI_DMMU, %l1 and %l1, %i4, %l1 or %i3, %l1, %i3 sethi %hi(KERNBASE), %i4 stxa %i3, [%i5] ASI_DMMU flush %i4 6: #if defined(SCHED_ULE) && defined(SMP) SET(blocked_lock, %l2, %l1) add %i1, TD_LOCK, %l2 7: ATOMIC_LOAD_LONG(%l2, %l3) cmp %l1, %l3 be,a,pn %xcc, 7b nop #endif /* * Done, return and load the new process's window from the stack. */ ret restore 8: cmp %i2, %g0 be,pn %xcc, 6b add %i0, TD_LOCK, %l4 #if defined(SCHED_ULE) && defined(SMP) membar #LoadStore | #StoreStore ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2) ba,pt %xcc, 6b nop #else ba,pt %xcc, 6b stx %i2, [%l4] #endif END(cpu_switch) ENTRY(savectx) save %sp, -CCFSZ, %sp flushw call savefpctx add %i0, PCB_UFP, %o0 stx %fp, [%i0 + PCB_SP] stx %i7, [%i0 + PCB_PC] ret restore %g0, 0, %o0 END(savectx) /* * void savefpctx(uint32_t *); */ ENTRY(savefpctx) wr %g0, FPRS_FEF, %fprs wr %g0, ASI_BLK_S, %asi stda %f0, [%o0 + (0 * 64)] %asi stda %f16, [%o0 + (1 * 64)] %asi stda %f32, [%o0 + (2 * 64)] %asi stda %f48, [%o0 + (3 * 64)] %asi membar #Sync retl wr %g0, 0, %fprs END(savefpctx)