Index: head/sys/amd64/amd64/locore.S =================================================================== --- head/sys/amd64/amd64/locore.S (revision 27483) +++ head/sys/amd64/amd64/locore.S (revision 27484) @@ -1,991 +1,996 @@ /*- * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.90 1997/06/27 23:19:43 fsmp Exp $ + * $Id: locore.s,v 1.91 1997/07/15 11:07:32 kato Exp $ * * originally from: locore.s, by William F. Jolitz * * Substantially rewritten by David Greenman, Rod Grimes, * Bruce Evans, Wolfgang Solfrank, Poul-Henning Kamp * and many others. */ #include "apm.h" #include "opt_cpu.h" #include "opt_ddb.h" #include "opt_userconfig.h" #include "opt_smp.h" #include #include #include #include #include #include #include #include #ifdef SMP #include #endif /* SMP */ #include "assym.s" /* * XXX * * Note: This version greatly munged to avoid various assembler errors * that may be fixed in newer versions of gas. Perhaps newer versions * will have more pleasant appearance. */ /* * PTmap is recursive pagemap at top of virtual address space. * Within PTmap, the page directory can be found (third indirection). */ .globl _PTmap,_PTD,_PTDpde .set _PTmap,(PTDPTDI << PDRSHIFT) .set _PTD,_PTmap + (PTDPTDI * PAGE_SIZE) .set _PTDpde,_PTD + (PTDPTDI * PDESIZE) /* * APTmap, APTD is the alternate recursive pagemap. * It's used when modifying another process's page tables. */ .globl _APTmap,_APTD,_APTDpde .set _APTmap,APTDPTDI << PDRSHIFT .set _APTD,_APTmap + (APTDPTDI * PAGE_SIZE) .set _APTDpde,_PTD + (APTDPTDI * PDESIZE) #ifdef SMP .globl _SMP_prvstart .set _SMP_prvstart,(MPPTDI << PDRSHIFT) .globl _SMP_prvpage,_SMP_prvpt,_lapic,_SMP_ioapic .set _SMP_prvpage,_SMP_prvstart .set _SMP_prvpt,_SMP_prvstart + PAGE_SIZE .set _lapic,_SMP_prvstart + (2 * PAGE_SIZE) .set _SMP_ioapic,_SMP_prvstart + (16 * PAGE_SIZE) .globl _cpuid,_curproc,_curpcb,_npxproc,_runtime,_cpu_lockid .globl _common_tss,_other_cpus .set _cpuid,_SMP_prvpage+0 .set _curproc,_SMP_prvpage+4 .set _curpcb,_SMP_prvpage+8 .set _npxproc,_SMP_prvpage+12 .set _runtime,_SMP_prvpage+16 /* 8 bytes struct timeval */ .set _cpu_lockid,_SMP_prvpage+24 .set _common_tss,_SMP_prvpage+28 /* 104 bytes long, next = 132 */ .set _other_cpus,_SMP_prvpage+132 /* bitmap of available CPUs, excluding ourself */ /* Fetch the .set's for the local apic */ #include "i386/i386/mp_apicdefs.s" #endif /* * Globals */ .data ALIGN_DATA /* just to be sure */ .globl HIDENAME(tmpstk) .space 0x2000 /* space for tmpstk - temporary stack */ HIDENAME(tmpstk): .globl _boothowto,_bootdev .globl _cpu,_cpu_vendor,_cpu_id,_bootinfo .globl _cpu_high, _cpu_feature _cpu: .long 0 /* are we 386, 386sx, or 486 */ _cpu_id: .long 0 /* stepping ID */ _cpu_high: .long 0 /* highest arg to CPUID */ _cpu_feature: .long 0 /* features */ _cpu_vendor: .space 20 /* CPU origin code */ _bootinfo: .space BOOTINFO_SIZE /* bootinfo that we can handle */ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ #ifdef SMP cpu0pp: .long 0 /* phys addr cpu0 private pg */ cpu0pt: .long 0 /* phys addr cpu0 private pt */ .globl _cpu0prvpage,_cpu0prvpt _cpu0prvpage: .long 0 /* relocated version */ _cpu0prvpt: .long 0 /* relocated version */ #endif /* SMP */ .globl _IdlePTD _IdlePTD: .long 0 /* phys addr of kernel PTD */ #ifdef SMP .globl _KPTphys #endif _KPTphys: .long 0 /* phys addr of kernel page tables */ .globl _proc0paddr _proc0paddr: .long 0 /* address of proc 0 address space */ p0upa: .long 0 /* phys addr of proc0's UPAGES */ #ifdef BDE_DEBUGGER .globl _bdb_exists /* flag to indicate BDE debugger is present */ _bdb_exists: .long 0 #endif /********************************************************************** * * Some handy macros * */ #define R(foo) ((foo)-KERNBASE) #define ALLOCPAGES(foo) \ movl R(physfree), %esi ; \ movl $((foo)*PAGE_SIZE), %eax ; \ addl %esi, %eax ; \ movl %eax, R(physfree) ; \ movl %esi, %edi ; \ movl $((foo)*PAGE_SIZE),%ecx ; \ xorl %eax,%eax ; \ cld ; \ rep ; \ stosb /* * fillkpt * eax = page frame address * ebx = index into page table * ecx = how many pages to map * base = base address of page dir/table * prot = protection bits */ #define fillkpt(base, prot) \ shll $2,%ebx ; \ addl base,%ebx ; \ orl $PG_V,%eax ; \ orl prot,%eax ; \ 1: movl %eax,(%ebx) ; \ addl $PAGE_SIZE,%eax ; /* increment physical address */ \ addl $4,%ebx ; /* next pte */ \ loop 1b /* * fillkptphys(prot) * eax = physical address * ecx = how many pages to map * prot = protection bits */ #define fillkptphys(prot) \ movl %eax, %ebx ; \ shrl $PAGE_SHIFT, %ebx ; \ fillkpt(R(_KPTphys), prot) .text /********************************************************************** * * This is where the bootblocks start us, set the ball rolling... * */ NON_GPROF_ENTRY(btext) #ifdef PC98 jmp 1f .globl _pc98_system_parameter .org 0x400 _pc98_system_parameter: .space 0x240 /* BIOS parameter block */ 1: /* save SYSTEM PARAMETER for resume (NS/T or other) */ movl $0xa1000,%esi movl $0x100000,%edi movl $0x0630,%ecx cld rep movsb #else /* IBM-PC */ #ifdef BDE_DEBUGGER #ifdef BIOS_STEALS_3K cmpl $0x0375c339,0x95504 #else cmpl $0x0375c339,0x96104 /* XXX - debugger signature */ #endif jne 1f movb $1,R(_bdb_exists) 1: #endif /* Tell the bios to warmboot next time */ movw $0x1234,0x472 #endif /* PC98 */ /* Set up a real frame in case the double return in newboot is executed. */ pushl %ebp movl %esp, %ebp /* Don't trust what the BIOS gives for eflags. */ pushl $PSL_KERNEL popfl /* * Don't trust what the BIOS gives for %fs and %gs. Trust the bootstrap * to set %cs, %ds, %es and %ss. */ mov %ds, %ax mov %ax, %fs mov %ax, %gs call recover_bootinfo /* Get onto a stack that we can trust. */ /* * XXX this step is delayed in case recover_bootinfo needs to return via * the old stack, but it need not be, since recover_bootinfo actually * returns via the old frame. */ movl $R(HIDENAME(tmpstk)),%esp #ifdef PC98 testb $0x02,0x100620 /* pc98_machine_type & M_EPSON_PC98 */ jz 3f cmpb $0x0b,0x100624 /* epson_machine_id <= 0x0b */ ja 3f /* count up memory */ movl $0x100000,%eax /* next, talley remaining memory */ movl $0xFFF-0x100,%ecx 1: movl 0(%eax),%ebx /* save location to check */ movl $0xa55a5aa5,0(%eax) /* write test pattern */ cmpl $0xa55a5aa5,0(%eax) /* does not check yet for rollover */ jne 2f movl %ebx,0(%eax) /* restore memory */ addl $PAGE_SIZE,%eax loop 1b 2: subl $0x100000,%eax shrl $17,%eax movb %al,0x100401 3: #endif call identify_cpu /* clear bss */ /* * XXX this should be done a little earlier. * * XXX we don't check that there is memory for our bss and page tables * before using it. * * XXX the boot program somewhat bogusly clears the bss. We still have * to do it in case we were unzipped by kzipboot. Then the boot program * only clears kzipboot's bss. * * XXX the gdt and idt are still somewhere in the boot program. We * depend on the convention that the boot program is below 1MB and we * are above 1MB to keep the gdt and idt away from the bss and page * tables. The idt is only used if BDE_DEBUGGER is enabled. */ movl $R(_end),%ecx movl $R(_edata),%edi subl %edi,%ecx xorl %eax,%eax cld rep stosb #if NAPM > 0 /* * XXX it's not clear that APM can live in the current environonment. * Only pc-relative addressing works. */ call _apm_setup #endif call create_pagetables #ifdef BDE_DEBUGGER /* * Adjust as much as possible for paging before enabling paging so that the * adjustments can be traced. */ call bdb_prepare_paging #endif /* Now enable paging */ movl R(_IdlePTD), %eax movl %eax,%cr3 /* load ptd addr into mmu */ movl %cr0,%eax /* get control word */ orl $CR0_PE|CR0_PG,%eax /* enable paging */ movl %eax,%cr0 /* and let's page NOW! */ #ifdef BDE_DEBUGGER /* * Complete the adjustments for paging so that we can keep tracing through * initi386() after the low (physical) addresses for the gdt and idt become * invalid. */ call bdb_commit_paging #endif pushl $begin /* jump to high virtualized address */ ret /* now running relocated at KERNBASE where the system is linked to run */ begin: /* set up bootstrap stack */ movl _proc0paddr,%esp /* location of in-kernel pages */ addl $UPAGES*PAGE_SIZE,%esp /* bootstrap stack end location */ xorl %eax,%eax /* mark end of frames */ movl %eax,%ebp movl _proc0paddr,%eax movl _IdlePTD, %esi movl %esi,PCB_CR3(%eax) movl physfree, %esi pushl %esi /* value of first for init386(first) */ call _init386 /* wire 386 chip for unix operation */ popl %esi .globl __ucodesel,__udatasel pushl $0 /* unused */ pushl __udatasel /* ss */ pushl $0 /* esp - filled in by execve() */ pushl $PSL_USER /* eflags (IOPL 0, int enab) */ pushl __ucodesel /* cs */ pushl $0 /* eip - filled in by execve() */ subl $(12*4),%esp /* space for rest of registers */ pushl %esp /* call main with frame pointer */ call _main /* autoconfiguration, mountroot etc */ hlt /* never returns to here */ /* * When starting init, call this to configure the process for user * mode. This will be inherited by other processes. */ NON_GPROF_ENTRY(prepare_usermode) /* * Now we've run main() and determined what cpu-type we are, we can * enable write protection and alignment checking on i486 cpus and * above. */ #if defined(I486_CPU) || defined(I586_CPU) || defined(I686_CPU) cmpl $CPUCLASS_386,_cpu_class je 1f movl %cr0,%eax /* get control word */ orl $CR0_WP|CR0_AM,%eax /* enable i486 features */ movl %eax,%cr0 /* and do it */ 1: #endif /* * on return from main(), we are process 1 * set up address space and stack so that we can 'return' to user mode */ movl __ucodesel,%eax movl __udatasel,%ecx #if 0 movl %cx,%ds #endif movl %cx,%es movl %ax,%fs /* double map cs to fs */ movl %cx,%gs /* and ds to gs */ ret /* goto user! */ #define LCALL(x,y) .byte 0x9a ; .long y ; .word x /* * Signal trampoline, copied to top of user stack */ NON_GPROF_ENTRY(sigcode) call SIGF_HANDLER(%esp) lea SIGF_SC(%esp),%eax /* scp (the call may have clobbered the */ /* copy at 8(%esp)) */ pushl %eax pushl %eax /* junk to fake return address */ movl $SYS_sigreturn,%eax /* sigreturn() */ LCALL(0x7,0) /* enter kernel with args on stack */ hlt /* never gets here */ ALIGN_TEXT _esigcode: .data .globl _szsigcode _szsigcode: .long _esigcode-_sigcode .text /********************************************************************** * * Recover the bootinfo passed to us from the boot program * */ recover_bootinfo: /* * This code is called in different ways depending on what loaded * and started the kernel. This is used to detect how we get the * arguments from the other code and what we do with them. * * Old disk boot blocks: * (*btext)(howto, bootdev, cyloffset, esym); * [return address == 0, and can NOT be returned to] * [cyloffset was not supported by the FreeBSD boot code * and always passed in as 0] * [esym is also known as total in the boot code, and * was never properly supported by the FreeBSD boot code] * * Old diskless netboot code: * (*btext)(0,0,0,0,&nfsdiskless,0,0,0); * [return address != 0, and can NOT be returned to] * If we are being booted by this code it will NOT work, * so we are just going to halt if we find this case. * * New uniform boot code: * (*btext)(howto, bootdev, 0, 0, 0, &bootinfo) * [return address != 0, and can be returned to] * * There may seem to be a lot of wasted arguments in here, but * that is so the newer boot code can still load very old kernels * and old boot code can load new kernels. */ /* * The old style disk boot blocks fake a frame on the stack and * did an lret to get here. The frame on the stack has a return * address of 0. */ cmpl $0,4(%ebp) je olddiskboot /* * We have some form of return address, so this is either the * old diskless netboot code, or the new uniform code. That can * be detected by looking at the 5th argument, if it is 0 * we are being booted by the new uniform boot code. */ cmpl $0,24(%ebp) je newboot /* * Seems we have been loaded by the old diskless boot code, we * don't stand a chance of running as the diskless structure * changed considerably between the two, so just halt. */ hlt /* * We have been loaded by the new uniform boot code. * Let's check the bootinfo version, and if we do not understand * it we return to the loader with a status of 1 to indicate this error */ newboot: movl 28(%ebp),%ebx /* &bootinfo.version */ movl BI_VERSION(%ebx),%eax cmpl $1,%eax /* We only understand version 1 */ je 1f movl $1,%eax /* Return status */ leave /* * XXX this returns to our caller's caller (as is required) since * we didn't set up a frame and our caller did. */ ret 1: /* * If we have a kernelname copy it in */ movl BI_KERNELNAME(%ebx),%esi cmpl $0,%esi je 2f /* No kernelname */ movl $MAXPATHLEN,%ecx /* Brute force!!! */ movl $R(_kernelname),%edi cmpb $'/',(%esi) /* Make sure it starts with a slash */ je 1f movb $'/',(%edi) incl %edi decl %ecx 1: cld rep movsb 2: /* * Determine the size of the boot loader's copy of the bootinfo * struct. This is impossible to do properly because old versions * of the struct don't contain a size field and there are 2 old * versions with the same version number. */ movl $BI_ENDCOMMON,%ecx /* prepare for sizeless version */ testl $RB_BOOTINFO,8(%ebp) /* bi_size (and bootinfo) valid? */ je got_bi_size /* no, sizeless version */ movl BI_SIZE(%ebx),%ecx got_bi_size: /* * Copy the common part of the bootinfo struct */ movl %ebx,%esi movl $R(_bootinfo),%edi cmpl $BOOTINFO_SIZE,%ecx jbe got_common_bi_size movl $BOOTINFO_SIZE,%ecx got_common_bi_size: cld rep movsb #ifdef NFS #ifndef BOOTP_NFSV3 /* * If we have a nfs_diskless structure copy it in */ movl BI_NFS_DISKLESS(%ebx),%esi cmpl $0,%esi je olddiskboot movl $R(_nfs_diskless),%edi movl $NFSDISKLESS_SIZE,%ecx cld rep movsb movl $R(_nfs_diskless_valid),%edi movl $1,(%edi) #endif #endif /* * The old style disk boot. * (*btext)(howto, bootdev, cyloffset, esym); * Note that the newer boot code just falls into here to pick * up howto and bootdev, cyloffset and esym are no longer used */ olddiskboot: movl 8(%ebp),%eax movl %eax,R(_boothowto) movl 12(%ebp),%eax movl %eax,R(_bootdev) #if defined(USERCONFIG_BOOT) && defined(USERCONFIG) #ifdef PC98 movl $0x90200, %esi #else movl $0x10200, %esi #endif movl $R(_userconfig_from_boot),%edi movl $512,%ecx cld rep movsb #endif /* USERCONFIG_BOOT */ ret /********************************************************************** * * Identify the CPU and initialize anything special about it * */ identify_cpu: /* Try to toggle alignment check flag; does not exist on 386. */ pushfl popl %eax movl %eax,%ecx orl $PSL_AC,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_AC,%eax pushl %ecx popfl testl %eax,%eax jnz try486 /* NexGen CPU does not have aligment check flag. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jz trynexgen popfl movl $CPU_386,R(_cpu) jmp 3f trynexgen: popfl movl $CPU_NX586,R(_cpu) movl $0x4778654e,R(_cpu_vendor) # store vendor string movl $0x72446e65,R(_cpu_vendor+4) movl $0x6e657669,R(_cpu_vendor+8) movl $0,R(_cpu_vendor+12) jmp 3f try486: /* Try to toggle identification flag; does not exist on early 486s. */ pushfl popl %eax movl %eax,%ecx xorl $PSL_ID,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_ID,%eax pushl %ecx popfl testl %eax,%eax jnz trycpuid movl $CPU_486,R(_cpu) /* * Check Cyrix CPU * Cyrix CPUs do not change the undefined flags following * execution of the divide instruction which divides 5 by 2. * * Note: CPUID is enabled on M2, so it passes another way. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jnc trycyrix popfl jmp 3f /* You may use Intel CPU. */ trycyrix: popfl /* * IBM Bluelighting CPU also doesn't change the undefined flags. * Because IBM doesn't disclose the information for Bluelighting * CPU, we couldn't distinguish it from Cyrix's (including IBM * brand of Cyrix CPUs). */ movl $0x69727943,R(_cpu_vendor) # store vendor string movl $0x736e4978,R(_cpu_vendor+4) movl $0x64616574,R(_cpu_vendor+8) jmp 3f trycpuid: /* Use the `cpuid' instruction. */ xorl %eax,%eax .byte 0x0f,0xa2 # cpuid 0 movl %eax,R(_cpu_high) # highest capability movl %ebx,R(_cpu_vendor) # store vendor string movl %edx,R(_cpu_vendor+4) movl %ecx,R(_cpu_vendor+8) movb $0,R(_cpu_vendor+12) movl $1,%eax .byte 0x0f,0xa2 # cpuid 1 movl %eax,R(_cpu_id) # store cpu_id movl %edx,R(_cpu_feature) # store cpu_feature rorl $8,%eax # extract family type andl $15,%eax cmpl $5,%eax jae 1f /* less than Pentium; must be 486 */ movl $CPU_486,R(_cpu) jmp 3f 1: /* a Pentium? */ cmpl $5,%eax jne 2f movl $CPU_586,R(_cpu) jmp 3f 2: /* Greater than Pentium...call it a Pentium Pro */ movl $CPU_686,R(_cpu) 3: ret /********************************************************************** * * Create the first page directory and its page tables. * */ create_pagetables: testl $CPUID_PGE, R(_cpu_feature) jz 1f movl %cr4, %eax orl $CR4_PGE, %eax movl %eax, %cr4 1: /* Find end of kernel image (rounded up to a page boundary). */ movl $R(_end),%esi /* include symbols in "kernel image" if they are loaded and useful */ #ifdef DDB movl R(_bootinfo+BI_ESYMTAB),%edi testl %edi,%edi je over_symalloc movl %edi,%esi movl $KERNBASE,%edi addl %edi,R(_bootinfo+BI_SYMTAB) addl %edi,R(_bootinfo+BI_ESYMTAB) over_symalloc: #endif addl $PAGE_MASK,%esi andl $~PAGE_MASK,%esi movl %esi,R(_KERNend) /* save end of kernel */ movl %esi,R(physfree) /* next free page is at end of kernel */ /* Allocate Kernel Page Tables */ ALLOCPAGES(NKPT) movl %esi,R(_KPTphys) /* Allocate Page Table Directory */ ALLOCPAGES(1) movl %esi,R(_IdlePTD) /* Allocate UPAGES */ ALLOCPAGES(UPAGES) movl %esi,R(p0upa) addl $KERNBASE, %esi movl %esi, R(_proc0paddr) #ifdef SMP /* Allocate cpu0's private data page */ ALLOCPAGES(1) movl %esi,R(cpu0pp) addl $KERNBASE, %esi movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ /* Allocate cpu0's private page table for mapping priv page, apic, etc */ ALLOCPAGES(1) movl %esi,R(cpu0pt) addl $KERNBASE, %esi movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ #endif /* SMP */ /* Map read-only from zero to the end of the kernel text section */ xorl %eax, %eax #ifdef BDE_DEBUGGER /* If the debugger is present, actually map everything read-write. */ cmpl $0,R(_bdb_exists) jne map_read_write #endif xorl %edx,%edx + +#if !defined(SMP) testl $CPUID_PGE, R(_cpu_feature) jz 2f orl $PG_G,%edx +#endif 2: movl $R(_etext),%ecx addl $PAGE_MASK,%ecx shrl $PAGE_SHIFT,%ecx fillkptphys(%edx) /* Map read-write, data, bss and symbols */ movl $R(_etext),%eax addl $PAGE_MASK, %eax andl $~PAGE_MASK, %eax map_read_write: movl $PG_RW,%edx +#if !defined(SMP) testl $CPUID_PGE, R(_cpu_feature) jz 1f orl $PG_G,%edx +#endif 1: movl R(_KERNend),%ecx subl %eax,%ecx shrl $PAGE_SHIFT,%ecx fillkptphys(%edx) /* Map page directory. */ movl R(_IdlePTD), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map proc0's UPAGES in the physical way ... */ movl R(p0upa), %eax movl $UPAGES, %ecx fillkptphys($PG_RW) /* Map ISA hole */ movl $ISA_HOLE_START, %eax movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkptphys($PG_RW) #ifdef SMP /* Map cpu0's private page into global kmem (4K @ cpu0prvpage) */ movl R(cpu0pp), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map cpu0's private page table into global kmem FWIW */ movl R(cpu0pt), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map the private page into the private page table into private space */ movl R(cpu0pp), %eax movl $0, %ebx /* pte offset = 0 */ movl $1, %ecx /* one private page coming right up */ fillkpt(R(cpu0pt), $PG_RW) /* Map the page table page into private space */ movl R(cpu0pt), %eax movl $1, %ebx /* pte offset = 1 */ movl $1, %ecx /* one private pt coming right up */ fillkpt(R(cpu0pt), $PG_RW) /* ... and put the page table table in the pde. */ movl R(cpu0pt), %eax movl $MPPTDI, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) #endif /* SMP */ /* install a pde for temporary double map of bottom of VA */ movl R(_KPTphys), %eax xorl %ebx, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) /* install pde's for pt's */ movl R(_KPTphys), %eax movl $KPTDI, %ebx movl $NKPT, %ecx fillkpt(R(_IdlePTD), $PG_RW) /* install a pde recursively mapping page directory as a page table */ movl R(_IdlePTD), %eax movl $PTDPTDI, %ebx movl $1,%ecx fillkpt(R(_IdlePTD), $PG_RW) ret #ifdef BDE_DEBUGGER bdb_prepare_paging: cmpl $0,R(_bdb_exists) je bdb_prepare_paging_exit subl $6,%esp /* * Copy and convert debugger entries from the bootstrap gdt and idt * to the kernel gdt and idt. Everything is still in low memory. * Tracing continues to work after paging is enabled because the * low memory addresses remain valid until everything is relocated. * However, tracing through the setidt() that initializes the trace * trap will crash. */ sgdt (%esp) movl 2(%esp),%esi /* base address of bootstrap gdt */ movl $R(_gdt),%edi movl %edi,2(%esp) /* prepare to load kernel gdt */ movl $8*18/4,%ecx cld rep /* copy gdt */ movsl movl $R(_gdt),-8+2(%edi) /* adjust gdt self-ptr */ movb $0x92,-8+5(%edi) lgdt (%esp) sidt (%esp) movl 2(%esp),%esi /* base address of current idt */ movl 8+4(%esi),%eax /* convert dbg descriptor to ... */ movw 8(%esi),%ax movl %eax,R(bdb_dbg_ljmp+1) /* ... immediate offset ... */ movl 8+2(%esi),%eax movw %ax,R(bdb_dbg_ljmp+5) /* ... and selector for ljmp */ movl 24+4(%esi),%eax /* same for bpt descriptor */ movw 24(%esi),%ax movl %eax,R(bdb_bpt_ljmp+1) movl 24+2(%esi),%eax movw %ax,R(bdb_bpt_ljmp+5) movl $R(_idt),%edi movl %edi,2(%esp) /* prepare to load kernel idt */ movl $8*4/4,%ecx cld rep /* copy idt */ movsl lidt (%esp) addl $6,%esp bdb_prepare_paging_exit: ret /* Relocate debugger gdt entries and gdt and idt pointers. */ bdb_commit_paging: cmpl $0,_bdb_exists je bdb_commit_paging_exit movl $_gdt+8*9,%eax /* adjust slots 9-17 */ movl $9,%ecx reloc_gdt: movb $KERNBASE>>24,7(%eax) /* top byte of base addresses, was 0, */ addl $8,%eax /* now KERNBASE>>24 */ loop reloc_gdt subl $6,%esp sgdt (%esp) addl $KERNBASE,2(%esp) lgdt (%esp) sidt (%esp) addl $KERNBASE,2(%esp) lidt (%esp) addl $6,%esp int $3 bdb_commit_paging_exit: ret #endif /* BDE_DEBUGGER */ Index: head/sys/amd64/amd64/locore.s =================================================================== --- head/sys/amd64/amd64/locore.s (revision 27483) +++ head/sys/amd64/amd64/locore.s (revision 27484) @@ -1,991 +1,996 @@ /*- * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.90 1997/06/27 23:19:43 fsmp Exp $ + * $Id: locore.s,v 1.91 1997/07/15 11:07:32 kato Exp $ * * originally from: locore.s, by William F. Jolitz * * Substantially rewritten by David Greenman, Rod Grimes, * Bruce Evans, Wolfgang Solfrank, Poul-Henning Kamp * and many others. */ #include "apm.h" #include "opt_cpu.h" #include "opt_ddb.h" #include "opt_userconfig.h" #include "opt_smp.h" #include #include #include #include #include #include #include #include #ifdef SMP #include #endif /* SMP */ #include "assym.s" /* * XXX * * Note: This version greatly munged to avoid various assembler errors * that may be fixed in newer versions of gas. Perhaps newer versions * will have more pleasant appearance. */ /* * PTmap is recursive pagemap at top of virtual address space. * Within PTmap, the page directory can be found (third indirection). */ .globl _PTmap,_PTD,_PTDpde .set _PTmap,(PTDPTDI << PDRSHIFT) .set _PTD,_PTmap + (PTDPTDI * PAGE_SIZE) .set _PTDpde,_PTD + (PTDPTDI * PDESIZE) /* * APTmap, APTD is the alternate recursive pagemap. * It's used when modifying another process's page tables. */ .globl _APTmap,_APTD,_APTDpde .set _APTmap,APTDPTDI << PDRSHIFT .set _APTD,_APTmap + (APTDPTDI * PAGE_SIZE) .set _APTDpde,_PTD + (APTDPTDI * PDESIZE) #ifdef SMP .globl _SMP_prvstart .set _SMP_prvstart,(MPPTDI << PDRSHIFT) .globl _SMP_prvpage,_SMP_prvpt,_lapic,_SMP_ioapic .set _SMP_prvpage,_SMP_prvstart .set _SMP_prvpt,_SMP_prvstart + PAGE_SIZE .set _lapic,_SMP_prvstart + (2 * PAGE_SIZE) .set _SMP_ioapic,_SMP_prvstart + (16 * PAGE_SIZE) .globl _cpuid,_curproc,_curpcb,_npxproc,_runtime,_cpu_lockid .globl _common_tss,_other_cpus .set _cpuid,_SMP_prvpage+0 .set _curproc,_SMP_prvpage+4 .set _curpcb,_SMP_prvpage+8 .set _npxproc,_SMP_prvpage+12 .set _runtime,_SMP_prvpage+16 /* 8 bytes struct timeval */ .set _cpu_lockid,_SMP_prvpage+24 .set _common_tss,_SMP_prvpage+28 /* 104 bytes long, next = 132 */ .set _other_cpus,_SMP_prvpage+132 /* bitmap of available CPUs, excluding ourself */ /* Fetch the .set's for the local apic */ #include "i386/i386/mp_apicdefs.s" #endif /* * Globals */ .data ALIGN_DATA /* just to be sure */ .globl HIDENAME(tmpstk) .space 0x2000 /* space for tmpstk - temporary stack */ HIDENAME(tmpstk): .globl _boothowto,_bootdev .globl _cpu,_cpu_vendor,_cpu_id,_bootinfo .globl _cpu_high, _cpu_feature _cpu: .long 0 /* are we 386, 386sx, or 486 */ _cpu_id: .long 0 /* stepping ID */ _cpu_high: .long 0 /* highest arg to CPUID */ _cpu_feature: .long 0 /* features */ _cpu_vendor: .space 20 /* CPU origin code */ _bootinfo: .space BOOTINFO_SIZE /* bootinfo that we can handle */ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ #ifdef SMP cpu0pp: .long 0 /* phys addr cpu0 private pg */ cpu0pt: .long 0 /* phys addr cpu0 private pt */ .globl _cpu0prvpage,_cpu0prvpt _cpu0prvpage: .long 0 /* relocated version */ _cpu0prvpt: .long 0 /* relocated version */ #endif /* SMP */ .globl _IdlePTD _IdlePTD: .long 0 /* phys addr of kernel PTD */ #ifdef SMP .globl _KPTphys #endif _KPTphys: .long 0 /* phys addr of kernel page tables */ .globl _proc0paddr _proc0paddr: .long 0 /* address of proc 0 address space */ p0upa: .long 0 /* phys addr of proc0's UPAGES */ #ifdef BDE_DEBUGGER .globl _bdb_exists /* flag to indicate BDE debugger is present */ _bdb_exists: .long 0 #endif /********************************************************************** * * Some handy macros * */ #define R(foo) ((foo)-KERNBASE) #define ALLOCPAGES(foo) \ movl R(physfree), %esi ; \ movl $((foo)*PAGE_SIZE), %eax ; \ addl %esi, %eax ; \ movl %eax, R(physfree) ; \ movl %esi, %edi ; \ movl $((foo)*PAGE_SIZE),%ecx ; \ xorl %eax,%eax ; \ cld ; \ rep ; \ stosb /* * fillkpt * eax = page frame address * ebx = index into page table * ecx = how many pages to map * base = base address of page dir/table * prot = protection bits */ #define fillkpt(base, prot) \ shll $2,%ebx ; \ addl base,%ebx ; \ orl $PG_V,%eax ; \ orl prot,%eax ; \ 1: movl %eax,(%ebx) ; \ addl $PAGE_SIZE,%eax ; /* increment physical address */ \ addl $4,%ebx ; /* next pte */ \ loop 1b /* * fillkptphys(prot) * eax = physical address * ecx = how many pages to map * prot = protection bits */ #define fillkptphys(prot) \ movl %eax, %ebx ; \ shrl $PAGE_SHIFT, %ebx ; \ fillkpt(R(_KPTphys), prot) .text /********************************************************************** * * This is where the bootblocks start us, set the ball rolling... * */ NON_GPROF_ENTRY(btext) #ifdef PC98 jmp 1f .globl _pc98_system_parameter .org 0x400 _pc98_system_parameter: .space 0x240 /* BIOS parameter block */ 1: /* save SYSTEM PARAMETER for resume (NS/T or other) */ movl $0xa1000,%esi movl $0x100000,%edi movl $0x0630,%ecx cld rep movsb #else /* IBM-PC */ #ifdef BDE_DEBUGGER #ifdef BIOS_STEALS_3K cmpl $0x0375c339,0x95504 #else cmpl $0x0375c339,0x96104 /* XXX - debugger signature */ #endif jne 1f movb $1,R(_bdb_exists) 1: #endif /* Tell the bios to warmboot next time */ movw $0x1234,0x472 #endif /* PC98 */ /* Set up a real frame in case the double return in newboot is executed. */ pushl %ebp movl %esp, %ebp /* Don't trust what the BIOS gives for eflags. */ pushl $PSL_KERNEL popfl /* * Don't trust what the BIOS gives for %fs and %gs. Trust the bootstrap * to set %cs, %ds, %es and %ss. */ mov %ds, %ax mov %ax, %fs mov %ax, %gs call recover_bootinfo /* Get onto a stack that we can trust. */ /* * XXX this step is delayed in case recover_bootinfo needs to return via * the old stack, but it need not be, since recover_bootinfo actually * returns via the old frame. */ movl $R(HIDENAME(tmpstk)),%esp #ifdef PC98 testb $0x02,0x100620 /* pc98_machine_type & M_EPSON_PC98 */ jz 3f cmpb $0x0b,0x100624 /* epson_machine_id <= 0x0b */ ja 3f /* count up memory */ movl $0x100000,%eax /* next, talley remaining memory */ movl $0xFFF-0x100,%ecx 1: movl 0(%eax),%ebx /* save location to check */ movl $0xa55a5aa5,0(%eax) /* write test pattern */ cmpl $0xa55a5aa5,0(%eax) /* does not check yet for rollover */ jne 2f movl %ebx,0(%eax) /* restore memory */ addl $PAGE_SIZE,%eax loop 1b 2: subl $0x100000,%eax shrl $17,%eax movb %al,0x100401 3: #endif call identify_cpu /* clear bss */ /* * XXX this should be done a little earlier. * * XXX we don't check that there is memory for our bss and page tables * before using it. * * XXX the boot program somewhat bogusly clears the bss. We still have * to do it in case we were unzipped by kzipboot. Then the boot program * only clears kzipboot's bss. * * XXX the gdt and idt are still somewhere in the boot program. We * depend on the convention that the boot program is below 1MB and we * are above 1MB to keep the gdt and idt away from the bss and page * tables. The idt is only used if BDE_DEBUGGER is enabled. */ movl $R(_end),%ecx movl $R(_edata),%edi subl %edi,%ecx xorl %eax,%eax cld rep stosb #if NAPM > 0 /* * XXX it's not clear that APM can live in the current environonment. * Only pc-relative addressing works. */ call _apm_setup #endif call create_pagetables #ifdef BDE_DEBUGGER /* * Adjust as much as possible for paging before enabling paging so that the * adjustments can be traced. */ call bdb_prepare_paging #endif /* Now enable paging */ movl R(_IdlePTD), %eax movl %eax,%cr3 /* load ptd addr into mmu */ movl %cr0,%eax /* get control word */ orl $CR0_PE|CR0_PG,%eax /* enable paging */ movl %eax,%cr0 /* and let's page NOW! */ #ifdef BDE_DEBUGGER /* * Complete the adjustments for paging so that we can keep tracing through * initi386() after the low (physical) addresses for the gdt and idt become * invalid. */ call bdb_commit_paging #endif pushl $begin /* jump to high virtualized address */ ret /* now running relocated at KERNBASE where the system is linked to run */ begin: /* set up bootstrap stack */ movl _proc0paddr,%esp /* location of in-kernel pages */ addl $UPAGES*PAGE_SIZE,%esp /* bootstrap stack end location */ xorl %eax,%eax /* mark end of frames */ movl %eax,%ebp movl _proc0paddr,%eax movl _IdlePTD, %esi movl %esi,PCB_CR3(%eax) movl physfree, %esi pushl %esi /* value of first for init386(first) */ call _init386 /* wire 386 chip for unix operation */ popl %esi .globl __ucodesel,__udatasel pushl $0 /* unused */ pushl __udatasel /* ss */ pushl $0 /* esp - filled in by execve() */ pushl $PSL_USER /* eflags (IOPL 0, int enab) */ pushl __ucodesel /* cs */ pushl $0 /* eip - filled in by execve() */ subl $(12*4),%esp /* space for rest of registers */ pushl %esp /* call main with frame pointer */ call _main /* autoconfiguration, mountroot etc */ hlt /* never returns to here */ /* * When starting init, call this to configure the process for user * mode. This will be inherited by other processes. */ NON_GPROF_ENTRY(prepare_usermode) /* * Now we've run main() and determined what cpu-type we are, we can * enable write protection and alignment checking on i486 cpus and * above. */ #if defined(I486_CPU) || defined(I586_CPU) || defined(I686_CPU) cmpl $CPUCLASS_386,_cpu_class je 1f movl %cr0,%eax /* get control word */ orl $CR0_WP|CR0_AM,%eax /* enable i486 features */ movl %eax,%cr0 /* and do it */ 1: #endif /* * on return from main(), we are process 1 * set up address space and stack so that we can 'return' to user mode */ movl __ucodesel,%eax movl __udatasel,%ecx #if 0 movl %cx,%ds #endif movl %cx,%es movl %ax,%fs /* double map cs to fs */ movl %cx,%gs /* and ds to gs */ ret /* goto user! */ #define LCALL(x,y) .byte 0x9a ; .long y ; .word x /* * Signal trampoline, copied to top of user stack */ NON_GPROF_ENTRY(sigcode) call SIGF_HANDLER(%esp) lea SIGF_SC(%esp),%eax /* scp (the call may have clobbered the */ /* copy at 8(%esp)) */ pushl %eax pushl %eax /* junk to fake return address */ movl $SYS_sigreturn,%eax /* sigreturn() */ LCALL(0x7,0) /* enter kernel with args on stack */ hlt /* never gets here */ ALIGN_TEXT _esigcode: .data .globl _szsigcode _szsigcode: .long _esigcode-_sigcode .text /********************************************************************** * * Recover the bootinfo passed to us from the boot program * */ recover_bootinfo: /* * This code is called in different ways depending on what loaded * and started the kernel. This is used to detect how we get the * arguments from the other code and what we do with them. * * Old disk boot blocks: * (*btext)(howto, bootdev, cyloffset, esym); * [return address == 0, and can NOT be returned to] * [cyloffset was not supported by the FreeBSD boot code * and always passed in as 0] * [esym is also known as total in the boot code, and * was never properly supported by the FreeBSD boot code] * * Old diskless netboot code: * (*btext)(0,0,0,0,&nfsdiskless,0,0,0); * [return address != 0, and can NOT be returned to] * If we are being booted by this code it will NOT work, * so we are just going to halt if we find this case. * * New uniform boot code: * (*btext)(howto, bootdev, 0, 0, 0, &bootinfo) * [return address != 0, and can be returned to] * * There may seem to be a lot of wasted arguments in here, but * that is so the newer boot code can still load very old kernels * and old boot code can load new kernels. */ /* * The old style disk boot blocks fake a frame on the stack and * did an lret to get here. The frame on the stack has a return * address of 0. */ cmpl $0,4(%ebp) je olddiskboot /* * We have some form of return address, so this is either the * old diskless netboot code, or the new uniform code. That can * be detected by looking at the 5th argument, if it is 0 * we are being booted by the new uniform boot code. */ cmpl $0,24(%ebp) je newboot /* * Seems we have been loaded by the old diskless boot code, we * don't stand a chance of running as the diskless structure * changed considerably between the two, so just halt. */ hlt /* * We have been loaded by the new uniform boot code. * Let's check the bootinfo version, and if we do not understand * it we return to the loader with a status of 1 to indicate this error */ newboot: movl 28(%ebp),%ebx /* &bootinfo.version */ movl BI_VERSION(%ebx),%eax cmpl $1,%eax /* We only understand version 1 */ je 1f movl $1,%eax /* Return status */ leave /* * XXX this returns to our caller's caller (as is required) since * we didn't set up a frame and our caller did. */ ret 1: /* * If we have a kernelname copy it in */ movl BI_KERNELNAME(%ebx),%esi cmpl $0,%esi je 2f /* No kernelname */ movl $MAXPATHLEN,%ecx /* Brute force!!! */ movl $R(_kernelname),%edi cmpb $'/',(%esi) /* Make sure it starts with a slash */ je 1f movb $'/',(%edi) incl %edi decl %ecx 1: cld rep movsb 2: /* * Determine the size of the boot loader's copy of the bootinfo * struct. This is impossible to do properly because old versions * of the struct don't contain a size field and there are 2 old * versions with the same version number. */ movl $BI_ENDCOMMON,%ecx /* prepare for sizeless version */ testl $RB_BOOTINFO,8(%ebp) /* bi_size (and bootinfo) valid? */ je got_bi_size /* no, sizeless version */ movl BI_SIZE(%ebx),%ecx got_bi_size: /* * Copy the common part of the bootinfo struct */ movl %ebx,%esi movl $R(_bootinfo),%edi cmpl $BOOTINFO_SIZE,%ecx jbe got_common_bi_size movl $BOOTINFO_SIZE,%ecx got_common_bi_size: cld rep movsb #ifdef NFS #ifndef BOOTP_NFSV3 /* * If we have a nfs_diskless structure copy it in */ movl BI_NFS_DISKLESS(%ebx),%esi cmpl $0,%esi je olddiskboot movl $R(_nfs_diskless),%edi movl $NFSDISKLESS_SIZE,%ecx cld rep movsb movl $R(_nfs_diskless_valid),%edi movl $1,(%edi) #endif #endif /* * The old style disk boot. * (*btext)(howto, bootdev, cyloffset, esym); * Note that the newer boot code just falls into here to pick * up howto and bootdev, cyloffset and esym are no longer used */ olddiskboot: movl 8(%ebp),%eax movl %eax,R(_boothowto) movl 12(%ebp),%eax movl %eax,R(_bootdev) #if defined(USERCONFIG_BOOT) && defined(USERCONFIG) #ifdef PC98 movl $0x90200, %esi #else movl $0x10200, %esi #endif movl $R(_userconfig_from_boot),%edi movl $512,%ecx cld rep movsb #endif /* USERCONFIG_BOOT */ ret /********************************************************************** * * Identify the CPU and initialize anything special about it * */ identify_cpu: /* Try to toggle alignment check flag; does not exist on 386. */ pushfl popl %eax movl %eax,%ecx orl $PSL_AC,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_AC,%eax pushl %ecx popfl testl %eax,%eax jnz try486 /* NexGen CPU does not have aligment check flag. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jz trynexgen popfl movl $CPU_386,R(_cpu) jmp 3f trynexgen: popfl movl $CPU_NX586,R(_cpu) movl $0x4778654e,R(_cpu_vendor) # store vendor string movl $0x72446e65,R(_cpu_vendor+4) movl $0x6e657669,R(_cpu_vendor+8) movl $0,R(_cpu_vendor+12) jmp 3f try486: /* Try to toggle identification flag; does not exist on early 486s. */ pushfl popl %eax movl %eax,%ecx xorl $PSL_ID,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_ID,%eax pushl %ecx popfl testl %eax,%eax jnz trycpuid movl $CPU_486,R(_cpu) /* * Check Cyrix CPU * Cyrix CPUs do not change the undefined flags following * execution of the divide instruction which divides 5 by 2. * * Note: CPUID is enabled on M2, so it passes another way. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jnc trycyrix popfl jmp 3f /* You may use Intel CPU. */ trycyrix: popfl /* * IBM Bluelighting CPU also doesn't change the undefined flags. * Because IBM doesn't disclose the information for Bluelighting * CPU, we couldn't distinguish it from Cyrix's (including IBM * brand of Cyrix CPUs). */ movl $0x69727943,R(_cpu_vendor) # store vendor string movl $0x736e4978,R(_cpu_vendor+4) movl $0x64616574,R(_cpu_vendor+8) jmp 3f trycpuid: /* Use the `cpuid' instruction. */ xorl %eax,%eax .byte 0x0f,0xa2 # cpuid 0 movl %eax,R(_cpu_high) # highest capability movl %ebx,R(_cpu_vendor) # store vendor string movl %edx,R(_cpu_vendor+4) movl %ecx,R(_cpu_vendor+8) movb $0,R(_cpu_vendor+12) movl $1,%eax .byte 0x0f,0xa2 # cpuid 1 movl %eax,R(_cpu_id) # store cpu_id movl %edx,R(_cpu_feature) # store cpu_feature rorl $8,%eax # extract family type andl $15,%eax cmpl $5,%eax jae 1f /* less than Pentium; must be 486 */ movl $CPU_486,R(_cpu) jmp 3f 1: /* a Pentium? */ cmpl $5,%eax jne 2f movl $CPU_586,R(_cpu) jmp 3f 2: /* Greater than Pentium...call it a Pentium Pro */ movl $CPU_686,R(_cpu) 3: ret /********************************************************************** * * Create the first page directory and its page tables. * */ create_pagetables: testl $CPUID_PGE, R(_cpu_feature) jz 1f movl %cr4, %eax orl $CR4_PGE, %eax movl %eax, %cr4 1: /* Find end of kernel image (rounded up to a page boundary). */ movl $R(_end),%esi /* include symbols in "kernel image" if they are loaded and useful */ #ifdef DDB movl R(_bootinfo+BI_ESYMTAB),%edi testl %edi,%edi je over_symalloc movl %edi,%esi movl $KERNBASE,%edi addl %edi,R(_bootinfo+BI_SYMTAB) addl %edi,R(_bootinfo+BI_ESYMTAB) over_symalloc: #endif addl $PAGE_MASK,%esi andl $~PAGE_MASK,%esi movl %esi,R(_KERNend) /* save end of kernel */ movl %esi,R(physfree) /* next free page is at end of kernel */ /* Allocate Kernel Page Tables */ ALLOCPAGES(NKPT) movl %esi,R(_KPTphys) /* Allocate Page Table Directory */ ALLOCPAGES(1) movl %esi,R(_IdlePTD) /* Allocate UPAGES */ ALLOCPAGES(UPAGES) movl %esi,R(p0upa) addl $KERNBASE, %esi movl %esi, R(_proc0paddr) #ifdef SMP /* Allocate cpu0's private data page */ ALLOCPAGES(1) movl %esi,R(cpu0pp) addl $KERNBASE, %esi movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ /* Allocate cpu0's private page table for mapping priv page, apic, etc */ ALLOCPAGES(1) movl %esi,R(cpu0pt) addl $KERNBASE, %esi movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ #endif /* SMP */ /* Map read-only from zero to the end of the kernel text section */ xorl %eax, %eax #ifdef BDE_DEBUGGER /* If the debugger is present, actually map everything read-write. */ cmpl $0,R(_bdb_exists) jne map_read_write #endif xorl %edx,%edx + +#if !defined(SMP) testl $CPUID_PGE, R(_cpu_feature) jz 2f orl $PG_G,%edx +#endif 2: movl $R(_etext),%ecx addl $PAGE_MASK,%ecx shrl $PAGE_SHIFT,%ecx fillkptphys(%edx) /* Map read-write, data, bss and symbols */ movl $R(_etext),%eax addl $PAGE_MASK, %eax andl $~PAGE_MASK, %eax map_read_write: movl $PG_RW,%edx +#if !defined(SMP) testl $CPUID_PGE, R(_cpu_feature) jz 1f orl $PG_G,%edx +#endif 1: movl R(_KERNend),%ecx subl %eax,%ecx shrl $PAGE_SHIFT,%ecx fillkptphys(%edx) /* Map page directory. */ movl R(_IdlePTD), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map proc0's UPAGES in the physical way ... */ movl R(p0upa), %eax movl $UPAGES, %ecx fillkptphys($PG_RW) /* Map ISA hole */ movl $ISA_HOLE_START, %eax movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkptphys($PG_RW) #ifdef SMP /* Map cpu0's private page into global kmem (4K @ cpu0prvpage) */ movl R(cpu0pp), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map cpu0's private page table into global kmem FWIW */ movl R(cpu0pt), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map the private page into the private page table into private space */ movl R(cpu0pp), %eax movl $0, %ebx /* pte offset = 0 */ movl $1, %ecx /* one private page coming right up */ fillkpt(R(cpu0pt), $PG_RW) /* Map the page table page into private space */ movl R(cpu0pt), %eax movl $1, %ebx /* pte offset = 1 */ movl $1, %ecx /* one private pt coming right up */ fillkpt(R(cpu0pt), $PG_RW) /* ... and put the page table table in the pde. */ movl R(cpu0pt), %eax movl $MPPTDI, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) #endif /* SMP */ /* install a pde for temporary double map of bottom of VA */ movl R(_KPTphys), %eax xorl %ebx, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) /* install pde's for pt's */ movl R(_KPTphys), %eax movl $KPTDI, %ebx movl $NKPT, %ecx fillkpt(R(_IdlePTD), $PG_RW) /* install a pde recursively mapping page directory as a page table */ movl R(_IdlePTD), %eax movl $PTDPTDI, %ebx movl $1,%ecx fillkpt(R(_IdlePTD), $PG_RW) ret #ifdef BDE_DEBUGGER bdb_prepare_paging: cmpl $0,R(_bdb_exists) je bdb_prepare_paging_exit subl $6,%esp /* * Copy and convert debugger entries from the bootstrap gdt and idt * to the kernel gdt and idt. Everything is still in low memory. * Tracing continues to work after paging is enabled because the * low memory addresses remain valid until everything is relocated. * However, tracing through the setidt() that initializes the trace * trap will crash. */ sgdt (%esp) movl 2(%esp),%esi /* base address of bootstrap gdt */ movl $R(_gdt),%edi movl %edi,2(%esp) /* prepare to load kernel gdt */ movl $8*18/4,%ecx cld rep /* copy gdt */ movsl movl $R(_gdt),-8+2(%edi) /* adjust gdt self-ptr */ movb $0x92,-8+5(%edi) lgdt (%esp) sidt (%esp) movl 2(%esp),%esi /* base address of current idt */ movl 8+4(%esi),%eax /* convert dbg descriptor to ... */ movw 8(%esi),%ax movl %eax,R(bdb_dbg_ljmp+1) /* ... immediate offset ... */ movl 8+2(%esi),%eax movw %ax,R(bdb_dbg_ljmp+5) /* ... and selector for ljmp */ movl 24+4(%esi),%eax /* same for bpt descriptor */ movw 24(%esi),%ax movl %eax,R(bdb_bpt_ljmp+1) movl 24+2(%esi),%eax movw %ax,R(bdb_bpt_ljmp+5) movl $R(_idt),%edi movl %edi,2(%esp) /* prepare to load kernel idt */ movl $8*4/4,%ecx cld rep /* copy idt */ movsl lidt (%esp) addl $6,%esp bdb_prepare_paging_exit: ret /* Relocate debugger gdt entries and gdt and idt pointers. */ bdb_commit_paging: cmpl $0,_bdb_exists je bdb_commit_paging_exit movl $_gdt+8*9,%eax /* adjust slots 9-17 */ movl $9,%ecx reloc_gdt: movb $KERNBASE>>24,7(%eax) /* top byte of base addresses, was 0, */ addl $8,%eax /* now KERNBASE>>24 */ loop reloc_gdt subl $6,%esp sgdt (%esp) addl $KERNBASE,2(%esp) lgdt (%esp) sidt (%esp) addl $KERNBASE,2(%esp) lidt (%esp) addl $6,%esp int $3 bdb_commit_paging_exit: ret #endif /* BDE_DEBUGGER */ Index: head/sys/amd64/amd64/mp_machdep.c =================================================================== --- head/sys/amd64/amd64/mp_machdep.c (revision 27483) +++ head/sys/amd64/amd64/mp_machdep.c (revision 27484) @@ -1,1830 +1,1834 @@ /* * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.14 1997/07/13 00:42:14 smp Exp smp $ + * $Id: mp_machdep.c,v 1.27 1997/07/13 01:22:43 fsmp Exp $ */ #include "opt_smp.h" #include /* for KERNBASE */ #include #include #include #include #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include #include #include #include #include #include #include #include /** TEST_DEFAULT_CONFIG, TEST_CPUSTOP _TEST1 */ #include #include #include /* cngetc() */ #if defined(APIC_IO) #include /* setidt() */ #include /* IPIs */ #include /* IPIs */ #endif /* APIC_IO */ #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define BIOS_BASE (0xf0000) #define BIOS_SIZE (0x10000) #define BIOS_COUNT (BIOS_SIZE/4) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) #define PROCENTRY_FLAG_EN 0x01 #define PROCENTRY_FLAG_BP 0x02 #define IOAPICENTRY_FLAG_EN 0x01 /* MP Floating Pointer Structure */ typedef struct MPFPS { char signature[4]; void *pap; u_char length; u_char spec_rev; u_char checksum; u_char mpfb1; u_char mpfb2; u_char mpfb3; u_char mpfb4; u_char mpfb5; } *mpfps_t; /* MP Configuration Table Header */ typedef struct MPCTH { char signature[4]; u_short base_table_length; u_char spec_rev; u_char checksum; u_char oem_id[8]; u_char product_id[12]; void *oem_table_pointer; u_short oem_table_size; u_short entry_count; void *apic_address; u_short extended_table_length; u_char extended_table_checksum; u_char reserved; } *mpcth_t; typedef struct PROCENTRY { u_char type; u_char apic_id; u_char apic_version; u_char cpu_flags; u_long cpu_signature; u_long feature_flags; u_long reserved1; u_long reserved2; } *proc_entry_ptr; typedef struct BUSENTRY { u_char type; u_char bus_id; char bus_type[6]; } *bus_entry_ptr; typedef struct IOAPICENTRY { u_char type; u_char apic_id; u_char apic_version; u_char apic_flags; void *apic_address; } *io_apic_entry_ptr; typedef struct INTENTRY { u_char type; u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } *int_entry_ptr; /* descriptions of MP basetable entries */ typedef struct BASETABLE_ENTRY { u_char type; u_char length; char name[16]; } basetable_entry; /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MP_START_POST 0x12 #define MP_ANNOUNCE_POST 0x13 #define MPTABLE_PASS1_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define MP_ENABLE_POST 0x16 #define START_ALL_APS_POST 0x17 #define INSTALL_AP_TRAMP_POST 0x18 #define START_AP_POST 0x19 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; /** XXX FIXME: what system files declare these??? */ extern struct region_descriptor r_gdt, r_idt; int mp_ncpus; /* # of CPUs, including BSP */ int mp_naps; /* # of Applications processors */ int mp_nbusses; /* # of busses */ int mp_napics; /* # of IO APICs */ int boot_cpu_id; /* designated BSP */ vm_offset_t cpu_apic_address; vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ u_int32_t cpu_apic_versions[NCPU]; u_int32_t io_apic_versions[NAPIC]; /* * APIC ID logical/physical mapping structures. * We oversize these to simplify boot-time config. */ int cpu_num_to_apic_id[NAPICID]; int io_num_to_apic_id[NAPICID]; int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; /* Boot of AP uses this PTD */ u_int *bootPTD; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t KPTphys; /* Virtual address of per-cpu common_tss */ extern struct i386tss common_tss; /* * Local data and functions. */ static int mp_capable; static u_int boot_address; static u_int base_memory; static int picmode; /* 0: virtual wire mode, 1: PIC mode */ static mpfps_t mpfps; static int search_for_sig(u_int32_t target, int count); static void mp_enable(u_int boot_addr); static int mptable_pass1(void); static int mptable_pass2(void); static void default_mp_table(int type); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); base_memory = basemem * 1024; /* convert to bytes */ boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ if ((base_memory - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } /* * Look for an Intel MP spec table (ie, SMP capable hardware). */ int mp_probe(void) { int x; u_long segment; u_int32_t target; POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { /* search first 1K of EBDA */ target = (u_int32_t) (segment << 4); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } else { /* last 1K of base memory, effective 'top of base' passed in */ target = (u_int32_t) (base_memory - 0x400); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } /* search the BIOS */ target = (u_int32_t) BIOS_BASE; if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) goto found; /* nothing found */ mpfps = (mpfps_t)0; mp_capable = 0; return 0; found: /* calculate needed resources */ mpfps = (mpfps_t)x; if (mptable_pass1()) panic("you must reconfigure your kernel"); /* flag fact that we are running multiple processors */ mp_capable = 1; return 1; } /* * Startup the SMP processors. */ void mp_start(void) { POSTCODE(MP_START_POST); /* look for MP capable motherboard */ if (mp_capable) mp_enable(boot_address); else panic("MP hardware not found!"); } /* * Print various information about the SMP system hardware and setup. */ void mp_announce(void) { int x; POSTCODE(MP_ANNOUNCE_POST); printf("FreeBSD/SMP: Multiprocessor motherboard\n"); printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); printf(", version: 0x%08x", cpu_apic_versions[0]); printf(", at 0x%08x\n", cpu_apic_address); for (x = 1; x <= mp_naps; ++x) { printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); printf(", version: 0x%08x", cpu_apic_versions[x]); printf(", at 0x%08x\n", cpu_apic_address); } #if defined(APIC_IO) for (x = 0; x < mp_napics; ++x) { printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); printf(", version: 0x%08x", io_apic_versions[x]); printf(", at 0x%08x\n", io_apic_address[x]); } #else printf(" Warning: APIC I/O disabled\n"); #endif /* APIC_IO */ } /* * AP cpu's call this to sync up protected mode. */ void init_secondary(void) { int gsel_tss, slot; r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; r_gdt.rd_base = (int) gdt; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); slot = NGDT + cpuid; gsel_tss = GSEL(slot, SEL_KPL); gdt[slot].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ PTD[0] = 0; + pmap_set_opt((unsigned *)PTD); + invltlb(); } #if defined(APIC_IO) /* * Final configuration of the BSP's local APIC: * - disable 'pic mode'. * - disable 'virtual wire mode'. * - enable NMI. */ void bsp_apic_configure(void) { u_char byte; u_int32_t temp; /* leave 'pic mode' if necessary */ if (picmode) { outb(0x22, 0x70); /* select IMCR */ byte = inb(0x23); /* current contents */ byte |= 0x01; /* mask external INTR */ outb(0x23, byte); /* disconnect 8259s/NMI */ } /* mask lint0 (the 8259 'virtual wire' connection) */ temp = lapic.lvt_lint0; temp |= APIC_LVT_M; /* set the mask */ lapic.lvt_lint0 = temp; /* setup lint1 to handle NMI */ temp = lapic.lvt_lint1; temp &= ~APIC_LVT_M; /* clear the mask */ lapic.lvt_lint1 = temp; if (bootverbose) apic_dump(); } #endif /* APIC_IO */ /******************************************************************* * local functions and data */ /* * start the SMP system */ static void mp_enable(u_int boot_addr) { int x; #if defined(APIC_IO) int apic; u_int ux; #endif /* APIC_IO */ POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); invltlb(); /* examine the MP table for needed info, uses physical addresses */ x = mptable_pass2(); *(int *)PTD = 0; invltlb(); /* can't process default configs till the CPU APIC is pmapped */ if (x) default_mp_table(x); #if defined(APIC_IO) /* fill the LOGICAL io_apic_versions table */ for (apic = 0; apic < mp_napics; ++apic) { ux = io_apic_read(apic, IOAPIC_VER); io_apic_versions[apic] = ux; } /* program each IO APIC in the system */ for (apic = 0; apic < mp_napics; ++apic) if (io_apic_setup(apic) < 0) panic("IO APIC setup failure"); /* install a 'Spurious INTerrupt' vector */ setidt(XSPURIOUSINT_OFFSET, Xspuriousint, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #if defined(TEST_CPUSTOP) /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_CPUSTOP */ #if defined(TEST_TEST1) /* install a 'Spurious INTerrupt' vector */ setidt(XTEST1_OFFSET, Xtest1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_TEST1 */ #endif /* APIC_IO */ /* start each Application Processor */ start_all_aps(boot_addr); /* * The init process might be started on a different CPU now, * and the boot CPU might not call prepare_usermode to get * cr0 correctly configured. Thus we initialize cr0 here. */ load_cr0(rcr0() | CR0_WP | CR0_AM); } /* * look for the MP spec signature */ /* string defined by the Intel MP Spec as identifying the MP table */ #define MP_SIG 0x5f504d5f /* _MP_ */ #define NEXT(X) ((X) += 4) static int search_for_sig(u_int32_t target, int count) { int x; u_int32_t *addr = (u_int32_t *) (KERNBASE + target); for (x = 0; x < count; NEXT(x)) if (addr[x] == MP_SIG) /* make array index a byte index */ return (target + (x * sizeof(u_int32_t))); return -1; } static basetable_entry basetable_entry_types[] = { {0, 20, "Processor"}, {1, 8, "Bus"}, {2, 8, "I/O APIC"}, {3, 8, "I/O INT"}, {4, 8, "Local INT"} }; typedef struct BUSDATA { u_char bus_id; enum busTypes bus_type; } bus_datum; typedef struct INTDATA { u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } io_int, local_int; typedef struct BUSTYPENAME { u_char type; char name[7]; } bus_type_name; static bus_type_name bus_type_table[] = { {CBUS, "CBUS"}, {CBUSII, "CBUSII"}, {EISA, "EISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {ISA, "ISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {PCI, "PCI"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {XPRESS, "XPRESS"}, {UNKNOWN_BUSTYPE, "---"} }; /* from MP spec v1.4, table 5-1 */ static int default_data[7][5] = { /* nbus, id0, type0, id1, type1 */ {1, 0, ISA, 255, 255}, {1, 0, EISA, 255, 255}, {1, 0, EISA, 255, 255}, {0, 255, 255, 255, 255},/* MCA not supported */ {2, 0, ISA, 1, PCI}, {2, 0, EISA, 1, PCI}, {0, 255, 255, 255, 255} /* MCA not supported */ }; /* the bus data */ bus_datum bus_data[NBUS]; /* the IO INT data, one entry per possible APIC INTerrupt */ io_int io_apic_ints[NINTR]; static int nintrs; static void fix_mp_table __P((void)); static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); /* * 1st pass on motherboard's Intel MP specification table. * * initializes: * mp_ncpus = 1 * * determines: * cpu_apic_address (common to all CPUs) * io_apic_address[N] * mp_naps * mp_nbusses * mp_napics * nintrs */ static int mptable_pass1(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int mustpanic; POSTCODE(MPTABLE_PASS1_POST); mustpanic = 0; /* clear various tables */ for (x = 0; x < NAPICID; ++x) { io_apic_address[x] = ~0; /* IO APIC address table */ } /* init everything to empty */ mp_naps = 0; mp_nbusses = 0; mp_napics = 0; nintrs = 0; /* check for use of 'default' configuration */ if (mpfps->mpfb1 != 0) { /* use default addresses */ cpu_apic_address = DEFAULT_APIC_BASE; io_apic_address[0] = DEFAULT_IO_APIC_BASE; /* fill in with defaults */ mp_naps = 2; /* includes BSP */ mp_nbusses = default_data[mpfps->mpfb1 - 1][0]; #if defined(APIC_IO) mp_napics = 1; nintrs = 16; #endif /* APIC_IO */ } else { if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); cpu_apic_address = (vm_offset_t) cth->apic_address; /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; while (count--) { switch (type = *(u_char *) position) { case 0: /* processor_entry */ if (((proc_entry_ptr)position)->cpu_flags & PROCENTRY_FLAG_EN) ++mp_naps; break; case 1: /* bus_entry */ ++mp_nbusses; break; case 2: /* io_apic_entry */ if (((io_apic_entry_ptr)position)->apic_flags & IOAPICENTRY_FLAG_EN) io_apic_address[mp_napics++] = (vm_offset_t)((io_apic_entry_ptr) position)->apic_address; break; case 3: /* int_entry */ ++nintrs; break; case 4: /* int_entry */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char*)position += basetable_entry_types[type].length; } } /* qualify the numbers */ if (mp_naps > NCPU) printf("Warning: only using %d of %d available CPUs!\n", NCPU, mp_naps); #if 0 /** XXX we consider this legal now (but should we?) */ mustpanic = 1; #endif if (mp_nbusses > NBUS) { printf("found %d busses, increase NBUS\n", mp_nbusses); mustpanic = 1; } if (mp_napics > NAPIC) { printf("found %d apics, increase NAPIC\n", mp_napics); mustpanic = 1; } if (nintrs > NINTR) { printf("found %d intrs, increase NINTR\n", nintrs); mustpanic = 1; } /* * Count the BSP. * This is also used as a counter while starting the APs. */ mp_ncpus = 1; --mp_naps; /* subtract the BSP */ return mustpanic; } /* * 2nd pass on motherboard's Intel MP specification table. * * sets: * boot_cpu_id * ID_TO_IO(N), phy APIC ID to log CPU/IO table * CPU_TO_ID(N), logical CPU to APIC ID table * IO_TO_ID(N), logical IO to APIC ID table * bus_data[N] * io_apic_ints[N] */ static int mptable_pass2(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int apic, bus, cpu, intr; POSTCODE(MPTABLE_PASS2_POST); /* clear various tables */ for (x = 0; x < NAPICID; ++x) { ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ } /* clear bus data table */ for (x = 0; x < NBUS; ++x) bus_data[x].bus_id = 0xff; /* clear IO APIC INT table */ for (x = 0; x < NINTR; ++x) io_apic_ints[x].int_type = 0xff; /* setup the cpu/apic mapping arrays */ boot_cpu_id = -1; /* record whether PIC or virtual-wire mode */ picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; /* check for use of 'default' configuration */ #if defined(TEST_DEFAULT_CONFIG) return TEST_DEFAULT_CONFIG; #else if (mpfps->mpfb1 != 0) return mpfps->mpfb1; /* return default configuration type */ #endif /* TEST_DEFAULT_CONFIG */ if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; apic = bus = intr = 0; cpu = 1; /* pre-count the BSP */ while (count--) { switch (type = *(u_char *) position) { case 0: if (processor_entry(position, cpu)) ++cpu; break; case 1: if (bus_entry(position, bus)) ++bus; break; case 2: if (io_apic_entry(position, apic)) ++apic; break; case 3: if (int_entry(position, intr)) ++intr; break; case 4: /* int_entry(position); */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char *) position += basetable_entry_types[type].length; } if (boot_cpu_id == -1) panic("NO BSP found!"); /* post scan cleanup */ fix_mp_table(); /* report fact that its NOT a default configuration */ return 0; } /* * parse an Intel MP specification table */ static void fix_mp_table(void) { int x; int id; int bus_0; int bus_pci; int num_pci_bus; /* * Fix mis-numbering of the PCI bus and its INT entries if the BIOS * did it wrong. The MP spec says that when more than 1 PCI bus * exists the BIOS must begin with bus entries for the PCI bus and use * actual PCI bus numbering. This implies that when only 1 PCI bus * exists the BIOS can choose to ignore this ordering, and indeed many * MP motherboards do ignore it. This causes a problem when the PCI * sub-system makes requests of the MP sub-system based on PCI bus * numbers. So here we look for the situation and renumber the * busses and associated INTs in an effort to "make it right". */ /* find bus 0, PCI bus, count the number of PCI busses */ for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_id == 0) { bus_0 = x; } if (bus_data[x].bus_type == PCI) { ++num_pci_bus; bus_pci = x; } } /* * bus_0 == slot of bus with ID of 0 * bus_pci == slot of last PCI bus encountered */ /* check the 1 PCI bus case for sanity */ if (num_pci_bus == 1) { /* if it is number 0 all is well */ if (bus_data[bus_pci].bus_id == 0) return; /* mis-numbered, swap with whichever bus uses slot 0 */ /* swap the bus entry types */ bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; /* swap each relavant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { io_apic_ints[x].src_bus_id = 0; } else if (io_apic_ints[x].src_bus_id == 0) { io_apic_ints[x].src_bus_id = id; } } } /* sanity check if more than 1 PCI bus */ else if (num_pci_bus > 1) { for (x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_type != PCI) continue; if (bus_data[x].bus_id >= num_pci_bus) panic("bad PCI bus numbering"); } } } static int processor_entry(proc_entry_ptr entry, int cpu) { /* check for usability */ if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) return 0; /* check for BSP flag */ if (entry->cpu_flags & PROCENTRY_FLAG_BP) { boot_cpu_id = entry->apic_id; CPU_TO_ID(0) = entry->apic_id; ID_TO_CPU(entry->apic_id) = 0; return 0; /* its already been counted */ } /* add another AP to list, if less than max number of CPUs */ else { CPU_TO_ID(cpu) = entry->apic_id; ID_TO_CPU(entry->apic_id) = cpu; return 1; } } static int bus_entry(bus_entry_ptr entry, int bus) { int x; char c, name[8]; /* encode the name into an index */ for (x = 0; x < 6; ++x) { if ((c = entry->bus_type[x]) == ' ') break; name[x] = c; } name[x] = '\0'; if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) panic("unknown bus type: '%s'", name); bus_data[bus].bus_id = entry->bus_id; bus_data[bus].bus_type = x; return 1; } static int io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; IO_TO_ID(apic) = entry->apic_id; ID_TO_IO(entry->apic_id) = apic; return 1; } static int lookup_bus_type(char *name) { int x; for (x = 0; x < MAX_BUSTYPE; ++x) if (strcmp(bus_type_table[x].name, name) == 0) return bus_type_table[x].type; return UNKNOWN_BUSTYPE; } static int int_entry(int_entry_ptr entry, int intr) { io_apic_ints[intr].int_type = entry->int_type; io_apic_ints[intr].int_flags = entry->int_flags; io_apic_ints[intr].src_bus_id = entry->src_bus_id; io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; } static int apic_int_is_bus_type(int intr, int bus_type) { int bus; for (bus = 0; bus < mp_nbusses; ++bus) if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) && ((int) bus_data[bus].bus_type == bus_type)) return 1; return 0; } /* * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { int isa_irq; int apic_pin; #if defined(SKIP_IRQ15_REDIRECT) if (isa_mask == (1 << 15)) { printf("skipping ISA IRQ15 redirect\n"); return isa_mask; } #endif /* SKIP_IRQ15_REDIRECT */ isa_irq = ffs(isa_mask); /* find its bit position */ if (isa_irq == 0) /* doesn't exist */ return 0; --isa_irq; /* make it zero based */ apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ if (apic_pin == -1) return 0; return (1 << apic_pin); /* convert pin# to a mask */ } /* * Determine which APIC pin an ISA/EISA INT is attached to. */ #define INTTYPE(I) (io_apic_ints[(I)].int_type) #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) int isa_apic_pin(int isa_irq) { int intr; #if defined(SMP_TIMER_NC) if (isa_irq == 0) return -1; #endif /* SMP_TIMER_NC */ for (intr = 0; intr < nintrs; ++intr) { /* check each record */ if (INTTYPE(intr) == 0) { /* standard INT */ if (SRCBUSIRQ(intr) == isa_irq) { if (apic_int_is_bus_type(intr, ISA) || apic_int_is_bus_type(intr, EISA)) return INTPIN(intr); /* found */ } } } return -1; /* NOT found */ } #undef SRCBUSIRQ /* * Determine which APIC pin a PCI INT is attached to. */ #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) int pci_apic_pin(int pciBus, int pciDevice, int pciInt) { int intr; --pciInt; /* zero based */ for (intr = 0; intr < nintrs; ++intr) /* check each record */ if ((INTTYPE(intr) == 0) /* standard INT */ && (SRCBUSID(intr) == pciBus) && (SRCBUSDEVICE(intr) == pciDevice) && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ if (apic_int_is_bus_type(intr, PCI)) return INTPIN(intr); /* exact match */ return -1; /* NOT found */ } #undef SRCBUSLINE #undef SRCBUSDEVICE #undef SRCBUSID #undef INTPIN #undef INTTYPE /* * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. * * XXX FIXME: * Exactly what this means is unclear at this point. It is a solution * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard * could route any of the ISA INTs to upper (>15) IRQ values. But most would * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an * option. */ int undirect_isa_irq(int rirq) { #if defined(READY) printf("Freeing redirected ISA irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); return 0; #endif /* READY */ } /* * Reprogram the MB chipset to NOT redirect a PCI INTerrupt */ int undirect_pci_irq(int rirq) { #if defined(READY) if (bootverbose) printf("Freeing redirected PCI irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else if (bootverbose) printf("Freeing (NOT implemented) redirected PCI irq %d.\n", rirq); return 0; #endif /* READY */ } /* * given a bus ID, return: * the bus type if found * -1 if NOT found */ int apic_bus_type(int id) { int x; for (x = 0; x < mp_nbusses; ++x) if (bus_data[x].bus_id == id) return bus_data[x].bus_type; return -1; } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus ID if found * -1 if NOT found */ int apic_src_bus_id(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_id); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus IRQ if found * -1 if NOT found */ int apic_src_bus_irq(int apic, int pin) { int x; for (x = 0; x < nintrs; x++) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_irq); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated INTerrupt type if found * -1 if NOT found */ int apic_int_type(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_type); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated trigger mode if found * -1 if NOT found */ int apic_trigger(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return ((io_apic_ints[x].int_flags >> 2) & 0x03); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated 'active' level if found * -1 if NOT found */ int apic_polarity(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_flags & 0x03); return -1; /* NOT found */ } /* * set data according to MP defaults * FIXME: probably not complete yet... */ static void default_mp_table(int type) { int ap_cpu_id; #if defined(APIC_IO) u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ #if 0 printf(" MP default config type: %d\n", type); switch (type) { case 1: printf(" bus: ISA, APIC: 82489DX\n"); break; case 2: printf(" bus: EISA, APIC: 82489DX\n"); break; case 3: printf(" bus: EISA, APIC: 82489DX\n"); break; case 4: printf(" bus: MCA, APIC: 82489DX\n"); break; case 5: printf(" bus: ISA+PCI, APIC: Integrated\n"); break; case 6: printf(" bus: EISA+PCI, APIC: Integrated\n"); break; case 7: printf(" bus: MCA+PCI, APIC: Integrated\n"); break; default: printf(" future type\n"); break; /* NOTREACHED */ } #endif /* 0 */ boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; /* BSP */ CPU_TO_ID(0) = boot_cpu_id; ID_TO_CPU(boot_cpu_id) = 0; /* one and only AP */ CPU_TO_ID(1) = ap_cpu_id; ID_TO_CPU(ap_cpu_id) = 1; #if defined(APIC_IO) /* one and only IO APIC */ io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; /* * sanity check, refer to MP spec section 3.6.6, last paragraph * necessary as some hardware isn't properly setting up the IO APIC */ #if defined(REALLY_ANAL_IOAPICID_VALUE) if (io_apic_id != 2) { #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ux &= ~APIC_ID_MASK; /* clear the ID field */ ux |= 0x02000000; /* set it to '2' */ io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ if ((ux & APIC_ID_MASK) != 0x02000000) panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; ID_TO_IO(io_apic_id) = 0; #endif /* APIC_IO */ /* fill out bus entries */ switch (type) { case 1: case 2: case 3: case 5: case 6: bus_data[0].bus_id = default_data[type - 1][1]; bus_data[0].bus_type = default_data[type - 1][2]; bus_data[1].bus_id = default_data[type - 1][3]; bus_data[1].bus_type = default_data[type - 1][4]; break; /* case 4: case 7: MCA NOT supported */ default: /* illegal/reserved */ panic("BAD default MP config: %d", type); /* NOTREACHED */ } #if defined(APIC_IO) /* general cases from MP v1.4, table 5-2 */ for (pin = 0; pin < 16; ++pin) { io_apic_ints[pin].int_type = 0; io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ io_apic_ints[pin].src_bus_id = 0; io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ io_apic_ints[pin].dst_apic_id = io_apic_id; io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ } /* special cases from MP v1.4, table 5-2 */ if (type == 2) { io_apic_ints[2].int_type = 0xff; /* N/C */ io_apic_ints[13].int_type = 0xff; /* N/C */ #if !defined(APIC_MIXED_MODE) /** FIXME: ??? */ panic("sorry, can't support type 2 default yet"); #endif /* APIC_MIXED_MODE */ } else io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ if (type == 7) io_apic_ints[0].int_type = 0xff; /* N/C */ else io_apic_ints[0].int_type = 3; /* vectored 8259 */ #endif /* APIC_IO */ } /* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { int x, i; u_char mpbiosreason; u_long mpbioswarmvec; pd_entry_t newptd; pt_entry_t newpt; int *newpp; POSTCODE(START_ALL_APS_POST); /** * NOTE: this needs further thought: * where does it get released? * should it be set to empy? * * get the initial mp_lock with a count of 1 for the BSP */ mp_lock = 1; /* this uses a LOGICAL cpu ID, ie BSP == 0 */ /* initialize BSP's local APIC */ apic_initialize(); /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* record BSP in CPU map */ all_cpus = 1; /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* HACK HACK HACK !!! */ /* alloc new page table directory */ newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* clone currently active one (ie: IdlePTD) */ bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ /* set up 0 -> 4MB P==V mapping for AP boot */ newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); /* store PTD for this AP */ bootPTD = (pd_entry_t)vtophys(newptd); /* alloc new page table page */ newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* set the new PTD's private page to point there */ newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt); /* install self referential entry */ newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd); /* get a new private data page */ newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ newpt[0] = PG_V | PG_RW | vtophys(newpp); /* wire the ptp into itself for access */ newpt[1] = PG_V | PG_RW | vtophys(newpt); /* and the local apic */ newpt[2] = SMP_prvpt[2]; /* and the IO apic mapping[s] */ for (i = 16; i < 32; i++) newpt[i] = SMP_prvpt[i]; /* prime data page for it to use */ newpp[0] = x; /* cpuid */ newpp[1] = 0; /* curproc */ newpp[2] = 0; /* curpcb */ newpp[3] = 0; /* npxproc */ newpp[4] = 0; /* runtime.tv_sec */ newpp[5] = 0; /* runtime.tv_usec */ newpp[6] = x << 24; /* cpu_lockid */ /* XXX NOTE: ABANDON bootPTD for now!!!! */ /* END REVOLTING HACKERY */ /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ /* record its version info */ cpu_apic_versions[x] = cpu_apic_versions[0]; all_cpus |= (1 << x); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ other_cpus = all_cpus & ~(1 << cpuid); /* fill in our (BSP) APIC version */ cpu_apic_versions[0] = lapic.version; /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); + + pmap_set_opt_bsp(); /* number of APs actually started */ return mp_ncpus - 1; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(u_int boot_addr) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_addr + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_addr + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; } /* * this function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It ain't pretty, * but it seems to work. */ static int start_ap(int logical_cpu, u_int boot_addr) { int physical_cpu; int vector; int cpus; u_long icr_lo, icr_hi; POSTCODE(START_AP_POST); /* get the PHYSICAL APIC ID# */ physical_cpu = CPU_TO_ID(logical_cpu); /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_ncpus; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* setup the address for the target AP */ icr_hi = lapic.icr_hi & ~APIC_ID_MASK; icr_hi |= (physical_cpu << 24); lapic.icr_hi = icr_hi; /* do an INIT IPI: assert RESET */ icr_lo = lapic.icr_lo & 0xfff00000; lapic.icr_lo = icr_lo | 0x0000c500; /* wait for pending status end */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* do an INIT IPI: deassert RESET */ lapic.icr_lo = icr_lo | 0x00008500; /* wait for pending status end */ u_sleep(10000); /* wait ~10mS */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* wait for it to start */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) if (mp_ncpus > cpus) return 1; /* return SUCCESS */ return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's * * XXX: Needs to handshake and wait for completion before proceding. */ void smp_invltlb(void) { #if defined(APIC_IO) if (smp_active && invltlb_ok) all_but_self_ipi(XINVLTLB_OFFSET); #endif /* APIC_IO */ } void invlpg(u_int addr) { __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); /* send a message to the other CPUs */ smp_invltlb(); } void invltlb(void) { u_long temp; /* * This should be implemented as load_cr3(rcr3()) when load_cr3() is * inlined. */ __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); /* send a message to the other CPUs */ smp_invltlb(); } #if defined(TEST_CPUSTOP) #if defined(DEBUG_CPUSTOP) void db_printf __P((const char *fmt, ...)); #endif /* DEBUG_CPUSTOP */ /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int stop_cpus( u_int map ) { if (!smp_active) return 0; /* send IPI to all CPUs in map */ #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d stopping CPUs: 0x%08x\n", cpuid, map); #endif /* DEBUG_CPUSTOP */ stopped_cpus = 0; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); #if defined(DEBUG_CPUSTOP) db_printf(" spin\n"); #endif /* DEBUG_CPUSTOP */ while (stopped_cpus != map) /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" spun\nstopped\n"); #endif /* DEBUG_CPUSTOP */ return 1; } /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus( u_int map ) { if (!smp_active) return 0; #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d restarting CPUs: 0x%08x (0x%08x)\n", cpuid, map, stopped_cpus); #endif /* DEBUG_CPUSTOP */ started_cpus = map; /* signal other cpus to restart */ while (started_cpus) /* wait for each to clear its bit */ /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" restarted\n"); #endif /* DEBUG_CPUSTOP */ return 1; } #endif /** TEST_CPUSTOP */ Index: head/sys/amd64/amd64/mptable.c =================================================================== --- head/sys/amd64/amd64/mptable.c (revision 27483) +++ head/sys/amd64/amd64/mptable.c (revision 27484) @@ -1,1830 +1,1834 @@ /* * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.14 1997/07/13 00:42:14 smp Exp smp $ + * $Id: mp_machdep.c,v 1.27 1997/07/13 01:22:43 fsmp Exp $ */ #include "opt_smp.h" #include /* for KERNBASE */ #include #include #include #include #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include #include #include #include #include #include #include #include /** TEST_DEFAULT_CONFIG, TEST_CPUSTOP _TEST1 */ #include #include #include /* cngetc() */ #if defined(APIC_IO) #include /* setidt() */ #include /* IPIs */ #include /* IPIs */ #endif /* APIC_IO */ #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define BIOS_BASE (0xf0000) #define BIOS_SIZE (0x10000) #define BIOS_COUNT (BIOS_SIZE/4) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) #define PROCENTRY_FLAG_EN 0x01 #define PROCENTRY_FLAG_BP 0x02 #define IOAPICENTRY_FLAG_EN 0x01 /* MP Floating Pointer Structure */ typedef struct MPFPS { char signature[4]; void *pap; u_char length; u_char spec_rev; u_char checksum; u_char mpfb1; u_char mpfb2; u_char mpfb3; u_char mpfb4; u_char mpfb5; } *mpfps_t; /* MP Configuration Table Header */ typedef struct MPCTH { char signature[4]; u_short base_table_length; u_char spec_rev; u_char checksum; u_char oem_id[8]; u_char product_id[12]; void *oem_table_pointer; u_short oem_table_size; u_short entry_count; void *apic_address; u_short extended_table_length; u_char extended_table_checksum; u_char reserved; } *mpcth_t; typedef struct PROCENTRY { u_char type; u_char apic_id; u_char apic_version; u_char cpu_flags; u_long cpu_signature; u_long feature_flags; u_long reserved1; u_long reserved2; } *proc_entry_ptr; typedef struct BUSENTRY { u_char type; u_char bus_id; char bus_type[6]; } *bus_entry_ptr; typedef struct IOAPICENTRY { u_char type; u_char apic_id; u_char apic_version; u_char apic_flags; void *apic_address; } *io_apic_entry_ptr; typedef struct INTENTRY { u_char type; u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } *int_entry_ptr; /* descriptions of MP basetable entries */ typedef struct BASETABLE_ENTRY { u_char type; u_char length; char name[16]; } basetable_entry; /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MP_START_POST 0x12 #define MP_ANNOUNCE_POST 0x13 #define MPTABLE_PASS1_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define MP_ENABLE_POST 0x16 #define START_ALL_APS_POST 0x17 #define INSTALL_AP_TRAMP_POST 0x18 #define START_AP_POST 0x19 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; /** XXX FIXME: what system files declare these??? */ extern struct region_descriptor r_gdt, r_idt; int mp_ncpus; /* # of CPUs, including BSP */ int mp_naps; /* # of Applications processors */ int mp_nbusses; /* # of busses */ int mp_napics; /* # of IO APICs */ int boot_cpu_id; /* designated BSP */ vm_offset_t cpu_apic_address; vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ u_int32_t cpu_apic_versions[NCPU]; u_int32_t io_apic_versions[NAPIC]; /* * APIC ID logical/physical mapping structures. * We oversize these to simplify boot-time config. */ int cpu_num_to_apic_id[NAPICID]; int io_num_to_apic_id[NAPICID]; int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; /* Boot of AP uses this PTD */ u_int *bootPTD; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t KPTphys; /* Virtual address of per-cpu common_tss */ extern struct i386tss common_tss; /* * Local data and functions. */ static int mp_capable; static u_int boot_address; static u_int base_memory; static int picmode; /* 0: virtual wire mode, 1: PIC mode */ static mpfps_t mpfps; static int search_for_sig(u_int32_t target, int count); static void mp_enable(u_int boot_addr); static int mptable_pass1(void); static int mptable_pass2(void); static void default_mp_table(int type); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); base_memory = basemem * 1024; /* convert to bytes */ boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ if ((base_memory - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } /* * Look for an Intel MP spec table (ie, SMP capable hardware). */ int mp_probe(void) { int x; u_long segment; u_int32_t target; POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { /* search first 1K of EBDA */ target = (u_int32_t) (segment << 4); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } else { /* last 1K of base memory, effective 'top of base' passed in */ target = (u_int32_t) (base_memory - 0x400); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } /* search the BIOS */ target = (u_int32_t) BIOS_BASE; if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) goto found; /* nothing found */ mpfps = (mpfps_t)0; mp_capable = 0; return 0; found: /* calculate needed resources */ mpfps = (mpfps_t)x; if (mptable_pass1()) panic("you must reconfigure your kernel"); /* flag fact that we are running multiple processors */ mp_capable = 1; return 1; } /* * Startup the SMP processors. */ void mp_start(void) { POSTCODE(MP_START_POST); /* look for MP capable motherboard */ if (mp_capable) mp_enable(boot_address); else panic("MP hardware not found!"); } /* * Print various information about the SMP system hardware and setup. */ void mp_announce(void) { int x; POSTCODE(MP_ANNOUNCE_POST); printf("FreeBSD/SMP: Multiprocessor motherboard\n"); printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); printf(", version: 0x%08x", cpu_apic_versions[0]); printf(", at 0x%08x\n", cpu_apic_address); for (x = 1; x <= mp_naps; ++x) { printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); printf(", version: 0x%08x", cpu_apic_versions[x]); printf(", at 0x%08x\n", cpu_apic_address); } #if defined(APIC_IO) for (x = 0; x < mp_napics; ++x) { printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); printf(", version: 0x%08x", io_apic_versions[x]); printf(", at 0x%08x\n", io_apic_address[x]); } #else printf(" Warning: APIC I/O disabled\n"); #endif /* APIC_IO */ } /* * AP cpu's call this to sync up protected mode. */ void init_secondary(void) { int gsel_tss, slot; r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; r_gdt.rd_base = (int) gdt; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); slot = NGDT + cpuid; gsel_tss = GSEL(slot, SEL_KPL); gdt[slot].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ PTD[0] = 0; + pmap_set_opt((unsigned *)PTD); + invltlb(); } #if defined(APIC_IO) /* * Final configuration of the BSP's local APIC: * - disable 'pic mode'. * - disable 'virtual wire mode'. * - enable NMI. */ void bsp_apic_configure(void) { u_char byte; u_int32_t temp; /* leave 'pic mode' if necessary */ if (picmode) { outb(0x22, 0x70); /* select IMCR */ byte = inb(0x23); /* current contents */ byte |= 0x01; /* mask external INTR */ outb(0x23, byte); /* disconnect 8259s/NMI */ } /* mask lint0 (the 8259 'virtual wire' connection) */ temp = lapic.lvt_lint0; temp |= APIC_LVT_M; /* set the mask */ lapic.lvt_lint0 = temp; /* setup lint1 to handle NMI */ temp = lapic.lvt_lint1; temp &= ~APIC_LVT_M; /* clear the mask */ lapic.lvt_lint1 = temp; if (bootverbose) apic_dump(); } #endif /* APIC_IO */ /******************************************************************* * local functions and data */ /* * start the SMP system */ static void mp_enable(u_int boot_addr) { int x; #if defined(APIC_IO) int apic; u_int ux; #endif /* APIC_IO */ POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); invltlb(); /* examine the MP table for needed info, uses physical addresses */ x = mptable_pass2(); *(int *)PTD = 0; invltlb(); /* can't process default configs till the CPU APIC is pmapped */ if (x) default_mp_table(x); #if defined(APIC_IO) /* fill the LOGICAL io_apic_versions table */ for (apic = 0; apic < mp_napics; ++apic) { ux = io_apic_read(apic, IOAPIC_VER); io_apic_versions[apic] = ux; } /* program each IO APIC in the system */ for (apic = 0; apic < mp_napics; ++apic) if (io_apic_setup(apic) < 0) panic("IO APIC setup failure"); /* install a 'Spurious INTerrupt' vector */ setidt(XSPURIOUSINT_OFFSET, Xspuriousint, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #if defined(TEST_CPUSTOP) /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_CPUSTOP */ #if defined(TEST_TEST1) /* install a 'Spurious INTerrupt' vector */ setidt(XTEST1_OFFSET, Xtest1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_TEST1 */ #endif /* APIC_IO */ /* start each Application Processor */ start_all_aps(boot_addr); /* * The init process might be started on a different CPU now, * and the boot CPU might not call prepare_usermode to get * cr0 correctly configured. Thus we initialize cr0 here. */ load_cr0(rcr0() | CR0_WP | CR0_AM); } /* * look for the MP spec signature */ /* string defined by the Intel MP Spec as identifying the MP table */ #define MP_SIG 0x5f504d5f /* _MP_ */ #define NEXT(X) ((X) += 4) static int search_for_sig(u_int32_t target, int count) { int x; u_int32_t *addr = (u_int32_t *) (KERNBASE + target); for (x = 0; x < count; NEXT(x)) if (addr[x] == MP_SIG) /* make array index a byte index */ return (target + (x * sizeof(u_int32_t))); return -1; } static basetable_entry basetable_entry_types[] = { {0, 20, "Processor"}, {1, 8, "Bus"}, {2, 8, "I/O APIC"}, {3, 8, "I/O INT"}, {4, 8, "Local INT"} }; typedef struct BUSDATA { u_char bus_id; enum busTypes bus_type; } bus_datum; typedef struct INTDATA { u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } io_int, local_int; typedef struct BUSTYPENAME { u_char type; char name[7]; } bus_type_name; static bus_type_name bus_type_table[] = { {CBUS, "CBUS"}, {CBUSII, "CBUSII"}, {EISA, "EISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {ISA, "ISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {PCI, "PCI"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {XPRESS, "XPRESS"}, {UNKNOWN_BUSTYPE, "---"} }; /* from MP spec v1.4, table 5-1 */ static int default_data[7][5] = { /* nbus, id0, type0, id1, type1 */ {1, 0, ISA, 255, 255}, {1, 0, EISA, 255, 255}, {1, 0, EISA, 255, 255}, {0, 255, 255, 255, 255},/* MCA not supported */ {2, 0, ISA, 1, PCI}, {2, 0, EISA, 1, PCI}, {0, 255, 255, 255, 255} /* MCA not supported */ }; /* the bus data */ bus_datum bus_data[NBUS]; /* the IO INT data, one entry per possible APIC INTerrupt */ io_int io_apic_ints[NINTR]; static int nintrs; static void fix_mp_table __P((void)); static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); /* * 1st pass on motherboard's Intel MP specification table. * * initializes: * mp_ncpus = 1 * * determines: * cpu_apic_address (common to all CPUs) * io_apic_address[N] * mp_naps * mp_nbusses * mp_napics * nintrs */ static int mptable_pass1(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int mustpanic; POSTCODE(MPTABLE_PASS1_POST); mustpanic = 0; /* clear various tables */ for (x = 0; x < NAPICID; ++x) { io_apic_address[x] = ~0; /* IO APIC address table */ } /* init everything to empty */ mp_naps = 0; mp_nbusses = 0; mp_napics = 0; nintrs = 0; /* check for use of 'default' configuration */ if (mpfps->mpfb1 != 0) { /* use default addresses */ cpu_apic_address = DEFAULT_APIC_BASE; io_apic_address[0] = DEFAULT_IO_APIC_BASE; /* fill in with defaults */ mp_naps = 2; /* includes BSP */ mp_nbusses = default_data[mpfps->mpfb1 - 1][0]; #if defined(APIC_IO) mp_napics = 1; nintrs = 16; #endif /* APIC_IO */ } else { if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); cpu_apic_address = (vm_offset_t) cth->apic_address; /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; while (count--) { switch (type = *(u_char *) position) { case 0: /* processor_entry */ if (((proc_entry_ptr)position)->cpu_flags & PROCENTRY_FLAG_EN) ++mp_naps; break; case 1: /* bus_entry */ ++mp_nbusses; break; case 2: /* io_apic_entry */ if (((io_apic_entry_ptr)position)->apic_flags & IOAPICENTRY_FLAG_EN) io_apic_address[mp_napics++] = (vm_offset_t)((io_apic_entry_ptr) position)->apic_address; break; case 3: /* int_entry */ ++nintrs; break; case 4: /* int_entry */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char*)position += basetable_entry_types[type].length; } } /* qualify the numbers */ if (mp_naps > NCPU) printf("Warning: only using %d of %d available CPUs!\n", NCPU, mp_naps); #if 0 /** XXX we consider this legal now (but should we?) */ mustpanic = 1; #endif if (mp_nbusses > NBUS) { printf("found %d busses, increase NBUS\n", mp_nbusses); mustpanic = 1; } if (mp_napics > NAPIC) { printf("found %d apics, increase NAPIC\n", mp_napics); mustpanic = 1; } if (nintrs > NINTR) { printf("found %d intrs, increase NINTR\n", nintrs); mustpanic = 1; } /* * Count the BSP. * This is also used as a counter while starting the APs. */ mp_ncpus = 1; --mp_naps; /* subtract the BSP */ return mustpanic; } /* * 2nd pass on motherboard's Intel MP specification table. * * sets: * boot_cpu_id * ID_TO_IO(N), phy APIC ID to log CPU/IO table * CPU_TO_ID(N), logical CPU to APIC ID table * IO_TO_ID(N), logical IO to APIC ID table * bus_data[N] * io_apic_ints[N] */ static int mptable_pass2(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int apic, bus, cpu, intr; POSTCODE(MPTABLE_PASS2_POST); /* clear various tables */ for (x = 0; x < NAPICID; ++x) { ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ } /* clear bus data table */ for (x = 0; x < NBUS; ++x) bus_data[x].bus_id = 0xff; /* clear IO APIC INT table */ for (x = 0; x < NINTR; ++x) io_apic_ints[x].int_type = 0xff; /* setup the cpu/apic mapping arrays */ boot_cpu_id = -1; /* record whether PIC or virtual-wire mode */ picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; /* check for use of 'default' configuration */ #if defined(TEST_DEFAULT_CONFIG) return TEST_DEFAULT_CONFIG; #else if (mpfps->mpfb1 != 0) return mpfps->mpfb1; /* return default configuration type */ #endif /* TEST_DEFAULT_CONFIG */ if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; apic = bus = intr = 0; cpu = 1; /* pre-count the BSP */ while (count--) { switch (type = *(u_char *) position) { case 0: if (processor_entry(position, cpu)) ++cpu; break; case 1: if (bus_entry(position, bus)) ++bus; break; case 2: if (io_apic_entry(position, apic)) ++apic; break; case 3: if (int_entry(position, intr)) ++intr; break; case 4: /* int_entry(position); */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char *) position += basetable_entry_types[type].length; } if (boot_cpu_id == -1) panic("NO BSP found!"); /* post scan cleanup */ fix_mp_table(); /* report fact that its NOT a default configuration */ return 0; } /* * parse an Intel MP specification table */ static void fix_mp_table(void) { int x; int id; int bus_0; int bus_pci; int num_pci_bus; /* * Fix mis-numbering of the PCI bus and its INT entries if the BIOS * did it wrong. The MP spec says that when more than 1 PCI bus * exists the BIOS must begin with bus entries for the PCI bus and use * actual PCI bus numbering. This implies that when only 1 PCI bus * exists the BIOS can choose to ignore this ordering, and indeed many * MP motherboards do ignore it. This causes a problem when the PCI * sub-system makes requests of the MP sub-system based on PCI bus * numbers. So here we look for the situation and renumber the * busses and associated INTs in an effort to "make it right". */ /* find bus 0, PCI bus, count the number of PCI busses */ for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_id == 0) { bus_0 = x; } if (bus_data[x].bus_type == PCI) { ++num_pci_bus; bus_pci = x; } } /* * bus_0 == slot of bus with ID of 0 * bus_pci == slot of last PCI bus encountered */ /* check the 1 PCI bus case for sanity */ if (num_pci_bus == 1) { /* if it is number 0 all is well */ if (bus_data[bus_pci].bus_id == 0) return; /* mis-numbered, swap with whichever bus uses slot 0 */ /* swap the bus entry types */ bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; /* swap each relavant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { io_apic_ints[x].src_bus_id = 0; } else if (io_apic_ints[x].src_bus_id == 0) { io_apic_ints[x].src_bus_id = id; } } } /* sanity check if more than 1 PCI bus */ else if (num_pci_bus > 1) { for (x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_type != PCI) continue; if (bus_data[x].bus_id >= num_pci_bus) panic("bad PCI bus numbering"); } } } static int processor_entry(proc_entry_ptr entry, int cpu) { /* check for usability */ if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) return 0; /* check for BSP flag */ if (entry->cpu_flags & PROCENTRY_FLAG_BP) { boot_cpu_id = entry->apic_id; CPU_TO_ID(0) = entry->apic_id; ID_TO_CPU(entry->apic_id) = 0; return 0; /* its already been counted */ } /* add another AP to list, if less than max number of CPUs */ else { CPU_TO_ID(cpu) = entry->apic_id; ID_TO_CPU(entry->apic_id) = cpu; return 1; } } static int bus_entry(bus_entry_ptr entry, int bus) { int x; char c, name[8]; /* encode the name into an index */ for (x = 0; x < 6; ++x) { if ((c = entry->bus_type[x]) == ' ') break; name[x] = c; } name[x] = '\0'; if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) panic("unknown bus type: '%s'", name); bus_data[bus].bus_id = entry->bus_id; bus_data[bus].bus_type = x; return 1; } static int io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; IO_TO_ID(apic) = entry->apic_id; ID_TO_IO(entry->apic_id) = apic; return 1; } static int lookup_bus_type(char *name) { int x; for (x = 0; x < MAX_BUSTYPE; ++x) if (strcmp(bus_type_table[x].name, name) == 0) return bus_type_table[x].type; return UNKNOWN_BUSTYPE; } static int int_entry(int_entry_ptr entry, int intr) { io_apic_ints[intr].int_type = entry->int_type; io_apic_ints[intr].int_flags = entry->int_flags; io_apic_ints[intr].src_bus_id = entry->src_bus_id; io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; } static int apic_int_is_bus_type(int intr, int bus_type) { int bus; for (bus = 0; bus < mp_nbusses; ++bus) if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) && ((int) bus_data[bus].bus_type == bus_type)) return 1; return 0; } /* * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { int isa_irq; int apic_pin; #if defined(SKIP_IRQ15_REDIRECT) if (isa_mask == (1 << 15)) { printf("skipping ISA IRQ15 redirect\n"); return isa_mask; } #endif /* SKIP_IRQ15_REDIRECT */ isa_irq = ffs(isa_mask); /* find its bit position */ if (isa_irq == 0) /* doesn't exist */ return 0; --isa_irq; /* make it zero based */ apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ if (apic_pin == -1) return 0; return (1 << apic_pin); /* convert pin# to a mask */ } /* * Determine which APIC pin an ISA/EISA INT is attached to. */ #define INTTYPE(I) (io_apic_ints[(I)].int_type) #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) int isa_apic_pin(int isa_irq) { int intr; #if defined(SMP_TIMER_NC) if (isa_irq == 0) return -1; #endif /* SMP_TIMER_NC */ for (intr = 0; intr < nintrs; ++intr) { /* check each record */ if (INTTYPE(intr) == 0) { /* standard INT */ if (SRCBUSIRQ(intr) == isa_irq) { if (apic_int_is_bus_type(intr, ISA) || apic_int_is_bus_type(intr, EISA)) return INTPIN(intr); /* found */ } } } return -1; /* NOT found */ } #undef SRCBUSIRQ /* * Determine which APIC pin a PCI INT is attached to. */ #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) int pci_apic_pin(int pciBus, int pciDevice, int pciInt) { int intr; --pciInt; /* zero based */ for (intr = 0; intr < nintrs; ++intr) /* check each record */ if ((INTTYPE(intr) == 0) /* standard INT */ && (SRCBUSID(intr) == pciBus) && (SRCBUSDEVICE(intr) == pciDevice) && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ if (apic_int_is_bus_type(intr, PCI)) return INTPIN(intr); /* exact match */ return -1; /* NOT found */ } #undef SRCBUSLINE #undef SRCBUSDEVICE #undef SRCBUSID #undef INTPIN #undef INTTYPE /* * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. * * XXX FIXME: * Exactly what this means is unclear at this point. It is a solution * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard * could route any of the ISA INTs to upper (>15) IRQ values. But most would * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an * option. */ int undirect_isa_irq(int rirq) { #if defined(READY) printf("Freeing redirected ISA irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); return 0; #endif /* READY */ } /* * Reprogram the MB chipset to NOT redirect a PCI INTerrupt */ int undirect_pci_irq(int rirq) { #if defined(READY) if (bootverbose) printf("Freeing redirected PCI irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else if (bootverbose) printf("Freeing (NOT implemented) redirected PCI irq %d.\n", rirq); return 0; #endif /* READY */ } /* * given a bus ID, return: * the bus type if found * -1 if NOT found */ int apic_bus_type(int id) { int x; for (x = 0; x < mp_nbusses; ++x) if (bus_data[x].bus_id == id) return bus_data[x].bus_type; return -1; } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus ID if found * -1 if NOT found */ int apic_src_bus_id(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_id); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus IRQ if found * -1 if NOT found */ int apic_src_bus_irq(int apic, int pin) { int x; for (x = 0; x < nintrs; x++) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_irq); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated INTerrupt type if found * -1 if NOT found */ int apic_int_type(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_type); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated trigger mode if found * -1 if NOT found */ int apic_trigger(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return ((io_apic_ints[x].int_flags >> 2) & 0x03); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated 'active' level if found * -1 if NOT found */ int apic_polarity(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_flags & 0x03); return -1; /* NOT found */ } /* * set data according to MP defaults * FIXME: probably not complete yet... */ static void default_mp_table(int type) { int ap_cpu_id; #if defined(APIC_IO) u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ #if 0 printf(" MP default config type: %d\n", type); switch (type) { case 1: printf(" bus: ISA, APIC: 82489DX\n"); break; case 2: printf(" bus: EISA, APIC: 82489DX\n"); break; case 3: printf(" bus: EISA, APIC: 82489DX\n"); break; case 4: printf(" bus: MCA, APIC: 82489DX\n"); break; case 5: printf(" bus: ISA+PCI, APIC: Integrated\n"); break; case 6: printf(" bus: EISA+PCI, APIC: Integrated\n"); break; case 7: printf(" bus: MCA+PCI, APIC: Integrated\n"); break; default: printf(" future type\n"); break; /* NOTREACHED */ } #endif /* 0 */ boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; /* BSP */ CPU_TO_ID(0) = boot_cpu_id; ID_TO_CPU(boot_cpu_id) = 0; /* one and only AP */ CPU_TO_ID(1) = ap_cpu_id; ID_TO_CPU(ap_cpu_id) = 1; #if defined(APIC_IO) /* one and only IO APIC */ io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; /* * sanity check, refer to MP spec section 3.6.6, last paragraph * necessary as some hardware isn't properly setting up the IO APIC */ #if defined(REALLY_ANAL_IOAPICID_VALUE) if (io_apic_id != 2) { #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ux &= ~APIC_ID_MASK; /* clear the ID field */ ux |= 0x02000000; /* set it to '2' */ io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ if ((ux & APIC_ID_MASK) != 0x02000000) panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; ID_TO_IO(io_apic_id) = 0; #endif /* APIC_IO */ /* fill out bus entries */ switch (type) { case 1: case 2: case 3: case 5: case 6: bus_data[0].bus_id = default_data[type - 1][1]; bus_data[0].bus_type = default_data[type - 1][2]; bus_data[1].bus_id = default_data[type - 1][3]; bus_data[1].bus_type = default_data[type - 1][4]; break; /* case 4: case 7: MCA NOT supported */ default: /* illegal/reserved */ panic("BAD default MP config: %d", type); /* NOTREACHED */ } #if defined(APIC_IO) /* general cases from MP v1.4, table 5-2 */ for (pin = 0; pin < 16; ++pin) { io_apic_ints[pin].int_type = 0; io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ io_apic_ints[pin].src_bus_id = 0; io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ io_apic_ints[pin].dst_apic_id = io_apic_id; io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ } /* special cases from MP v1.4, table 5-2 */ if (type == 2) { io_apic_ints[2].int_type = 0xff; /* N/C */ io_apic_ints[13].int_type = 0xff; /* N/C */ #if !defined(APIC_MIXED_MODE) /** FIXME: ??? */ panic("sorry, can't support type 2 default yet"); #endif /* APIC_MIXED_MODE */ } else io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ if (type == 7) io_apic_ints[0].int_type = 0xff; /* N/C */ else io_apic_ints[0].int_type = 3; /* vectored 8259 */ #endif /* APIC_IO */ } /* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { int x, i; u_char mpbiosreason; u_long mpbioswarmvec; pd_entry_t newptd; pt_entry_t newpt; int *newpp; POSTCODE(START_ALL_APS_POST); /** * NOTE: this needs further thought: * where does it get released? * should it be set to empy? * * get the initial mp_lock with a count of 1 for the BSP */ mp_lock = 1; /* this uses a LOGICAL cpu ID, ie BSP == 0 */ /* initialize BSP's local APIC */ apic_initialize(); /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* record BSP in CPU map */ all_cpus = 1; /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* HACK HACK HACK !!! */ /* alloc new page table directory */ newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* clone currently active one (ie: IdlePTD) */ bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ /* set up 0 -> 4MB P==V mapping for AP boot */ newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); /* store PTD for this AP */ bootPTD = (pd_entry_t)vtophys(newptd); /* alloc new page table page */ newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* set the new PTD's private page to point there */ newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt); /* install self referential entry */ newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd); /* get a new private data page */ newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ newpt[0] = PG_V | PG_RW | vtophys(newpp); /* wire the ptp into itself for access */ newpt[1] = PG_V | PG_RW | vtophys(newpt); /* and the local apic */ newpt[2] = SMP_prvpt[2]; /* and the IO apic mapping[s] */ for (i = 16; i < 32; i++) newpt[i] = SMP_prvpt[i]; /* prime data page for it to use */ newpp[0] = x; /* cpuid */ newpp[1] = 0; /* curproc */ newpp[2] = 0; /* curpcb */ newpp[3] = 0; /* npxproc */ newpp[4] = 0; /* runtime.tv_sec */ newpp[5] = 0; /* runtime.tv_usec */ newpp[6] = x << 24; /* cpu_lockid */ /* XXX NOTE: ABANDON bootPTD for now!!!! */ /* END REVOLTING HACKERY */ /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ /* record its version info */ cpu_apic_versions[x] = cpu_apic_versions[0]; all_cpus |= (1 << x); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ other_cpus = all_cpus & ~(1 << cpuid); /* fill in our (BSP) APIC version */ cpu_apic_versions[0] = lapic.version; /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); + + pmap_set_opt_bsp(); /* number of APs actually started */ return mp_ncpus - 1; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(u_int boot_addr) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_addr + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_addr + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; } /* * this function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It ain't pretty, * but it seems to work. */ static int start_ap(int logical_cpu, u_int boot_addr) { int physical_cpu; int vector; int cpus; u_long icr_lo, icr_hi; POSTCODE(START_AP_POST); /* get the PHYSICAL APIC ID# */ physical_cpu = CPU_TO_ID(logical_cpu); /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_ncpus; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* setup the address for the target AP */ icr_hi = lapic.icr_hi & ~APIC_ID_MASK; icr_hi |= (physical_cpu << 24); lapic.icr_hi = icr_hi; /* do an INIT IPI: assert RESET */ icr_lo = lapic.icr_lo & 0xfff00000; lapic.icr_lo = icr_lo | 0x0000c500; /* wait for pending status end */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* do an INIT IPI: deassert RESET */ lapic.icr_lo = icr_lo | 0x00008500; /* wait for pending status end */ u_sleep(10000); /* wait ~10mS */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* wait for it to start */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) if (mp_ncpus > cpus) return 1; /* return SUCCESS */ return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's * * XXX: Needs to handshake and wait for completion before proceding. */ void smp_invltlb(void) { #if defined(APIC_IO) if (smp_active && invltlb_ok) all_but_self_ipi(XINVLTLB_OFFSET); #endif /* APIC_IO */ } void invlpg(u_int addr) { __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); /* send a message to the other CPUs */ smp_invltlb(); } void invltlb(void) { u_long temp; /* * This should be implemented as load_cr3(rcr3()) when load_cr3() is * inlined. */ __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); /* send a message to the other CPUs */ smp_invltlb(); } #if defined(TEST_CPUSTOP) #if defined(DEBUG_CPUSTOP) void db_printf __P((const char *fmt, ...)); #endif /* DEBUG_CPUSTOP */ /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int stop_cpus( u_int map ) { if (!smp_active) return 0; /* send IPI to all CPUs in map */ #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d stopping CPUs: 0x%08x\n", cpuid, map); #endif /* DEBUG_CPUSTOP */ stopped_cpus = 0; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); #if defined(DEBUG_CPUSTOP) db_printf(" spin\n"); #endif /* DEBUG_CPUSTOP */ while (stopped_cpus != map) /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" spun\nstopped\n"); #endif /* DEBUG_CPUSTOP */ return 1; } /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus( u_int map ) { if (!smp_active) return 0; #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d restarting CPUs: 0x%08x (0x%08x)\n", cpuid, map, stopped_cpus); #endif /* DEBUG_CPUSTOP */ started_cpus = map; /* signal other cpus to restart */ while (started_cpus) /* wait for each to clear its bit */ /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" restarted\n"); #endif /* DEBUG_CPUSTOP */ return 1; } #endif /** TEST_CPUSTOP */ Index: head/sys/amd64/amd64/pmap.c =================================================================== --- head/sys/amd64/amd64/pmap.c (revision 27483) +++ head/sys/amd64/amd64/pmap.c (revision 27484) @@ -1,3357 +1,3357 @@ /* * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.147 1997/06/25 20:07:50 tegge Exp $ + * $Id: pmap.c,v 1.148 1997/07/17 04:33:38 dyson Exp $ */ /* * Manages physical address maps. * * In addition to hardware address maps, this * module is called upon to provide software-use-only * maps which may or may not be stored in the same * form as hardware maps. These pseudo-maps are * used to store intermediate results from copy * operations to and from address spaces. * * Since the information managed by this module is * also stored by the logical address mapping module, * this module may throw away valid virtual-to-physical * mappings at almost any time. However, invalidations * of virtual-to-physical mappings must be done as * requested. * * In order to cope with hardware architectures which * make virtual-to-physical map invalidates expensive, * this module may delay invalidate or reduced protection * operations until such time as they are actually * necessary. This module is given full information as * to which processors are currently using which maps, * and to when physical maps must be made correct. */ #include "opt_cpu.h" #define PMAP_LOCK 1 #define PMAP_PVLIST 1 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(SMP) || defined(APIC_IO) #include #include #include #endif /* SMP || APIC_IO */ #define PMAP_KEEP_PDIRS #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif #if defined(DIAGNOSTIC) #define PMAP_DIAGNOSTIC #endif #if !defined(PMAP_DIAGNOSTIC) #define PMAP_INLINE __inline #else #define PMAP_INLINE #endif #define PTPHINT static void init_pv_entries __P((int)); /* * Get PDEs and PTEs for user/kernel address space */ #define pmap_pde(m, v) (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT])) #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT]) #define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0) #define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0) #define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0) #define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0) #define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) #define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W)) #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) /* * Given a map and a machine independent protection code, * convert to a vax protection code. */ #define pte_prot(m, p) (protection_codes[p]) static int protection_codes[8]; #define pa_index(pa) atop((pa) - vm_first_phys) #define pa_to_pvh(pa) (&pv_table[pa_index(pa)]) static struct pmap kernel_pmap_store; pmap_t kernel_pmap; vm_offset_t avail_start; /* PA of first available physical page */ vm_offset_t avail_end; /* PA of last available physical page */ vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ static vm_offset_t vm_first_phys; static int pgeflag; /* PG_G or-in */ static int pseflag; /* PG_PS or-in */ static int nkpt; static vm_page_t nkpg; vm_offset_t kernel_vm_end; extern vm_offset_t clean_sva, clean_eva; #define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2) /* * Data for the pv entry allocation mechanism */ static int pv_freelistcnt; TAILQ_HEAD (,pv_entry) pv_freelist = {0}; static vm_offset_t pvva; static int npvvapg; /* * All those kernel PT submaps that BSD is so fond of */ pt_entry_t *CMAP1 = 0; static pt_entry_t *CMAP2, *ptmmap; static pv_table_t *pv_table; caddr_t CADDR1 = 0, ptvmmap = 0; static caddr_t CADDR2; static pt_entry_t *msgbufmap; struct msgbuf *msgbufp=0; pt_entry_t *PMAP1 = 0; unsigned *PADDR1 = 0; static PMAP_INLINE void free_pv_entry __P((pv_entry_t pv)); static unsigned * get_ptbase __P((pmap_t pmap)); static pv_entry_t get_pv_entry __P((void)); static void i386_protection_init __P((void)); static void pmap_alloc_pv_entry __P((void)); static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); static PMAP_INLINE int pmap_is_managed __P((vm_offset_t pa)); static void pmap_remove_all __P((vm_offset_t pa)); static vm_page_t pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_page_t mpte)); static int pmap_remove_pte __P((struct pmap *pmap, unsigned *ptq, vm_offset_t sva)); static void pmap_remove_page __P((struct pmap *pmap, vm_offset_t va)); static int pmap_remove_entry __P((struct pmap *pmap, pv_table_t *pv, vm_offset_t va)); static boolean_t pmap_testbit __P((vm_offset_t pa, int bit)); static void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_offset_t pa)); static vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va)); static int pmap_release_free_page __P((pmap_t pmap, vm_page_t p)); static vm_page_t _pmap_allocpte __P((pmap_t pmap, unsigned ptepindex)); static unsigned * pmap_pte_quick __P((pmap_t pmap, vm_offset_t va)); static vm_page_t pmap_page_alloc __P((vm_object_t object, vm_pindex_t pindex)); static vm_page_t pmap_page_lookup __P((vm_object_t object, vm_pindex_t pindex)); static int pmap_unuse_pt __P((pmap_t, vm_offset_t, vm_page_t)); vm_offset_t pmap_kmem_choose(vm_offset_t addr) ; #define PDSTACKMAX 6 static vm_offset_t pdstack[PDSTACKMAX]; static int pdstackptr; unsigned pdir4mb; /* * Routine: pmap_pte * Function: * Extract the page table entry associated * with the given map/virtual_address pair. */ PMAP_INLINE unsigned * pmap_pte(pmap, va) register pmap_t pmap; vm_offset_t va; { if (pmap && *pmap_pde(pmap, va)) { return get_ptbase(pmap) + i386_btop(va); } return (0); } /* * Move the kernel virtual free pointer to the next * 4MB. This is used to help improve performance * by using a large (4MB) page for much of the kernel * (.text, .data, .bss) */ vm_offset_t pmap_kmem_choose(vm_offset_t addr) { vm_offset_t newaddr = addr; if (cpu_feature & CPUID_PSE) { newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1); } return newaddr; } /* * Bootstrap the system enough to run with virtual memory. * * On the i386 this is called after mapping has already been enabled * and just syncs the pmap module with what has already been done. * [We can't call it easily with mapping off since the kernel is not * mapped with PA == VA, hence we would have to relocate every address * from the linked base (virtual) address "KERNBASE" to the actual * (physical) address starting relative to 0] */ void pmap_bootstrap(firstaddr, loadaddr) vm_offset_t firstaddr; vm_offset_t loadaddr; { vm_offset_t va; pt_entry_t *pte; int i, j; avail_start = firstaddr; /* * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too * large. It should instead be correctly calculated in locore.s and * not based on 'first' (which is a physical address, not a virtual * address, for the start of unused physical memory). The kernel * page tables are NOT double mapped and thus should not be included * in this calculation. */ virtual_avail = (vm_offset_t) KERNBASE + firstaddr; virtual_avail = pmap_kmem_choose(virtual_avail); virtual_end = VM_MAX_KERNEL_ADDRESS; /* * Initialize protection array. */ i386_protection_init(); /* * The kernel's pmap is statically allocated so we don't have to use * pmap_create, which is unlikely to work correctly at this part of * the boot sequence (XXX and which no longer exists). */ kernel_pmap = &kernel_pmap_store; kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD); kernel_pmap->pm_count = 1; #if PMAP_PVLIST TAILQ_INIT(&kernel_pmap->pm_pvlist); #endif nkpt = NKPT; /* * Reserve some special page table entries/VA space for temporary * mapping of pages. */ #define SYSMAP(c, p, v, n) \ v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); va = virtual_avail; pte = (pt_entry_t *) pmap_pte(kernel_pmap, va); /* * CMAP1/CMAP2 are used for zeroing and copying pages. */ SYSMAP(caddr_t, CMAP1, CADDR1, 1) SYSMAP(caddr_t, CMAP2, CADDR2, 1) /* * ptvmmap is used for reading arbitrary physical pages via /dev/mem. * XXX ptmmap is not used. */ SYSMAP(caddr_t, ptmmap, ptvmmap, 1) /* * msgbufp is used to map the system message buffer. * XXX msgbufmap is not used. */ SYSMAP(struct msgbuf *, msgbufmap, msgbufp, atop(round_page(sizeof(struct msgbuf)))) /* * ptemap is used for pmap_pte_quick */ SYSMAP(unsigned *, PMAP1, PADDR1, 1); virtual_avail = va; *(int *) CMAP1 = *(int *) CMAP2 = 0; *(int *) PTD = 0; #ifdef SMP if (cpu_apic_address == 0) panic("pmap_bootstrap: no local apic!"); /* 0 = private page */ /* 1 = page table page */ /* 2 = local apic */ /* 16-31 = io apics */ SMP_prvpt[2] = PG_V | PG_RW | ((u_long)cpu_apic_address & PG_FRAME); for (i = 0; i < mp_napics; i++) { for (j = 0; j < 16; j++) { /* same page frame as a previous IO apic? */ if (((u_long)SMP_prvpt[j + 16] & PG_FRAME) == ((u_long)io_apic_address[0] & PG_FRAME)) { ioapic[i] = (ioapic_t *)&SMP_ioapic[j * PAGE_SIZE]; break; } /* use this slot if available */ if (((u_long)SMP_prvpt[j + 16] & PG_FRAME) == 0) { SMP_prvpt[j + 16] = PG_V | PG_RW | ((u_long)io_apic_address[i] & PG_FRAME); ioapic[i] = (ioapic_t *)&SMP_ioapic[j * PAGE_SIZE]; break; } } if (j == 16) panic("no space to map IO apic %d!", i); } #endif invltlb(); pgeflag = 0; #if !defined(SMP) - if (cpu_feature & CPUID_PGE) + if (cpu_feature & CPUID_PGE) { pgeflag = PG_G; -#endif /* !SMP */ + } +#endif /* * Initialize the 4MB page size flag */ pseflag = 0; /* * The 4MB page version of the initial * kernel page mapping. */ pdir4mb = 0; +#if !defined(DISABLE_PSE) if (cpu_feature & CPUID_PSE) { unsigned ptditmp; /* * Enable the PSE mode */ load_cr4(rcr4() | CR4_PSE); /* * Note that we have enabled PSE mode */ pseflag = PG_PS; ptditmp = (unsigned) kernel_pmap->pm_pdir[KPTDI]; ptditmp &= ~(NBPDR - 1); ptditmp |= PG_V | PG_RW | PG_PS | PG_U | pgeflag; pdir4mb = ptditmp; /* * We can do the mapping here for the single processor * case. We simply ignore the old page table page from * now on. */ #if !defined(SMP) PTD[KPTDI] = (pd_entry_t) ptditmp; kernel_pmap->pm_pdir[KPTDI] = (pd_entry_t) ptditmp; invltlb(); #endif } +#endif } -#if defined(SMP) /* * Set 4mb pdir for mp startup, and global flags */ void pmap_set_opt(unsigned *pdir) { int i; if (pseflag && (cpu_feature & CPUID_PSE)) { load_cr4(rcr4() | CR4_PSE); if (pdir4mb) { (unsigned) pdir[KPTDI] = pdir4mb; } } - if (cpu_feature & CPUID_PGE) { + if (pgeflag && (cpu_feature & CPUID_PGE)) { load_cr4(rcr4() | CR4_PGE); for(i = KPTDI; i < KPTDI + nkpt; i++) { if (pdir[i]) { pdir[i] |= PG_G; } } } } /* * Setup the PTD for the boot processor */ void pmap_set_opt_bsp(void) { pmap_set_opt((unsigned *)kernel_pmap->pm_pdir); pmap_set_opt((unsigned *)PTD); invltlb(); } -#endif /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. * pmap_init has been enhanced to support in a fairly consistant * way, discontiguous physical memory. */ void pmap_init(phys_start, phys_end) vm_offset_t phys_start, phys_end; { vm_offset_t addr; vm_size_t s; int i, npg; /* * calculate the number of pv_entries needed */ vm_first_phys = phys_avail[0]; for (i = 0; phys_avail[i + 1]; i += 2); npg = (phys_avail[(i - 2) + 1] - vm_first_phys) / PAGE_SIZE; /* * Allocate memory for random pmap data structures. Includes the * pv_head_table. */ s = (vm_size_t) (sizeof(pv_table_t) * npg); s = round_page(s); addr = (vm_offset_t) kmem_alloc(kernel_map, s); pv_table = (pv_table_t *) addr; for(i = 0; i < npg; i++) { vm_offset_t pa; TAILQ_INIT(&pv_table[i].pv_list); pv_table[i].pv_list_count = 0; pa = vm_first_phys + i * PAGE_SIZE; pv_table[i].pv_vm_page = PHYS_TO_VM_PAGE(pa); } TAILQ_INIT(&pv_freelist); /* * init the pv free list */ init_pv_entries(npg); /* * Now it is safe to enable pv_table recording. */ pmap_initialized = TRUE; } /* * Used to map a range of physical addresses into kernel * virtual address space. * * For now, VM is already on, we only need to map the * specified memory. */ vm_offset_t pmap_map(virt, start, end, prot) vm_offset_t virt; vm_offset_t start; vm_offset_t end; int prot; { while (start < end) { pmap_enter(kernel_pmap, virt, start, prot, FALSE); virt += PAGE_SIZE; start += PAGE_SIZE; } return (virt); } /*************************************************** * Low level helper routines..... ***************************************************/ #if defined(PMAP_DIAGNOSTIC) /* * This code checks for non-writeable/modified pages. * This should be an invalid condition. */ static int pmap_nw_modified(pt_entry_t ptea) { int pte; pte = (int) ptea; if ((pte & (PG_M|PG_RW)) == PG_M) return 1; else return 0; } #endif /* * this routine defines the region(s) of memory that should * not be tested for the modified bit. */ static PMAP_INLINE int pmap_track_modified( vm_offset_t va) { if ((va < clean_sva) || (va >= clean_eva)) return 1; else return 0; } static PMAP_INLINE void invltlb_1pg( vm_offset_t va) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) { invltlb(); } else #endif { invlpg(va); } } static PMAP_INLINE void invltlb_2pg( vm_offset_t va1, vm_offset_t va2) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) { invltlb(); } else #endif { invlpg(va1); invlpg(va2); } } static unsigned * get_ptbase(pmap) pmap_t pmap; { unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; /* are we current address space or kernel? */ if (pmap == kernel_pmap || frame == (((unsigned) PTDpde) & PG_FRAME)) { return (unsigned *) PTmap; } /* otherwise, we are alternate address space */ if (frame != (((unsigned) APTDpde) & PG_FRAME)) { APTDpde = (pd_entry_t) (frame | PG_RW | PG_V); invltlb(); } return (unsigned *) APTmap; } /* * Super fast pmap_pte routine best used when scanning * the pv lists. This eliminates many coarse-grained * invltlb calls. Note that many of the pv list * scans are across different pmaps. It is very wasteful * to do an entire invltlb for checking a single mapping. */ static unsigned * pmap_pte_quick(pmap, va) register pmap_t pmap; vm_offset_t va; { unsigned pde, newpf; if (pde = (unsigned) pmap->pm_pdir[va >> PDRSHIFT]) { unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; unsigned index = i386_btop(va); /* are we current address space or kernel? */ if ((pmap == kernel_pmap) || (frame == (((unsigned) PTDpde) & PG_FRAME))) { return (unsigned *) PTmap + index; } newpf = pde & PG_FRAME; if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) { * (unsigned *) PMAP1 = newpf | PG_RW | PG_V; invltlb_1pg((vm_offset_t) PADDR1); } return PADDR1 + ((unsigned) index & (NPTEPG - 1)); } return (0); } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_offset_t pmap_extract(pmap, va) register pmap_t pmap; vm_offset_t va; { vm_offset_t rtval; vm_offset_t pdirindex; pdirindex = va >> PDRSHIFT; if (pmap) { unsigned *pte; if (((rtval = (unsigned) pmap->pm_pdir[pdirindex]) & PG_PS) != 0) { rtval &= ~(NBPDR - 1); rtval |= va & (NBPDR - 1); return rtval; } pte = get_ptbase(pmap) + i386_btop(va); rtval = ((*pte & PG_FRAME) | (va & PAGE_MASK)); return rtval; } return 0; } /* * determine if a page is managed (memory vs. device) */ static PMAP_INLINE int pmap_is_managed(pa) vm_offset_t pa; { int i; if (!pmap_initialized) return 0; for (i = 0; phys_avail[i + 1]; i += 2) { if (pa < phys_avail[i + 1] && pa >= phys_avail[i]) return 1; } return 0; } /*************************************************** * Low level mapping routines..... ***************************************************/ /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. */ void pmap_qenter(va, m, count) vm_offset_t va; vm_page_t *m; int count; { int i; register unsigned *pte; for (i = 0; i < count; i++) { vm_offset_t tva = va + i * PAGE_SIZE; unsigned npte = VM_PAGE_TO_PHYS(m[i]) | PG_RW | PG_V | pgeflag; unsigned opte; pte = (unsigned *)vtopte(tva); opte = *pte; *pte = npte; if (opte) invltlb_1pg(tva); } } /* * this routine jerks page mappings from the * kernel -- it is meant only for temporary mappings. */ void pmap_qremove(va, count) vm_offset_t va; int count; { int i; register unsigned *pte; for (i = 0; i < count; i++) { pte = (unsigned *)vtopte(va); *pte = 0; invltlb_1pg(va); va += PAGE_SIZE; } } /* * add a wired page to the kva * note that in order for the mapping to take effect -- you * should do a invltlb after doing the pmap_kenter... */ PMAP_INLINE void pmap_kenter(va, pa) vm_offset_t va; register vm_offset_t pa; { register unsigned *pte; unsigned npte, opte; npte = pa | PG_RW | PG_V | pgeflag; pte = (unsigned *)vtopte(va); opte = *pte; *pte = npte; if (opte) invltlb_1pg(va); } /* * remove a page from the kernel pagetables */ PMAP_INLINE void pmap_kremove(va) vm_offset_t va; { register unsigned *pte; pte = (unsigned *)vtopte(va); *pte = 0; invltlb_1pg(va); } static vm_page_t pmap_page_alloc(object, pindex) vm_object_t object; vm_pindex_t pindex; { vm_page_t m; m = vm_page_alloc(object, pindex, VM_ALLOC_ZERO); if (m == NULL) { VM_WAIT; } return m; } static vm_page_t pmap_page_lookup(object, pindex) vm_object_t object; vm_pindex_t pindex; { vm_page_t m; retry: m = vm_page_lookup(object, pindex); if (m) { if (m->flags & PG_BUSY) { m->flags |= PG_WANTED; tsleep(m, PVM, "pplookp", 0); goto retry; } } return m; } /* * Create the UPAGES for a new process. * This routine directly affects the fork perf for a process. */ void pmap_new_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; struct user *up; unsigned *ptek; /* * allocate object for the upages */ upobj = vm_object_allocate( OBJT_DEFAULT, UPAGES); p->p_upages_obj = upobj; /* get a kernel virtual address for the UPAGES for this proc */ up = (struct user *) kmem_alloc_pageable(u_map, UPAGES * PAGE_SIZE); if (up == NULL) panic("pmap_new_proc: u_map allocation failed"); ptek = (unsigned *) vtopte((vm_offset_t) up); for(i=0;iwire_count++; ++cnt.v_wire_count; /* * Enter the page into the kernel address space. */ *(ptek + i) = VM_PAGE_TO_PHYS(m) | PG_RW | PG_V | pgeflag; m->flags &= ~(PG_ZERO|PG_BUSY); m->flags |= PG_MAPPED|PG_WRITEABLE; m->valid = VM_PAGE_BITS_ALL; } p->p_addr = up; } /* * Dispose the UPAGES for a process that has exited. * This routine directly impacts the exit perf of a process. */ void pmap_dispose_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; unsigned *ptek; ptek = (unsigned *) vtopte((vm_offset_t) p->p_addr); upobj = p->p_upages_obj; for(i=0;ip_addr + i * PAGE_SIZE); vm_page_unwire(m); vm_page_free(m); } vm_object_deallocate(upobj); kmem_free(u_map, (vm_offset_t)p->p_addr, ctob(UPAGES)); } /* * Allow the UPAGES for a process to be prejudicially paged out. */ void pmap_swapout_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; upobj = p->p_upages_obj; /* * let the upages be paged */ for(i=0;idirty = VM_PAGE_BITS_ALL; vm_page_unwire(m); vm_page_deactivate(m); pmap_kremove( (vm_offset_t) p->p_addr + PAGE_SIZE * i); } } /* * Bring the UPAGES for a specified process back in. */ void pmap_swapin_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; unsigned *pte; upobj = p->p_upages_obj; for(i=0;iflags & PG_BUSY) || m->busy) { m->flags |= PG_WANTED; tsleep(m, PVM, "swinuw",0); goto retry; } m->flags |= PG_BUSY; } vm_page_wire(m); splx(s); pmap_kenter(((vm_offset_t) p->p_addr) + i * PAGE_SIZE, VM_PAGE_TO_PHYS(m)); if (m->valid != VM_PAGE_BITS_ALL) { int rv; rv = vm_pager_get_pages(upobj, &m, 1, 0); if (rv != VM_PAGER_OK) panic("pmap_swapin_proc: cannot get upages for proc: %d\n", p->p_pid); m->valid = VM_PAGE_BITS_ALL; } PAGE_WAKEUP(m); m->flags |= PG_MAPPED|PG_WRITEABLE; } } /*************************************************** * Page table page management routines..... ***************************************************/ /* * This routine unholds page table pages, and if the hold count * drops to zero, then it decrements the wire count. */ static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { int s; if (m->flags & PG_BUSY) { s = splvm(); while (m->flags & PG_BUSY) { m->flags |= PG_WANTED; tsleep(m, PVM, "pmuwpt", 0); } splx(s); } if (m->hold_count == 0) { vm_offset_t pteva; /* * unmap the page table page */ pmap->pm_pdir[m->pindex] = 0; --pmap->pm_stats.resident_count; if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) == (((unsigned) PTDpde) & PG_FRAME)) { /* * Do a invltlb to make the invalidated mapping * take effect immediately. */ pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex); invltlb_1pg(pteva); } #if defined(PTPHINT) if (pmap->pm_ptphint == m) pmap->pm_ptphint = NULL; #endif /* * If the page is finally unwired, simply free it. */ --m->wire_count; if (m->wire_count == 0) { if (m->flags & PG_WANTED) { m->flags &= ~PG_WANTED; wakeup(m); } vm_page_free_zero(m); --cnt.v_wire_count; } return 1; } return 0; } __inline static int pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { vm_page_unhold(m); if (m->hold_count == 0) return _pmap_unwire_pte_hold(pmap, m); else return 0; } /* * After removing a page table entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ static int pmap_unuse_pt(pmap, va, mpte) pmap_t pmap; vm_offset_t va; vm_page_t mpte; { unsigned ptepindex; if (va >= UPT_MIN_ADDRESS) return 0; if (mpte == NULL) { ptepindex = (va >> PDRSHIFT); #if defined(PTPHINT) if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == ptepindex)) { mpte = pmap->pm_ptphint; } else { mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); pmap->pm_ptphint = mpte; } #else mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); #endif } return pmap_unwire_pte_hold(pmap, mpte); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ void pmap_pinit(pmap) register struct pmap *pmap; { vm_page_t ptdpg; /* * No need to allocate page table space yet but we do need a valid * page directory table. */ if (pdstackptr > 0) { --pdstackptr; pmap->pm_pdir = (pd_entry_t *)pdstack[pdstackptr]; } else { pmap->pm_pdir = (pd_entry_t *)kmem_alloc_pageable(kernel_map, PAGE_SIZE); } /* * allocate object for the ptes */ pmap->pm_pteobj = vm_object_allocate( OBJT_DEFAULT, PTDPTDI + 1); /* * allocate the page directory page */ retry: ptdpg = pmap_page_alloc( pmap->pm_pteobj, PTDPTDI); if (ptdpg == NULL) goto retry; ptdpg->wire_count = 1; ++cnt.v_wire_count; ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */ ptdpg->valid = VM_PAGE_BITS_ALL; pmap_kenter((vm_offset_t) pmap->pm_pdir, VM_PAGE_TO_PHYS(ptdpg)); if ((ptdpg->flags & PG_ZERO) == 0) bzero(pmap->pm_pdir, PAGE_SIZE); /* wire in kernel global address entries */ /* XXX copies current process, does not fill in MPPTDI */ bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE); /* install self-referential address mapping entry */ *(unsigned *) (pmap->pm_pdir + PTDPTDI) = VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW; pmap->pm_flags = 0; pmap->pm_count = 1; pmap->pm_ptphint = NULL; #if PMAP_PVLIST TAILQ_INIT(&pmap->pm_pvlist); #endif } static int pmap_release_free_page(pmap, p) struct pmap *pmap; vm_page_t p; { int s; unsigned *pde = (unsigned *) pmap->pm_pdir; /* * This code optimizes the case of freeing non-busy * page-table pages. Those pages are zero now, and * might as well be placed directly into the zero queue. */ s = splvm(); if (p->flags & PG_BUSY) { p->flags |= PG_WANTED; tsleep(p, PVM, "pmaprl", 0); splx(s); return 0; } if (p->flags & PG_WANTED) { p->flags &= ~PG_WANTED; wakeup(p); } /* * Remove the page table page from the processes address space. */ pde[p->pindex] = 0; --pmap->pm_stats.resident_count; if (p->hold_count) { panic("pmap_release: freeing held page table page"); } /* * Page directory pages need to have the kernel * stuff cleared, so they can go into the zero queue also. */ if (p->pindex == PTDPTDI) { bzero(pde + KPTDI, nkpt * PTESIZE); #ifdef SMP pde[MPPTDI] = 0; #endif pde[APTDPTDI] = 0; pmap_kremove((vm_offset_t) pmap->pm_pdir); } #if defined(PTPHINT) if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == p->pindex)) pmap->pm_ptphint = NULL; #endif vm_page_free_zero(p); splx(s); return 1; } /* * this routine is called if the page table page is not * mapped correctly. */ static vm_page_t _pmap_allocpte(pmap, ptepindex) pmap_t pmap; unsigned ptepindex; { vm_offset_t pteva, ptepa; vm_page_t m; int needszero = 0; /* * Find or fabricate a new pagetable page */ retry: m = vm_page_lookup(pmap->pm_pteobj, ptepindex); if (m == NULL) { m = pmap_page_alloc(pmap->pm_pteobj, ptepindex); if (m == NULL) goto retry; if ((m->flags & PG_ZERO) == 0) needszero = 1; m->flags &= ~(PG_ZERO|PG_BUSY); m->valid = VM_PAGE_BITS_ALL; } else { if ((m->flags & PG_BUSY) || m->busy) { m->flags |= PG_WANTED; tsleep(m, PVM, "ptewai", 0); goto retry; } } if (m->queue != PQ_NONE) { int s = splvm(); vm_page_unqueue(m); splx(s); } if (m->wire_count == 0) ++cnt.v_wire_count; ++m->wire_count; /* * Increment the hold count for the page table page * (denoting a new mapping.) */ ++m->hold_count; /* * Map the pagetable page into the process address space, if * it isn't already there. */ pmap->pm_stats.resident_count++; ptepa = VM_PAGE_TO_PHYS(m); pmap->pm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V); #if defined(PTPHINT) /* * Set the page table hint */ pmap->pm_ptphint = m; #endif /* * Try to use the new mapping, but if we cannot, then * do it with the routine that maps the page explicitly. */ if (needszero) { if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) == (((unsigned) PTDpde) & PG_FRAME)) { pteva = UPT_MIN_ADDRESS + i386_ptob(ptepindex); bzero((caddr_t) pteva, PAGE_SIZE); } else { pmap_zero_page(ptepa); } } m->valid = VM_PAGE_BITS_ALL; m->flags |= PG_MAPPED; return m; } static vm_page_t pmap_allocpte(pmap, va) pmap_t pmap; vm_offset_t va; { unsigned ptepindex; vm_offset_t ptepa; vm_page_t m; /* * Calculate pagetable page index */ ptepindex = va >> PDRSHIFT; /* * Get the page directory entry */ ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex]; /* * This supports switching from a 4MB page to a * normal 4K page. */ if (ptepa & PG_PS) { pmap->pm_pdir[ptepindex] = 0; ptepa = 0; invltlb(); } /* * If the page table page is mapped, we just increment the * hold count, and activate it. */ if (ptepa) { #if defined(PTPHINT) /* * In order to get the page table page, try the * hint first. */ if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == ptepindex)) { m = pmap->pm_ptphint; } else { m = pmap_page_lookup( pmap->pm_pteobj, ptepindex); pmap->pm_ptphint = m; } #else m = pmap_page_lookup( pmap->pm_pteobj, ptepindex); #endif ++m->hold_count; return m; } /* * Here if the pte page isn't mapped, or if it has been deallocated. */ return _pmap_allocpte(pmap, ptepindex); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap) register struct pmap *pmap; { vm_page_t p,n,ptdpg; vm_object_t object = pmap->pm_pteobj; #if defined(DIAGNOSTIC) if (object->ref_count != 1) panic("pmap_release: pteobj reference count != 1"); #endif ptdpg = NULL; retry: for (p = TAILQ_FIRST(&object->memq); p != NULL; p = n) { n = TAILQ_NEXT(p, listq); if (p->pindex == PTDPTDI) { ptdpg = p; continue; } if (!pmap_release_free_page(pmap, p)) goto retry; } if (ptdpg && !pmap_release_free_page(pmap, ptdpg)) goto retry; vm_object_deallocate(object); if (pdstackptr < PDSTACKMAX) { pdstack[pdstackptr] = (vm_offset_t) pmap->pm_pdir; ++pdstackptr; } else { kmem_free(kernel_map, (vm_offset_t) pmap->pm_pdir, PAGE_SIZE); } pmap->pm_pdir = 0; } /* * grow the number of kernel page table entries, if needed */ void pmap_growkernel(vm_offset_t addr) { struct proc *p; struct pmap *pmap; int s; s = splhigh(); if (kernel_vm_end == 0) { kernel_vm_end = KERNBASE; nkpt = 0; while (pdir_pde(PTD, kernel_vm_end)) { kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); ++nkpt; } } addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); while (kernel_vm_end < addr) { if (pdir_pde(PTD, kernel_vm_end)) { kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); continue; } ++nkpt; if (!nkpg) { vm_offset_t ptpkva = (vm_offset_t) vtopte(addr); /* * This index is bogus, but out of the way */ vm_pindex_t ptpidx = (ptpkva >> PAGE_SHIFT); nkpg = vm_page_alloc(kernel_object, ptpidx, VM_ALLOC_SYSTEM); if (!nkpg) panic("pmap_growkernel: no memory to grow kernel"); vm_page_wire(nkpg); vm_page_remove(nkpg); pmap_zero_page(VM_PAGE_TO_PHYS(nkpg)); } pdir_pde(PTD, kernel_vm_end) = (pd_entry_t) (VM_PAGE_TO_PHYS(nkpg) | PG_V | PG_RW | pgeflag); nkpg = NULL; for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { if (p->p_vmspace) { pmap = &p->p_vmspace->vm_pmap; *pmap_pde(pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); } } *pmap_pde(kernel_pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); } splx(s); } /* * Retire the given physical map from service. * Should only be called if the map contains * no valid mappings. */ void pmap_destroy(pmap) register pmap_t pmap; { int count; if (pmap == NULL) return; count = --pmap->pm_count; if (count == 0) { pmap_release(pmap); free((caddr_t) pmap, M_VMPMAP); } } /* * Add a reference to the specified pmap. */ void pmap_reference(pmap) pmap_t pmap; { if (pmap != NULL) { pmap->pm_count++; } } /*************************************************** * page management routines. ***************************************************/ /* * free the pv_entry back to the free list */ static PMAP_INLINE void free_pv_entry(pv) pv_entry_t pv; { ++pv_freelistcnt; TAILQ_INSERT_HEAD(&pv_freelist, pv, pv_list); } /* * get a new pv_entry, allocating a block from the system * when needed. * the memory allocation is performed bypassing the malloc code * because of the possibility of allocations at interrupt time. */ static pv_entry_t get_pv_entry() { pv_entry_t tmp; /* * get more pv_entry pages if needed */ if (pv_freelistcnt < PV_FREELIST_MIN || !TAILQ_FIRST(&pv_freelist)) { pmap_alloc_pv_entry(); } /* * get a pv_entry off of the free list */ --pv_freelistcnt; tmp = TAILQ_FIRST(&pv_freelist); TAILQ_REMOVE(&pv_freelist, tmp, pv_list); return tmp; } /* * This *strange* allocation routine eliminates the possibility of a malloc * failure (*FATAL*) for a pv_entry_t data structure. * also -- this code is MUCH MUCH faster than the malloc equiv... * We really need to do the slab allocator thingie here. */ static void pmap_alloc_pv_entry() { /* * do we have any pre-allocated map-pages left? */ if (npvvapg) { vm_page_t m; /* * allocate a physical page out of the vm system */ m = vm_page_alloc(kernel_object, OFF_TO_IDX(pvva - vm_map_min(kernel_map)), VM_ALLOC_INTERRUPT); if (m) { int newentries; int i; pv_entry_t entry; newentries = (PAGE_SIZE / sizeof(struct pv_entry)); /* * wire the page */ vm_page_wire(m); m->flags &= ~PG_BUSY; /* * let the kernel see it */ pmap_kenter(pvva, VM_PAGE_TO_PHYS(m)); entry = (pv_entry_t) pvva; /* * update the allocation pointers */ pvva += PAGE_SIZE; --npvvapg; /* * free the entries into the free list */ for (i = 0; i < newentries; i++) { free_pv_entry(entry); entry++; } } } if (!TAILQ_FIRST(&pv_freelist)) panic("get_pv_entry: cannot get a pv_entry_t"); } /* * init the pv_entry allocation system */ void init_pv_entries(npg) int npg; { /* * Allocate enough kvm space for one entry per page, and * each process having PMAP_SHPGPERPROC pages shared with other * processes. (The system can panic if this is too small, but also * can fail on bootup if this is too big.) * XXX The pv management mechanism needs to be fixed so that systems * with lots of shared mappings amongst lots of processes will still * work. The fix will likely be that once we run out of pv entries * we will free other entries (and the associated mappings), with * some policy yet to be determined. */ npvvapg = ((PMAP_SHPGPERPROC * maxproc + npg) * sizeof(struct pv_entry) + PAGE_SIZE - 1) / PAGE_SIZE; pvva = kmem_alloc_pageable(kernel_map, npvvapg * PAGE_SIZE); /* * get the first batch of entries */ pmap_alloc_pv_entry(); } /* * If it is the first entry on the list, it is actually * in the header and we must copy the following entry up * to the header. Otherwise we must search the list for * the entry. In either case we free the now unused entry. */ static int pmap_remove_entry(pmap, ppv, va) struct pmap *pmap; pv_table_t *ppv; vm_offset_t va; { pv_entry_t pv; int rtval; int s; s = splvm(); #if PMAP_PVLIST if (ppv->pv_list_count < pmap->pm_stats.resident_count) { #endif for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { if (pmap == pv->pv_pmap && va == pv->pv_va) break; } #if PMAP_PVLIST } else { for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = TAILQ_NEXT(pv, pv_plist)) { if (va == pv->pv_va) break; } } #endif rtval = 0; if (pv) { rtval = pmap_unuse_pt(pmap, va, pv->pv_ptem); TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); --ppv->pv_list_count; if (TAILQ_FIRST(&ppv->pv_list) == NULL) { ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE); } #if PMAP_PVLIST TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); #endif free_pv_entry(pv); } splx(s); return rtval; } /* * Create a pv entry for page at pa for * (pmap, va). */ static void pmap_insert_entry(pmap, va, mpte, pa) pmap_t pmap; vm_offset_t va; vm_page_t mpte; vm_offset_t pa; { int s; pv_entry_t pv; pv_table_t *ppv; s = splvm(); pv = get_pv_entry(); pv->pv_va = va; pv->pv_pmap = pmap; pv->pv_ptem = mpte; #if PMAP_PVLIST TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); #endif ppv = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&ppv->pv_list, pv, pv_list); ++ppv->pv_list_count; splx(s); } /* * pmap_remove_pte: do the things to unmap a page in a process */ static int pmap_remove_pte(pmap, ptq, va) struct pmap *pmap; unsigned *ptq; vm_offset_t va; { unsigned oldpte; pv_table_t *ppv; oldpte = *ptq; *ptq = 0; if (oldpte & PG_W) pmap->pm_stats.wired_count -= 1; /* * Machines that don't support invlpg, also don't support * PG_G. */ if (oldpte & PG_G) invlpg(va); pmap->pm_stats.resident_count -= 1; if (oldpte & PG_MANAGED) { ppv = pa_to_pvh(oldpte); if (oldpte & PG_M) { #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) oldpte)) { printf("pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, (int) oldpte); } #endif if (pmap_track_modified(va)) ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } return pmap_remove_entry(pmap, ppv, va); } else { return pmap_unuse_pt(pmap, va, NULL); } return 0; } /* * Remove a single page from a process address space */ static void pmap_remove_page(pmap, va) struct pmap *pmap; register vm_offset_t va; { register unsigned *ptq; /* * if there is no pte for this address, just skip it!!! */ if (*pmap_pde(pmap, va) == 0) { return; } /* * get a local va for mappings for this pmap. */ ptq = get_ptbase(pmap) + i386_btop(va); if (*ptq) { (void) pmap_remove_pte(pmap, ptq, va); invltlb_1pg(va); } return; } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ void pmap_remove(pmap, sva, eva) struct pmap *pmap; register vm_offset_t sva; register vm_offset_t eva; { register unsigned *ptbase; vm_offset_t pdnxt; vm_offset_t ptpaddr; vm_offset_t sindex, eindex; int anyvalid; if (pmap == NULL) return; if (pmap->pm_stats.resident_count == 0) return; /* * special handling of removing one page. a very * common operation and easy to short circuit some * code. */ if (((sva + PAGE_SIZE) == eva) && (((unsigned) pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) { pmap_remove_page(pmap, sva); return; } anyvalid = 0; /* * Get a local virtual address for the mappings that are being * worked with. */ ptbase = get_ptbase(pmap); sindex = i386_btop(sva); eindex = i386_btop(eva); for (; sindex < eindex; sindex = pdnxt) { unsigned pdirindex; /* * Calculate index for next page table. */ pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); if (pmap->pm_stats.resident_count == 0) break; pdirindex = sindex / NPDEPG; if (((ptpaddr = (unsigned) pmap->pm_pdir[pdirindex]) & PG_PS) != 0) { pmap->pm_pdir[pdirindex] = 0; pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; anyvalid++; continue; } /* * Weed out invalid mappings. Note: we assume that the page * directory table is always allocated, and in kernel virtual. */ if (ptpaddr == 0) continue; /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the * range being removed. */ if (pdnxt > eindex) { pdnxt = eindex; } for ( ;sindex != pdnxt; sindex++) { vm_offset_t va; if (ptbase[sindex] == 0) { continue; } va = i386_ptob(sindex); anyvalid++; if (pmap_remove_pte(pmap, ptbase + sindex, va)) break; } } if (anyvalid) { invltlb(); } } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ static void pmap_remove_all(pa) vm_offset_t pa; { register pv_entry_t pv; pv_table_t *ppv; register unsigned *pte, tpte; int nmodify; int update_needed; int s; nmodify = 0; update_needed = 0; #if defined(PMAP_DIAGNOSTIC) /* * XXX this makes pmap_page_protect(NONE) illegal for non-managed * pages! */ if (!pmap_is_managed(pa)) { panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", pa); } #endif s = splvm(); ppv = pa_to_pvh(pa); while ((pv = TAILQ_FIRST(&ppv->pv_list)) != NULL) { pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); pv->pv_pmap->pm_stats.resident_count--; tpte = *pte; *pte = 0; if (tpte & PG_W) pv->pv_pmap->pm_stats.wired_count--; /* * Update the vm_page_t clean and reference bits. */ if (tpte & PG_M) { #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) tpte)) { printf("pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n", pv->pv_va, tpte); } #endif if (pmap_track_modified(pv->pv_va)) ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } if (!update_needed && ((!curproc || (&curproc->p_vmspace->vm_pmap == pv->pv_pmap)) || (pv->pv_pmap == kernel_pmap))) { update_needed = 1; } #if PMAP_PVLIST TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); #endif TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); --ppv->pv_list_count; pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); free_pv_entry(pv); } ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE); if (update_needed) invltlb(); splx(s); return; } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap, sva, eva, prot) register pmap_t pmap; vm_offset_t sva, eva; vm_prot_t prot; { register unsigned *ptbase; vm_offset_t pdnxt; vm_offset_t ptpaddr; vm_offset_t sindex, eindex; int anychanged; if (pmap == NULL) return; if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; } anychanged = 0; ptbase = get_ptbase(pmap); sindex = i386_btop(sva); eindex = i386_btop(eva); for (; sindex < eindex; sindex = pdnxt) { unsigned pdirindex; pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); pdirindex = sindex / NPDEPG; if (((ptpaddr = (unsigned) pmap->pm_pdir[pdirindex]) & PG_PS) != 0) { (unsigned) pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW); pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; anychanged++; continue; } /* * Weed out invalid mappings. Note: we assume that the page * directory table is always allocated, and in kernel virtual. */ if (ptpaddr == 0) continue; if (pdnxt > eindex) { pdnxt = eindex; } for (; sindex != pdnxt; sindex++) { unsigned pbits = ptbase[sindex]; if (prot & VM_PROT_WRITE) { if ((pbits & (PG_RW|PG_V)) == PG_V) { if (pbits & PG_MANAGED) { vm_page_t m = PHYS_TO_VM_PAGE(pbits); m->flags |= PG_WRITEABLE; m->object->flags |= OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY; } ptbase[sindex] = pbits | PG_RW; anychanged = 1; } } else if (pbits & PG_RW) { if (pbits & PG_M) { vm_offset_t sva = i386_ptob(sindex); if ((pbits & PG_MANAGED) && pmap_track_modified(sva)) { vm_page_t m = PHYS_TO_VM_PAGE(pbits); m->dirty = VM_PAGE_BITS_ALL; } } ptbase[sindex] = pbits & ~(PG_M|PG_RW); anychanged = 1; } } } if (anychanged) invltlb(); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ void pmap_enter(pmap, va, pa, prot, wired) register pmap_t pmap; vm_offset_t va; register vm_offset_t pa; vm_prot_t prot; boolean_t wired; { register unsigned *pte; vm_offset_t opa; vm_offset_t origpte, newpte; vm_page_t mpte; if (pmap == NULL) return; va &= PG_FRAME; #ifdef PMAP_DIAGNOSTIC if (va > VM_MAX_KERNEL_ADDRESS) panic("pmap_enter: toobig"); if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS)) panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)", va); #endif mpte = NULL; /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < UPT_MIN_ADDRESS) mpte = pmap_allocpte(pmap, va); pte = pmap_pte(pmap, va); /* * Page Directory table entry not valid, we need a new PT page */ if (pte == NULL) { panic("pmap_enter: invalid page directory, pdir=%p, va=0x%lx\n", pmap->pm_pdir[PTDPTDI], va); } origpte = *(vm_offset_t *)pte; pa &= PG_FRAME; opa = origpte & PG_FRAME; if (origpte & PG_PS) panic("pmap_enter: attempted pmap_enter on 4MB page"); /* * Mapping has not changed, must be protection or wiring change. */ if (origpte && (opa == pa)) { /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there * are valid mappings in them. Hence, if a user page is wired, * the PT page will be also. */ if (wired && ((origpte & PG_W) == 0)) pmap->pm_stats.wired_count++; else if (!wired && (origpte & PG_W)) pmap->pm_stats.wired_count--; #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) origpte)) { printf("pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, origpte); } #endif /* * We might be turning off write access to the page, * so we go ahead and sense modify status. */ if (origpte & PG_MANAGED) { vm_page_t m; if (origpte & PG_M) { if (pmap_track_modified(va)) { m = PHYS_TO_VM_PAGE(pa); m->dirty = VM_PAGE_BITS_ALL; } } pa |= PG_MANAGED; } if (mpte) --mpte->hold_count; goto validate; } /* * Mapping has changed, invalidate old range and fall through to * handle validating new mapping. */ if (opa) { int err; err = pmap_remove_pte(pmap, pte, va); if (err) panic("pmap_enter: pte vanished, va: 0x%x", va); } /* * Enter on the PV list if part of our managed memory Note that we * raise IPL while manipulating pv_table since pmap_enter can be * called at interrupt time. */ if (pmap_is_managed(pa)) { pmap_insert_entry(pmap, va, mpte, pa); pa |= PG_MANAGED; } /* * Increment counters */ pmap->pm_stats.resident_count++; if (wired) pmap->pm_stats.wired_count++; validate: /* * Now validate mapping with desired protection/wiring. */ newpte = (vm_offset_t) (pa | pte_prot(pmap, prot) | PG_V); if (wired) newpte |= PG_W; if (va < UPT_MIN_ADDRESS) newpte |= PG_U; if (pmap == kernel_pmap) newpte |= pgeflag; /* * if the mapping or permission bits are different, we need * to update the pte. */ if ((origpte & ~(PG_M|PG_A)) != newpte) { *pte = newpte; if (origpte) invltlb_1pg(va); } } /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No page table pages. * 5. Tlbflush is deferred to calling procedure. * 6. Page IS managed. * but is *MUCH* faster than pmap_enter... */ static vm_page_t pmap_enter_quick(pmap, va, pa, mpte) register pmap_t pmap; vm_offset_t va; register vm_offset_t pa; vm_page_t mpte; { register unsigned *pte; /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < UPT_MIN_ADDRESS) { unsigned ptepindex; vm_offset_t ptepa; /* * Calculate pagetable page index */ ptepindex = va >> PDRSHIFT; if (mpte && (mpte->pindex == ptepindex)) { ++mpte->hold_count; } else { retry: /* * Get the page directory entry */ ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex]; /* * If the page table page is mapped, we just increment * the hold count, and activate it. */ if (ptepa) { if (ptepa & PG_PS) panic("pmap_enter_quick: unexpected mapping into 4MB page"); #if defined(PTPHINT) if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == ptepindex)) { mpte = pmap->pm_ptphint; } else { mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); pmap->pm_ptphint = mpte; } #else mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); #endif if (mpte == NULL) goto retry; ++mpte->hold_count; } else { mpte = _pmap_allocpte(pmap, ptepindex); } } } else { mpte = NULL; } /* * This call to vtopte makes the assumption that we are * entering the page into the current pmap. In order to support * quick entry into any pmap, one would likely use pmap_pte_quick. * But that isn't as quick as vtopte. */ pte = (unsigned *)vtopte(va); if (*pte) { if (mpte) pmap_unwire_pte_hold(pmap, mpte); return 0; } /* * Enter on the PV list if part of our managed memory Note that we * raise IPL while manipulating pv_table since pmap_enter can be * called at interrupt time. */ pmap_insert_entry(pmap, va, mpte, pa); /* * Increment counters */ pmap->pm_stats.resident_count++; /* * Now validate mapping with RO protection */ *pte = pa | PG_V | PG_U | PG_MANAGED; return mpte; } #define MAX_INIT_PT (96) /* * pmap_object_init_pt preloads the ptes for a given object * into the specified pmap. This eliminates the blast of soft * faults on process startup and immediately after an mmap. */ void pmap_object_init_pt(pmap, addr, object, pindex, size, limit) pmap_t pmap; vm_offset_t addr; vm_object_t object; vm_pindex_t pindex; vm_size_t size; int limit; { vm_offset_t tmpidx; int psize; vm_page_t p, mpte; int objpgs; if (!pmap) return; /* * This code maps large physical mmap regions into the * processor address space. Note that some shortcuts * are taken, but the code works. */ if (pseflag && (object->type == OBJT_DEVICE) && + ((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0) ) { int i; int s; vm_page_t m[1]; unsigned int ptepindex; int npdes; vm_offset_t ptepa; if (pmap->pm_pdir[ptepindex = (addr >> PDRSHIFT)]) return; s = splhigh(); retry: p = vm_page_lookup(object, pindex); if (p && (p->flags & PG_BUSY)) { tsleep(p, PVM, "init4p", 0); goto retry; } splx(s); if (p == NULL) { p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL); if (p == NULL) return; m[0] = p; if (vm_pager_get_pages(object, m, 1, 0) != VM_PAGER_OK) { PAGE_WAKEUP(p); vm_page_free(p); return; } p = vm_page_lookup(object, pindex); PAGE_WAKEUP(p); } ptepa = (vm_offset_t) VM_PAGE_TO_PHYS(p); if (ptepa & (NBPDR - 1)) { return; } p->valid = VM_PAGE_BITS_ALL; pmap->pm_stats.resident_count += size >> PAGE_SHIFT; npdes = size >> PDRSHIFT; for(i=0;ipm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_PS); ptepa += NBPDR; ptepindex += 1; } p->flags |= PG_MAPPED; -#if 0 invltlb(); -#endif return; } psize = i386_btop(size); if ((object->type != OBJT_VNODE) || (limit && (psize > MAX_INIT_PT) && (object->resident_page_count > MAX_INIT_PT))) { return; } if (psize + pindex > object->size) psize = object->size - pindex; mpte = NULL; /* * if we are processing a major portion of the object, then scan the * entire thing. */ if (psize > (object->size >> 2)) { objpgs = psize; for (p = TAILQ_FIRST(&object->memq); ((objpgs > 0) && (p != NULL)); p = TAILQ_NEXT(p, listq)) { tmpidx = p->pindex; if (tmpidx < pindex) { continue; } tmpidx -= pindex; if (tmpidx >= psize) { continue; } if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((p->queue - p->pc) == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; mpte = pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), VM_PAGE_TO_PHYS(p), mpte); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } objpgs -= 1; } } else { /* * else lookup the pages one-by-one. */ for (tmpidx = 0; tmpidx < psize; tmpidx += 1) { p = vm_page_lookup(object, tmpidx + pindex); if (p && ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((p->queue - p->pc) == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; mpte = pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), VM_PAGE_TO_PHYS(p), mpte); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } } } return; } /* * pmap_prefault provides a quick way of clustering * pagefaults into a processes address space. It is a "cousin" * of pmap_object_init_pt, except it runs at page fault time instead * of mmap time. */ #define PFBAK 2 #define PFFOR 2 #define PAGEORDER_SIZE (PFBAK+PFFOR) static int pmap_prefault_pageorder[] = { -PAGE_SIZE, PAGE_SIZE, -2 * PAGE_SIZE, 2 * PAGE_SIZE }; void pmap_prefault(pmap, addra, entry, object) pmap_t pmap; vm_offset_t addra; vm_map_entry_t entry; vm_object_t object; { int i; vm_offset_t starta; vm_offset_t addr; vm_pindex_t pindex; vm_page_t m, mpte; if (entry->object.vm_object != object) return; if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) return; starta = addra - PFBAK * PAGE_SIZE; if (starta < entry->start) { starta = entry->start; } else if (starta > addra) { starta = 0; } mpte = NULL; for (i = 0; i < PAGEORDER_SIZE; i++) { vm_object_t lobject; unsigned *pte; addr = addra + pmap_prefault_pageorder[i]; if (addr < starta || addr >= entry->end) continue; if ((*pmap_pde(pmap, addr)) == NULL) continue; pte = (unsigned *) vtopte(addr); if (*pte) continue; pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT; lobject = object; for (m = vm_page_lookup(lobject, pindex); (!m && (lobject->type == OBJT_DEFAULT) && (lobject->backing_object)); lobject = lobject->backing_object) { if (lobject->backing_object_offset & PAGE_MASK) break; pindex += (lobject->backing_object_offset >> PAGE_SHIFT); m = vm_page_lookup(lobject->backing_object, pindex); } /* * give-up when a page is not in memory */ if (m == NULL) break; if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (m->busy == 0) && (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((m->queue - m->pc) == PQ_CACHE) { vm_page_deactivate(m); } m->flags |= PG_BUSY; mpte = pmap_enter_quick(pmap, addr, VM_PAGE_TO_PHYS(m), mpte); m->flags |= PG_MAPPED; PAGE_WAKEUP(m); } } } /* * Routine: pmap_change_wiring * Function: Change the wiring attribute for a map/virtual-address * pair. * In/out conditions: * The mapping must already exist in the pmap. */ void pmap_change_wiring(pmap, va, wired) register pmap_t pmap; vm_offset_t va; boolean_t wired; { register unsigned *pte; if (pmap == NULL) return; pte = pmap_pte(pmap, va); if (wired && !pmap_pte_w(pte)) pmap->pm_stats.wired_count++; else if (!wired && pmap_pte_w(pte)) pmap->pm_stats.wired_count--; /* * Wiring is not a hardware characteristic so there is no need to * invalidate TLB. */ pmap_pte_set_w(pte, wired); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) pmap_t dst_pmap, src_pmap; vm_offset_t dst_addr; vm_size_t len; vm_offset_t src_addr; { vm_offset_t addr; vm_offset_t end_addr = src_addr + len; vm_offset_t pdnxt; unsigned src_frame, dst_frame; if (dst_addr != src_addr) return; src_frame = ((unsigned) src_pmap->pm_pdir[PTDPTDI]) & PG_FRAME; if (src_frame != (((unsigned) PTDpde) & PG_FRAME)) { return; } dst_frame = ((unsigned) dst_pmap->pm_pdir[PTDPTDI]) & PG_FRAME; if (dst_frame != (((unsigned) APTDpde) & PG_FRAME)) { APTDpde = (pd_entry_t) (dst_frame | PG_RW | PG_V); invltlb(); } for(addr = src_addr; addr < end_addr; addr = pdnxt) { unsigned *src_pte, *dst_pte; vm_page_t dstmpte, srcmpte; vm_offset_t srcptepaddr; unsigned ptepindex; if (addr >= UPT_MIN_ADDRESS) panic("pmap_copy: invalid to pmap_copy page tables\n"); pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1)); ptepindex = addr >> PDRSHIFT; srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[ptepindex]; if (srcptepaddr == 0) continue; if (srcptepaddr & PG_PS) { if (dst_pmap->pm_pdir[ptepindex] == 0) { dst_pmap->pm_pdir[ptepindex] = (pd_entry_t) srcptepaddr; dst_pmap->pm_stats.resident_count += NBPDR; } continue; } srcmpte = vm_page_lookup(src_pmap->pm_pteobj, ptepindex); if ((srcmpte->hold_count == 0) || (srcmpte->flags & PG_BUSY)) continue; if (pdnxt > end_addr) pdnxt = end_addr; src_pte = (unsigned *) vtopte(addr); dst_pte = (unsigned *) avtopte(addr); while (addr < pdnxt) { unsigned ptetemp; ptetemp = *src_pte; /* * we only virtual copy managed pages */ if ((ptetemp & PG_MANAGED) != 0) { /* * We have to check after allocpte for the * pte still being around... allocpte can * block. */ dstmpte = pmap_allocpte(dst_pmap, addr); if ((*dst_pte == 0) && (ptetemp = *src_pte)) { /* * Clear the modified and * accessed (referenced) bits * during the copy. */ *dst_pte = ptetemp & ~(PG_M|PG_A); dst_pmap->pm_stats.resident_count++; pmap_insert_entry(dst_pmap, addr, dstmpte, (ptetemp & PG_FRAME)); } else { pmap_unwire_pte_hold(dst_pmap, dstmpte); } if (dstmpte->hold_count >= srcmpte->hold_count) break; } addr += PAGE_SIZE; ++src_pte; ++dst_pte; } } } /* * Routine: pmap_kernel * Function: * Returns the physical map handle for the kernel. */ pmap_t pmap_kernel() { return (kernel_pmap); } /* * pmap_zero_page zeros the specified (machine independent) * page by mapping the page into virtual memory and using * bzero to clear its contents, one machine dependent page * at a time. */ void pmap_zero_page(phys) vm_offset_t phys; { if (*(int *) CMAP2) panic("pmap_zero_page: CMAP busy"); *(int *) CMAP2 = PG_V | PG_RW | (phys & PG_FRAME); bzero(CADDR2, PAGE_SIZE); *(int *) CMAP2 = 0; invltlb_1pg((vm_offset_t) CADDR2); } /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ void pmap_copy_page(src, dst) vm_offset_t src; vm_offset_t dst; { if (*(int *) CMAP1 || *(int *) CMAP2) panic("pmap_copy_page: CMAP busy"); *(int *) CMAP1 = PG_V | PG_RW | (src & PG_FRAME); *(int *) CMAP2 = PG_V | PG_RW | (dst & PG_FRAME); bcopy(CADDR1, CADDR2, PAGE_SIZE); *(int *) CMAP1 = 0; *(int *) CMAP2 = 0; invltlb_2pg( (vm_offset_t) CADDR1, (vm_offset_t) CADDR2); } /* * Routine: pmap_pageable * Function: * Make the specified pages (by pmap, offset) * pageable (or not) as requested. * * A page which is not pageable may not take * a fault; therefore, its page table entry * must remain valid for the duration. * * This routine is merely advisory; pmap_enter * will specify that these pages are to be wired * down (or not) as appropriate. */ void pmap_pageable(pmap, sva, eva, pageable) pmap_t pmap; vm_offset_t sva, eva; boolean_t pageable; { } /* * this routine returns true if a physical page resides * in the given pmap. */ boolean_t pmap_page_exists(pmap, pa) pmap_t pmap; vm_offset_t pa; { register pv_entry_t pv; pv_table_t *ppv; int s; if (!pmap_is_managed(pa)) return FALSE; s = splvm(); ppv = pa_to_pvh(pa); /* * Not found, check current mappings returning immediately if found. */ for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { if (pv->pv_pmap == pmap) { splx(s); return TRUE; } } splx(s); return (FALSE); } #define PMAP_REMOVE_PAGES_CURPROC_ONLY /* * Remove all pages from specified address space * this aids process exit speeds. Also, this code * is special cased for current process only, but * can have the more generic (and slightly slower) * mode enabled. This is much faster than pmap_remove * in the case of running down an entire address space. */ void pmap_remove_pages(pmap, sva, eva) pmap_t pmap; vm_offset_t sva, eva; { unsigned *pte, tpte; pv_table_t *ppv; pv_entry_t pv, npv; int s; #if PMAP_PVLIST #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) { printf("warning: pmap_remove_pages called with non-current pmap\n"); return; } #endif s = splvm(); for(pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { if (pv->pv_va >= eva || pv->pv_va < sva) { npv = TAILQ_NEXT(pv, pv_plist); continue; } #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY pte = (unsigned *)vtopte(pv->pv_va); #else pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); #endif tpte = *pte; /* * We cannot remove wired pages from a process' mapping at this time */ if (tpte & PG_W) { npv = TAILQ_NEXT(pv, pv_plist); continue; } *pte = 0; ppv = pa_to_pvh(tpte); pv->pv_pmap->pm_stats.resident_count--; /* * Update the vm_page_t clean and reference bits. */ if (tpte & PG_M) { ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } npv = TAILQ_NEXT(pv, pv_plist); TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); --ppv->pv_list_count; TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); if (TAILQ_FIRST(&ppv->pv_list) == NULL) { ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE); } pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); free_pv_entry(pv); } splx(s); invltlb(); #endif } /* * pmap_testbit tests bits in pte's * note that the testbit/changebit routines are inline, * and a lot of things compile-time evaluate. */ static boolean_t pmap_testbit(pa, bit) register vm_offset_t pa; int bit; { register pv_entry_t pv; pv_table_t *ppv; unsigned *pte; int s; if (!pmap_is_managed(pa)) return FALSE; ppv = pa_to_pvh(pa); if (TAILQ_FIRST(&ppv->pv_list) == NULL) return FALSE; s = splvm(); for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { /* * if the bit being tested is the modified bit, then * mark clean_map and ptes as never * modified. */ if (bit & (PG_A|PG_M)) { if (!pmap_track_modified(pv->pv_va)) continue; } #if defined(PMAP_DIAGNOSTIC) if (!pv->pv_pmap) { printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); continue; } #endif pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (*pte & bit) { splx(s); return TRUE; } } splx(s); return (FALSE); } /* * this routine is used to modify bits in ptes */ static void pmap_changebit(pa, bit, setem) vm_offset_t pa; int bit; boolean_t setem; { register pv_entry_t pv; pv_table_t *ppv; register unsigned *pte; int changed; int s; if (!pmap_is_managed(pa)) return; s = splvm(); changed = 0; ppv = pa_to_pvh(pa); /* * Loop over all current mappings setting/clearing as appropos If * setting RO do we need to clear the VAC? */ for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { /* * don't write protect pager mappings */ if (!setem && (bit == PG_RW)) { if (!pmap_track_modified(pv->pv_va)) continue; } #if defined(PMAP_DIAGNOSTIC) if (!pv->pv_pmap) { printf("Null pmap (cb) at va: 0x%lx\n", pv->pv_va); continue; } #endif pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (setem) { *(int *)pte |= bit; changed = 1; } else { vm_offset_t pbits = *(vm_offset_t *)pte; if (pbits & bit) { changed = 1; if (bit == PG_RW) { if (pbits & PG_M) { ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } *(int *)pte = pbits & ~(PG_M|PG_RW); } else { *(int *)pte = pbits & ~bit; } } } } splx(s); if (changed) invltlb(); } /* * pmap_page_protect: * * Lower the permission for all mappings to a given page. */ void pmap_page_protect(phys, prot) vm_offset_t phys; vm_prot_t prot; { if ((prot & VM_PROT_WRITE) == 0) { if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { pmap_changebit(phys, PG_RW, FALSE); } else { pmap_remove_all(phys); } } } vm_offset_t pmap_phys_address(ppn) int ppn; { return (i386_ptob(ppn)); } /* * pmap_ts_referenced: * * Return the count of reference bits for a page, clearing all of them. * */ int pmap_ts_referenced(vm_offset_t pa) { register pv_entry_t pv; pv_table_t *ppv; unsigned *pte; int s; int rtval = 0; if (!pmap_is_managed(pa)) return FALSE; s = splvm(); ppv = pa_to_pvh(pa); if (TAILQ_FIRST(&ppv->pv_list) == NULL) { splx(s); return 0; } /* * Not found, check current mappings returning immediately if found. */ for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { /* * if the bit being tested is the modified bit, then * mark clean_map and ptes as never * modified. */ if (!pmap_track_modified(pv->pv_va)) continue; pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (pte == NULL) { continue; } if (*pte & PG_A) { rtval++; *pte &= ~PG_A; } } splx(s); if (rtval) { invltlb(); } return (rtval); } /* * pmap_is_modified: * * Return whether or not the specified physical page was modified * in any physical maps. */ boolean_t pmap_is_modified(vm_offset_t pa) { return pmap_testbit((pa), PG_M); } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_offset_t pa) { pmap_changebit((pa), PG_M, FALSE); } /* * pmap_clear_reference: * * Clear the reference bit on the specified physical page. */ void pmap_clear_reference(vm_offset_t pa) { pmap_changebit((pa), PG_A, FALSE); } /* * Miscellaneous support routines follow */ static void i386_protection_init() { register int *kp, prot; kp = protection_codes; for (prot = 0; prot < 8; prot++) { switch (prot) { case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: /* * Read access is also 0. There isn't any execute bit, * so just make it readable. */ case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: *kp++ = 0; break; case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: *kp++ = PG_RW; break; } } } /* * Map a set of physical memory pages into the kernel virtual * address space. Return a pointer to where it is mapped. This * routine is intended to be used for mapping device memory, * NOT real memory. The non-cacheable bits are set on each * mapped page. */ void * pmap_mapdev(pa, size) vm_offset_t pa; vm_size_t size; { vm_offset_t va, tmpva; unsigned *pte; size = roundup(size, PAGE_SIZE); va = kmem_alloc_pageable(kernel_map, size); if (!va) panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); pa = pa & PG_FRAME; for (tmpva = va; size > 0;) { pte = (unsigned *)vtopte(tmpva); *pte = pa | PG_RW | PG_V | pgeflag; size -= PAGE_SIZE; tmpva += PAGE_SIZE; pa += PAGE_SIZE; } invltlb(); return ((void *) va); } /* * perform the pmap work for mincore */ int pmap_mincore(pmap, addr) pmap_t pmap; vm_offset_t addr; { unsigned *ptep, pte; int val = 0; ptep = pmap_pte(pmap, addr); if (ptep == 0) { return 0; } if (pte = *ptep) { vm_offset_t pa; val = MINCORE_INCORE; pa = pte & PG_FRAME; /* * Modified by us */ if (pte & PG_M) val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER; /* * Modified by someone */ else if (PHYS_TO_VM_PAGE(pa)->dirty || pmap_is_modified(pa)) val |= MINCORE_MODIFIED_OTHER; /* * Referenced by us */ if (pte & PG_U) val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER; /* * Referenced by someone */ else if ((PHYS_TO_VM_PAGE(pa)->flags & PG_REFERENCED) || pmap_ts_referenced(pa)) { val |= MINCORE_REFERENCED_OTHER; PHYS_TO_VM_PAGE(pa)->flags |= PG_REFERENCED; } } return val; } void pmap_activate(struct proc *p) { load_cr3(p->p_addr->u_pcb.pcb_cr3 = vtophys(p->p_vmspace->vm_pmap.pm_pdir)); } vm_offset_t pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) { if ((size < NBPDR) || (obj->type != OBJT_DEVICE)) { return addr; } addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1); return addr; } #if defined(PMAP_DEBUG) pmap_pid_dump(int pid) { pmap_t pmap; struct proc *p; int npte = 0; int index; for (p = allproc.lh_first; p != NULL; p = p->p_list.le_next) { if (p->p_pid != pid) continue; if (p->p_vmspace) { int i,j; index = 0; pmap = &p->p_vmspace->vm_pmap; for(i=0;i<1024;i++) { pd_entry_t *pde; unsigned *pte; unsigned base = i << PDRSHIFT; pde = &pmap->pm_pdir[i]; if (pde && pmap_pde_v(pde)) { for(j=0;j<1024;j++) { unsigned va = base + (j << PAGE_SHIFT); if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) { if (index) { index = 0; printf("\n"); } return npte; } pte = pmap_pte_quick( pmap, va); if (pte && pmap_pte_v(pte)) { vm_offset_t pa; vm_page_t m; pa = *(int *)pte; m = PHYS_TO_VM_PAGE((pa & PG_FRAME)); printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x", va, pa, m->hold_count, m->wire_count, m->flags); npte++; index++; if (index >= 2) { index = 0; printf("\n"); } else { printf(" "); } } } } } } } return npte; } #endif #if defined(DEBUG) static void pads __P((pmap_t pm)); static void pmap_pvdump __P((vm_offset_t pa)); /* print address space of pmap*/ static void pads(pm) pmap_t pm; { unsigned va, i, j; unsigned *ptep; if (pm == kernel_pmap) return; for (i = 0; i < 1024; i++) if (pm->pm_pdir[i]) for (j = 0; j < 1024; j++) { va = (i << PDRSHIFT) + (j << PAGE_SHIFT); if (pm == kernel_pmap && va < KERNBASE) continue; if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) continue; ptep = pmap_pte_quick(pm, va); if (pmap_pte_v(ptep)) printf("%x:%x ", va, *(int *) ptep); }; } static void pmap_pvdump(pa) vm_offset_t pa; { pv_table_t *ppv; register pv_entry_t pv; printf("pa %x", pa); ppv = pa_to_pvh(pa); for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { #ifdef used_to_be printf(" -> pmap %x, va %x, flags %x", pv->pv_pmap, pv->pv_va, pv->pv_flags); #endif printf(" -> pmap %x, va %x", pv->pv_pmap, pv->pv_va); pads(pv->pv_pmap); } printf(" "); } #endif Index: head/sys/amd64/include/mptable.h =================================================================== --- head/sys/amd64/include/mptable.h (revision 27483) +++ head/sys/amd64/include/mptable.h (revision 27484) @@ -1,1830 +1,1834 @@ /* * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.14 1997/07/13 00:42:14 smp Exp smp $ + * $Id: mp_machdep.c,v 1.27 1997/07/13 01:22:43 fsmp Exp $ */ #include "opt_smp.h" #include /* for KERNBASE */ #include #include #include #include #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include #include #include #include #include #include #include #include /** TEST_DEFAULT_CONFIG, TEST_CPUSTOP _TEST1 */ #include #include #include /* cngetc() */ #if defined(APIC_IO) #include /* setidt() */ #include /* IPIs */ #include /* IPIs */ #endif /* APIC_IO */ #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define BIOS_BASE (0xf0000) #define BIOS_SIZE (0x10000) #define BIOS_COUNT (BIOS_SIZE/4) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) #define PROCENTRY_FLAG_EN 0x01 #define PROCENTRY_FLAG_BP 0x02 #define IOAPICENTRY_FLAG_EN 0x01 /* MP Floating Pointer Structure */ typedef struct MPFPS { char signature[4]; void *pap; u_char length; u_char spec_rev; u_char checksum; u_char mpfb1; u_char mpfb2; u_char mpfb3; u_char mpfb4; u_char mpfb5; } *mpfps_t; /* MP Configuration Table Header */ typedef struct MPCTH { char signature[4]; u_short base_table_length; u_char spec_rev; u_char checksum; u_char oem_id[8]; u_char product_id[12]; void *oem_table_pointer; u_short oem_table_size; u_short entry_count; void *apic_address; u_short extended_table_length; u_char extended_table_checksum; u_char reserved; } *mpcth_t; typedef struct PROCENTRY { u_char type; u_char apic_id; u_char apic_version; u_char cpu_flags; u_long cpu_signature; u_long feature_flags; u_long reserved1; u_long reserved2; } *proc_entry_ptr; typedef struct BUSENTRY { u_char type; u_char bus_id; char bus_type[6]; } *bus_entry_ptr; typedef struct IOAPICENTRY { u_char type; u_char apic_id; u_char apic_version; u_char apic_flags; void *apic_address; } *io_apic_entry_ptr; typedef struct INTENTRY { u_char type; u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } *int_entry_ptr; /* descriptions of MP basetable entries */ typedef struct BASETABLE_ENTRY { u_char type; u_char length; char name[16]; } basetable_entry; /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MP_START_POST 0x12 #define MP_ANNOUNCE_POST 0x13 #define MPTABLE_PASS1_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define MP_ENABLE_POST 0x16 #define START_ALL_APS_POST 0x17 #define INSTALL_AP_TRAMP_POST 0x18 #define START_AP_POST 0x19 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; /** XXX FIXME: what system files declare these??? */ extern struct region_descriptor r_gdt, r_idt; int mp_ncpus; /* # of CPUs, including BSP */ int mp_naps; /* # of Applications processors */ int mp_nbusses; /* # of busses */ int mp_napics; /* # of IO APICs */ int boot_cpu_id; /* designated BSP */ vm_offset_t cpu_apic_address; vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ u_int32_t cpu_apic_versions[NCPU]; u_int32_t io_apic_versions[NAPIC]; /* * APIC ID logical/physical mapping structures. * We oversize these to simplify boot-time config. */ int cpu_num_to_apic_id[NAPICID]; int io_num_to_apic_id[NAPICID]; int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; /* Boot of AP uses this PTD */ u_int *bootPTD; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t KPTphys; /* Virtual address of per-cpu common_tss */ extern struct i386tss common_tss; /* * Local data and functions. */ static int mp_capable; static u_int boot_address; static u_int base_memory; static int picmode; /* 0: virtual wire mode, 1: PIC mode */ static mpfps_t mpfps; static int search_for_sig(u_int32_t target, int count); static void mp_enable(u_int boot_addr); static int mptable_pass1(void); static int mptable_pass2(void); static void default_mp_table(int type); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); base_memory = basemem * 1024; /* convert to bytes */ boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ if ((base_memory - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } /* * Look for an Intel MP spec table (ie, SMP capable hardware). */ int mp_probe(void) { int x; u_long segment; u_int32_t target; POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { /* search first 1K of EBDA */ target = (u_int32_t) (segment << 4); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } else { /* last 1K of base memory, effective 'top of base' passed in */ target = (u_int32_t) (base_memory - 0x400); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } /* search the BIOS */ target = (u_int32_t) BIOS_BASE; if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) goto found; /* nothing found */ mpfps = (mpfps_t)0; mp_capable = 0; return 0; found: /* calculate needed resources */ mpfps = (mpfps_t)x; if (mptable_pass1()) panic("you must reconfigure your kernel"); /* flag fact that we are running multiple processors */ mp_capable = 1; return 1; } /* * Startup the SMP processors. */ void mp_start(void) { POSTCODE(MP_START_POST); /* look for MP capable motherboard */ if (mp_capable) mp_enable(boot_address); else panic("MP hardware not found!"); } /* * Print various information about the SMP system hardware and setup. */ void mp_announce(void) { int x; POSTCODE(MP_ANNOUNCE_POST); printf("FreeBSD/SMP: Multiprocessor motherboard\n"); printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); printf(", version: 0x%08x", cpu_apic_versions[0]); printf(", at 0x%08x\n", cpu_apic_address); for (x = 1; x <= mp_naps; ++x) { printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); printf(", version: 0x%08x", cpu_apic_versions[x]); printf(", at 0x%08x\n", cpu_apic_address); } #if defined(APIC_IO) for (x = 0; x < mp_napics; ++x) { printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); printf(", version: 0x%08x", io_apic_versions[x]); printf(", at 0x%08x\n", io_apic_address[x]); } #else printf(" Warning: APIC I/O disabled\n"); #endif /* APIC_IO */ } /* * AP cpu's call this to sync up protected mode. */ void init_secondary(void) { int gsel_tss, slot; r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; r_gdt.rd_base = (int) gdt; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); slot = NGDT + cpuid; gsel_tss = GSEL(slot, SEL_KPL); gdt[slot].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ PTD[0] = 0; + pmap_set_opt((unsigned *)PTD); + invltlb(); } #if defined(APIC_IO) /* * Final configuration of the BSP's local APIC: * - disable 'pic mode'. * - disable 'virtual wire mode'. * - enable NMI. */ void bsp_apic_configure(void) { u_char byte; u_int32_t temp; /* leave 'pic mode' if necessary */ if (picmode) { outb(0x22, 0x70); /* select IMCR */ byte = inb(0x23); /* current contents */ byte |= 0x01; /* mask external INTR */ outb(0x23, byte); /* disconnect 8259s/NMI */ } /* mask lint0 (the 8259 'virtual wire' connection) */ temp = lapic.lvt_lint0; temp |= APIC_LVT_M; /* set the mask */ lapic.lvt_lint0 = temp; /* setup lint1 to handle NMI */ temp = lapic.lvt_lint1; temp &= ~APIC_LVT_M; /* clear the mask */ lapic.lvt_lint1 = temp; if (bootverbose) apic_dump(); } #endif /* APIC_IO */ /******************************************************************* * local functions and data */ /* * start the SMP system */ static void mp_enable(u_int boot_addr) { int x; #if defined(APIC_IO) int apic; u_int ux; #endif /* APIC_IO */ POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); invltlb(); /* examine the MP table for needed info, uses physical addresses */ x = mptable_pass2(); *(int *)PTD = 0; invltlb(); /* can't process default configs till the CPU APIC is pmapped */ if (x) default_mp_table(x); #if defined(APIC_IO) /* fill the LOGICAL io_apic_versions table */ for (apic = 0; apic < mp_napics; ++apic) { ux = io_apic_read(apic, IOAPIC_VER); io_apic_versions[apic] = ux; } /* program each IO APIC in the system */ for (apic = 0; apic < mp_napics; ++apic) if (io_apic_setup(apic) < 0) panic("IO APIC setup failure"); /* install a 'Spurious INTerrupt' vector */ setidt(XSPURIOUSINT_OFFSET, Xspuriousint, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #if defined(TEST_CPUSTOP) /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_CPUSTOP */ #if defined(TEST_TEST1) /* install a 'Spurious INTerrupt' vector */ setidt(XTEST1_OFFSET, Xtest1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_TEST1 */ #endif /* APIC_IO */ /* start each Application Processor */ start_all_aps(boot_addr); /* * The init process might be started on a different CPU now, * and the boot CPU might not call prepare_usermode to get * cr0 correctly configured. Thus we initialize cr0 here. */ load_cr0(rcr0() | CR0_WP | CR0_AM); } /* * look for the MP spec signature */ /* string defined by the Intel MP Spec as identifying the MP table */ #define MP_SIG 0x5f504d5f /* _MP_ */ #define NEXT(X) ((X) += 4) static int search_for_sig(u_int32_t target, int count) { int x; u_int32_t *addr = (u_int32_t *) (KERNBASE + target); for (x = 0; x < count; NEXT(x)) if (addr[x] == MP_SIG) /* make array index a byte index */ return (target + (x * sizeof(u_int32_t))); return -1; } static basetable_entry basetable_entry_types[] = { {0, 20, "Processor"}, {1, 8, "Bus"}, {2, 8, "I/O APIC"}, {3, 8, "I/O INT"}, {4, 8, "Local INT"} }; typedef struct BUSDATA { u_char bus_id; enum busTypes bus_type; } bus_datum; typedef struct INTDATA { u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } io_int, local_int; typedef struct BUSTYPENAME { u_char type; char name[7]; } bus_type_name; static bus_type_name bus_type_table[] = { {CBUS, "CBUS"}, {CBUSII, "CBUSII"}, {EISA, "EISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {ISA, "ISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {PCI, "PCI"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {XPRESS, "XPRESS"}, {UNKNOWN_BUSTYPE, "---"} }; /* from MP spec v1.4, table 5-1 */ static int default_data[7][5] = { /* nbus, id0, type0, id1, type1 */ {1, 0, ISA, 255, 255}, {1, 0, EISA, 255, 255}, {1, 0, EISA, 255, 255}, {0, 255, 255, 255, 255},/* MCA not supported */ {2, 0, ISA, 1, PCI}, {2, 0, EISA, 1, PCI}, {0, 255, 255, 255, 255} /* MCA not supported */ }; /* the bus data */ bus_datum bus_data[NBUS]; /* the IO INT data, one entry per possible APIC INTerrupt */ io_int io_apic_ints[NINTR]; static int nintrs; static void fix_mp_table __P((void)); static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); /* * 1st pass on motherboard's Intel MP specification table. * * initializes: * mp_ncpus = 1 * * determines: * cpu_apic_address (common to all CPUs) * io_apic_address[N] * mp_naps * mp_nbusses * mp_napics * nintrs */ static int mptable_pass1(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int mustpanic; POSTCODE(MPTABLE_PASS1_POST); mustpanic = 0; /* clear various tables */ for (x = 0; x < NAPICID; ++x) { io_apic_address[x] = ~0; /* IO APIC address table */ } /* init everything to empty */ mp_naps = 0; mp_nbusses = 0; mp_napics = 0; nintrs = 0; /* check for use of 'default' configuration */ if (mpfps->mpfb1 != 0) { /* use default addresses */ cpu_apic_address = DEFAULT_APIC_BASE; io_apic_address[0] = DEFAULT_IO_APIC_BASE; /* fill in with defaults */ mp_naps = 2; /* includes BSP */ mp_nbusses = default_data[mpfps->mpfb1 - 1][0]; #if defined(APIC_IO) mp_napics = 1; nintrs = 16; #endif /* APIC_IO */ } else { if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); cpu_apic_address = (vm_offset_t) cth->apic_address; /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; while (count--) { switch (type = *(u_char *) position) { case 0: /* processor_entry */ if (((proc_entry_ptr)position)->cpu_flags & PROCENTRY_FLAG_EN) ++mp_naps; break; case 1: /* bus_entry */ ++mp_nbusses; break; case 2: /* io_apic_entry */ if (((io_apic_entry_ptr)position)->apic_flags & IOAPICENTRY_FLAG_EN) io_apic_address[mp_napics++] = (vm_offset_t)((io_apic_entry_ptr) position)->apic_address; break; case 3: /* int_entry */ ++nintrs; break; case 4: /* int_entry */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char*)position += basetable_entry_types[type].length; } } /* qualify the numbers */ if (mp_naps > NCPU) printf("Warning: only using %d of %d available CPUs!\n", NCPU, mp_naps); #if 0 /** XXX we consider this legal now (but should we?) */ mustpanic = 1; #endif if (mp_nbusses > NBUS) { printf("found %d busses, increase NBUS\n", mp_nbusses); mustpanic = 1; } if (mp_napics > NAPIC) { printf("found %d apics, increase NAPIC\n", mp_napics); mustpanic = 1; } if (nintrs > NINTR) { printf("found %d intrs, increase NINTR\n", nintrs); mustpanic = 1; } /* * Count the BSP. * This is also used as a counter while starting the APs. */ mp_ncpus = 1; --mp_naps; /* subtract the BSP */ return mustpanic; } /* * 2nd pass on motherboard's Intel MP specification table. * * sets: * boot_cpu_id * ID_TO_IO(N), phy APIC ID to log CPU/IO table * CPU_TO_ID(N), logical CPU to APIC ID table * IO_TO_ID(N), logical IO to APIC ID table * bus_data[N] * io_apic_ints[N] */ static int mptable_pass2(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int apic, bus, cpu, intr; POSTCODE(MPTABLE_PASS2_POST); /* clear various tables */ for (x = 0; x < NAPICID; ++x) { ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ } /* clear bus data table */ for (x = 0; x < NBUS; ++x) bus_data[x].bus_id = 0xff; /* clear IO APIC INT table */ for (x = 0; x < NINTR; ++x) io_apic_ints[x].int_type = 0xff; /* setup the cpu/apic mapping arrays */ boot_cpu_id = -1; /* record whether PIC or virtual-wire mode */ picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; /* check for use of 'default' configuration */ #if defined(TEST_DEFAULT_CONFIG) return TEST_DEFAULT_CONFIG; #else if (mpfps->mpfb1 != 0) return mpfps->mpfb1; /* return default configuration type */ #endif /* TEST_DEFAULT_CONFIG */ if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; apic = bus = intr = 0; cpu = 1; /* pre-count the BSP */ while (count--) { switch (type = *(u_char *) position) { case 0: if (processor_entry(position, cpu)) ++cpu; break; case 1: if (bus_entry(position, bus)) ++bus; break; case 2: if (io_apic_entry(position, apic)) ++apic; break; case 3: if (int_entry(position, intr)) ++intr; break; case 4: /* int_entry(position); */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char *) position += basetable_entry_types[type].length; } if (boot_cpu_id == -1) panic("NO BSP found!"); /* post scan cleanup */ fix_mp_table(); /* report fact that its NOT a default configuration */ return 0; } /* * parse an Intel MP specification table */ static void fix_mp_table(void) { int x; int id; int bus_0; int bus_pci; int num_pci_bus; /* * Fix mis-numbering of the PCI bus and its INT entries if the BIOS * did it wrong. The MP spec says that when more than 1 PCI bus * exists the BIOS must begin with bus entries for the PCI bus and use * actual PCI bus numbering. This implies that when only 1 PCI bus * exists the BIOS can choose to ignore this ordering, and indeed many * MP motherboards do ignore it. This causes a problem when the PCI * sub-system makes requests of the MP sub-system based on PCI bus * numbers. So here we look for the situation and renumber the * busses and associated INTs in an effort to "make it right". */ /* find bus 0, PCI bus, count the number of PCI busses */ for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_id == 0) { bus_0 = x; } if (bus_data[x].bus_type == PCI) { ++num_pci_bus; bus_pci = x; } } /* * bus_0 == slot of bus with ID of 0 * bus_pci == slot of last PCI bus encountered */ /* check the 1 PCI bus case for sanity */ if (num_pci_bus == 1) { /* if it is number 0 all is well */ if (bus_data[bus_pci].bus_id == 0) return; /* mis-numbered, swap with whichever bus uses slot 0 */ /* swap the bus entry types */ bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; /* swap each relavant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { io_apic_ints[x].src_bus_id = 0; } else if (io_apic_ints[x].src_bus_id == 0) { io_apic_ints[x].src_bus_id = id; } } } /* sanity check if more than 1 PCI bus */ else if (num_pci_bus > 1) { for (x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_type != PCI) continue; if (bus_data[x].bus_id >= num_pci_bus) panic("bad PCI bus numbering"); } } } static int processor_entry(proc_entry_ptr entry, int cpu) { /* check for usability */ if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) return 0; /* check for BSP flag */ if (entry->cpu_flags & PROCENTRY_FLAG_BP) { boot_cpu_id = entry->apic_id; CPU_TO_ID(0) = entry->apic_id; ID_TO_CPU(entry->apic_id) = 0; return 0; /* its already been counted */ } /* add another AP to list, if less than max number of CPUs */ else { CPU_TO_ID(cpu) = entry->apic_id; ID_TO_CPU(entry->apic_id) = cpu; return 1; } } static int bus_entry(bus_entry_ptr entry, int bus) { int x; char c, name[8]; /* encode the name into an index */ for (x = 0; x < 6; ++x) { if ((c = entry->bus_type[x]) == ' ') break; name[x] = c; } name[x] = '\0'; if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) panic("unknown bus type: '%s'", name); bus_data[bus].bus_id = entry->bus_id; bus_data[bus].bus_type = x; return 1; } static int io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; IO_TO_ID(apic) = entry->apic_id; ID_TO_IO(entry->apic_id) = apic; return 1; } static int lookup_bus_type(char *name) { int x; for (x = 0; x < MAX_BUSTYPE; ++x) if (strcmp(bus_type_table[x].name, name) == 0) return bus_type_table[x].type; return UNKNOWN_BUSTYPE; } static int int_entry(int_entry_ptr entry, int intr) { io_apic_ints[intr].int_type = entry->int_type; io_apic_ints[intr].int_flags = entry->int_flags; io_apic_ints[intr].src_bus_id = entry->src_bus_id; io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; } static int apic_int_is_bus_type(int intr, int bus_type) { int bus; for (bus = 0; bus < mp_nbusses; ++bus) if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) && ((int) bus_data[bus].bus_type == bus_type)) return 1; return 0; } /* * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { int isa_irq; int apic_pin; #if defined(SKIP_IRQ15_REDIRECT) if (isa_mask == (1 << 15)) { printf("skipping ISA IRQ15 redirect\n"); return isa_mask; } #endif /* SKIP_IRQ15_REDIRECT */ isa_irq = ffs(isa_mask); /* find its bit position */ if (isa_irq == 0) /* doesn't exist */ return 0; --isa_irq; /* make it zero based */ apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ if (apic_pin == -1) return 0; return (1 << apic_pin); /* convert pin# to a mask */ } /* * Determine which APIC pin an ISA/EISA INT is attached to. */ #define INTTYPE(I) (io_apic_ints[(I)].int_type) #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) int isa_apic_pin(int isa_irq) { int intr; #if defined(SMP_TIMER_NC) if (isa_irq == 0) return -1; #endif /* SMP_TIMER_NC */ for (intr = 0; intr < nintrs; ++intr) { /* check each record */ if (INTTYPE(intr) == 0) { /* standard INT */ if (SRCBUSIRQ(intr) == isa_irq) { if (apic_int_is_bus_type(intr, ISA) || apic_int_is_bus_type(intr, EISA)) return INTPIN(intr); /* found */ } } } return -1; /* NOT found */ } #undef SRCBUSIRQ /* * Determine which APIC pin a PCI INT is attached to. */ #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) int pci_apic_pin(int pciBus, int pciDevice, int pciInt) { int intr; --pciInt; /* zero based */ for (intr = 0; intr < nintrs; ++intr) /* check each record */ if ((INTTYPE(intr) == 0) /* standard INT */ && (SRCBUSID(intr) == pciBus) && (SRCBUSDEVICE(intr) == pciDevice) && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ if (apic_int_is_bus_type(intr, PCI)) return INTPIN(intr); /* exact match */ return -1; /* NOT found */ } #undef SRCBUSLINE #undef SRCBUSDEVICE #undef SRCBUSID #undef INTPIN #undef INTTYPE /* * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. * * XXX FIXME: * Exactly what this means is unclear at this point. It is a solution * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard * could route any of the ISA INTs to upper (>15) IRQ values. But most would * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an * option. */ int undirect_isa_irq(int rirq) { #if defined(READY) printf("Freeing redirected ISA irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); return 0; #endif /* READY */ } /* * Reprogram the MB chipset to NOT redirect a PCI INTerrupt */ int undirect_pci_irq(int rirq) { #if defined(READY) if (bootverbose) printf("Freeing redirected PCI irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else if (bootverbose) printf("Freeing (NOT implemented) redirected PCI irq %d.\n", rirq); return 0; #endif /* READY */ } /* * given a bus ID, return: * the bus type if found * -1 if NOT found */ int apic_bus_type(int id) { int x; for (x = 0; x < mp_nbusses; ++x) if (bus_data[x].bus_id == id) return bus_data[x].bus_type; return -1; } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus ID if found * -1 if NOT found */ int apic_src_bus_id(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_id); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus IRQ if found * -1 if NOT found */ int apic_src_bus_irq(int apic, int pin) { int x; for (x = 0; x < nintrs; x++) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_irq); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated INTerrupt type if found * -1 if NOT found */ int apic_int_type(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_type); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated trigger mode if found * -1 if NOT found */ int apic_trigger(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return ((io_apic_ints[x].int_flags >> 2) & 0x03); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated 'active' level if found * -1 if NOT found */ int apic_polarity(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_flags & 0x03); return -1; /* NOT found */ } /* * set data according to MP defaults * FIXME: probably not complete yet... */ static void default_mp_table(int type) { int ap_cpu_id; #if defined(APIC_IO) u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ #if 0 printf(" MP default config type: %d\n", type); switch (type) { case 1: printf(" bus: ISA, APIC: 82489DX\n"); break; case 2: printf(" bus: EISA, APIC: 82489DX\n"); break; case 3: printf(" bus: EISA, APIC: 82489DX\n"); break; case 4: printf(" bus: MCA, APIC: 82489DX\n"); break; case 5: printf(" bus: ISA+PCI, APIC: Integrated\n"); break; case 6: printf(" bus: EISA+PCI, APIC: Integrated\n"); break; case 7: printf(" bus: MCA+PCI, APIC: Integrated\n"); break; default: printf(" future type\n"); break; /* NOTREACHED */ } #endif /* 0 */ boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; /* BSP */ CPU_TO_ID(0) = boot_cpu_id; ID_TO_CPU(boot_cpu_id) = 0; /* one and only AP */ CPU_TO_ID(1) = ap_cpu_id; ID_TO_CPU(ap_cpu_id) = 1; #if defined(APIC_IO) /* one and only IO APIC */ io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; /* * sanity check, refer to MP spec section 3.6.6, last paragraph * necessary as some hardware isn't properly setting up the IO APIC */ #if defined(REALLY_ANAL_IOAPICID_VALUE) if (io_apic_id != 2) { #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ux &= ~APIC_ID_MASK; /* clear the ID field */ ux |= 0x02000000; /* set it to '2' */ io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ if ((ux & APIC_ID_MASK) != 0x02000000) panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; ID_TO_IO(io_apic_id) = 0; #endif /* APIC_IO */ /* fill out bus entries */ switch (type) { case 1: case 2: case 3: case 5: case 6: bus_data[0].bus_id = default_data[type - 1][1]; bus_data[0].bus_type = default_data[type - 1][2]; bus_data[1].bus_id = default_data[type - 1][3]; bus_data[1].bus_type = default_data[type - 1][4]; break; /* case 4: case 7: MCA NOT supported */ default: /* illegal/reserved */ panic("BAD default MP config: %d", type); /* NOTREACHED */ } #if defined(APIC_IO) /* general cases from MP v1.4, table 5-2 */ for (pin = 0; pin < 16; ++pin) { io_apic_ints[pin].int_type = 0; io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ io_apic_ints[pin].src_bus_id = 0; io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ io_apic_ints[pin].dst_apic_id = io_apic_id; io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ } /* special cases from MP v1.4, table 5-2 */ if (type == 2) { io_apic_ints[2].int_type = 0xff; /* N/C */ io_apic_ints[13].int_type = 0xff; /* N/C */ #if !defined(APIC_MIXED_MODE) /** FIXME: ??? */ panic("sorry, can't support type 2 default yet"); #endif /* APIC_MIXED_MODE */ } else io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ if (type == 7) io_apic_ints[0].int_type = 0xff; /* N/C */ else io_apic_ints[0].int_type = 3; /* vectored 8259 */ #endif /* APIC_IO */ } /* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { int x, i; u_char mpbiosreason; u_long mpbioswarmvec; pd_entry_t newptd; pt_entry_t newpt; int *newpp; POSTCODE(START_ALL_APS_POST); /** * NOTE: this needs further thought: * where does it get released? * should it be set to empy? * * get the initial mp_lock with a count of 1 for the BSP */ mp_lock = 1; /* this uses a LOGICAL cpu ID, ie BSP == 0 */ /* initialize BSP's local APIC */ apic_initialize(); /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* record BSP in CPU map */ all_cpus = 1; /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* HACK HACK HACK !!! */ /* alloc new page table directory */ newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* clone currently active one (ie: IdlePTD) */ bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ /* set up 0 -> 4MB P==V mapping for AP boot */ newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); /* store PTD for this AP */ bootPTD = (pd_entry_t)vtophys(newptd); /* alloc new page table page */ newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* set the new PTD's private page to point there */ newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt); /* install self referential entry */ newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd); /* get a new private data page */ newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ newpt[0] = PG_V | PG_RW | vtophys(newpp); /* wire the ptp into itself for access */ newpt[1] = PG_V | PG_RW | vtophys(newpt); /* and the local apic */ newpt[2] = SMP_prvpt[2]; /* and the IO apic mapping[s] */ for (i = 16; i < 32; i++) newpt[i] = SMP_prvpt[i]; /* prime data page for it to use */ newpp[0] = x; /* cpuid */ newpp[1] = 0; /* curproc */ newpp[2] = 0; /* curpcb */ newpp[3] = 0; /* npxproc */ newpp[4] = 0; /* runtime.tv_sec */ newpp[5] = 0; /* runtime.tv_usec */ newpp[6] = x << 24; /* cpu_lockid */ /* XXX NOTE: ABANDON bootPTD for now!!!! */ /* END REVOLTING HACKERY */ /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ /* record its version info */ cpu_apic_versions[x] = cpu_apic_versions[0]; all_cpus |= (1 << x); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ other_cpus = all_cpus & ~(1 << cpuid); /* fill in our (BSP) APIC version */ cpu_apic_versions[0] = lapic.version; /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); + + pmap_set_opt_bsp(); /* number of APs actually started */ return mp_ncpus - 1; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(u_int boot_addr) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_addr + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_addr + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; } /* * this function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It ain't pretty, * but it seems to work. */ static int start_ap(int logical_cpu, u_int boot_addr) { int physical_cpu; int vector; int cpus; u_long icr_lo, icr_hi; POSTCODE(START_AP_POST); /* get the PHYSICAL APIC ID# */ physical_cpu = CPU_TO_ID(logical_cpu); /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_ncpus; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* setup the address for the target AP */ icr_hi = lapic.icr_hi & ~APIC_ID_MASK; icr_hi |= (physical_cpu << 24); lapic.icr_hi = icr_hi; /* do an INIT IPI: assert RESET */ icr_lo = lapic.icr_lo & 0xfff00000; lapic.icr_lo = icr_lo | 0x0000c500; /* wait for pending status end */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* do an INIT IPI: deassert RESET */ lapic.icr_lo = icr_lo | 0x00008500; /* wait for pending status end */ u_sleep(10000); /* wait ~10mS */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* wait for it to start */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) if (mp_ncpus > cpus) return 1; /* return SUCCESS */ return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's * * XXX: Needs to handshake and wait for completion before proceding. */ void smp_invltlb(void) { #if defined(APIC_IO) if (smp_active && invltlb_ok) all_but_self_ipi(XINVLTLB_OFFSET); #endif /* APIC_IO */ } void invlpg(u_int addr) { __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); /* send a message to the other CPUs */ smp_invltlb(); } void invltlb(void) { u_long temp; /* * This should be implemented as load_cr3(rcr3()) when load_cr3() is * inlined. */ __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); /* send a message to the other CPUs */ smp_invltlb(); } #if defined(TEST_CPUSTOP) #if defined(DEBUG_CPUSTOP) void db_printf __P((const char *fmt, ...)); #endif /* DEBUG_CPUSTOP */ /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int stop_cpus( u_int map ) { if (!smp_active) return 0; /* send IPI to all CPUs in map */ #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d stopping CPUs: 0x%08x\n", cpuid, map); #endif /* DEBUG_CPUSTOP */ stopped_cpus = 0; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); #if defined(DEBUG_CPUSTOP) db_printf(" spin\n"); #endif /* DEBUG_CPUSTOP */ while (stopped_cpus != map) /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" spun\nstopped\n"); #endif /* DEBUG_CPUSTOP */ return 1; } /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus( u_int map ) { if (!smp_active) return 0; #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d restarting CPUs: 0x%08x (0x%08x)\n", cpuid, map, stopped_cpus); #endif /* DEBUG_CPUSTOP */ started_cpus = map; /* signal other cpus to restart */ while (started_cpus) /* wait for each to clear its bit */ /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" restarted\n"); #endif /* DEBUG_CPUSTOP */ return 1; } #endif /** TEST_CPUSTOP */ Index: head/sys/i386/i386/locore.s =================================================================== --- head/sys/i386/i386/locore.s (revision 27483) +++ head/sys/i386/i386/locore.s (revision 27484) @@ -1,991 +1,996 @@ /*- * Copyright (c) 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)locore.s 7.3 (Berkeley) 5/13/91 - * $Id: locore.s,v 1.90 1997/06/27 23:19:43 fsmp Exp $ + * $Id: locore.s,v 1.91 1997/07/15 11:07:32 kato Exp $ * * originally from: locore.s, by William F. Jolitz * * Substantially rewritten by David Greenman, Rod Grimes, * Bruce Evans, Wolfgang Solfrank, Poul-Henning Kamp * and many others. */ #include "apm.h" #include "opt_cpu.h" #include "opt_ddb.h" #include "opt_userconfig.h" #include "opt_smp.h" #include #include #include #include #include #include #include #include #ifdef SMP #include #endif /* SMP */ #include "assym.s" /* * XXX * * Note: This version greatly munged to avoid various assembler errors * that may be fixed in newer versions of gas. Perhaps newer versions * will have more pleasant appearance. */ /* * PTmap is recursive pagemap at top of virtual address space. * Within PTmap, the page directory can be found (third indirection). */ .globl _PTmap,_PTD,_PTDpde .set _PTmap,(PTDPTDI << PDRSHIFT) .set _PTD,_PTmap + (PTDPTDI * PAGE_SIZE) .set _PTDpde,_PTD + (PTDPTDI * PDESIZE) /* * APTmap, APTD is the alternate recursive pagemap. * It's used when modifying another process's page tables. */ .globl _APTmap,_APTD,_APTDpde .set _APTmap,APTDPTDI << PDRSHIFT .set _APTD,_APTmap + (APTDPTDI * PAGE_SIZE) .set _APTDpde,_PTD + (APTDPTDI * PDESIZE) #ifdef SMP .globl _SMP_prvstart .set _SMP_prvstart,(MPPTDI << PDRSHIFT) .globl _SMP_prvpage,_SMP_prvpt,_lapic,_SMP_ioapic .set _SMP_prvpage,_SMP_prvstart .set _SMP_prvpt,_SMP_prvstart + PAGE_SIZE .set _lapic,_SMP_prvstart + (2 * PAGE_SIZE) .set _SMP_ioapic,_SMP_prvstart + (16 * PAGE_SIZE) .globl _cpuid,_curproc,_curpcb,_npxproc,_runtime,_cpu_lockid .globl _common_tss,_other_cpus .set _cpuid,_SMP_prvpage+0 .set _curproc,_SMP_prvpage+4 .set _curpcb,_SMP_prvpage+8 .set _npxproc,_SMP_prvpage+12 .set _runtime,_SMP_prvpage+16 /* 8 bytes struct timeval */ .set _cpu_lockid,_SMP_prvpage+24 .set _common_tss,_SMP_prvpage+28 /* 104 bytes long, next = 132 */ .set _other_cpus,_SMP_prvpage+132 /* bitmap of available CPUs, excluding ourself */ /* Fetch the .set's for the local apic */ #include "i386/i386/mp_apicdefs.s" #endif /* * Globals */ .data ALIGN_DATA /* just to be sure */ .globl HIDENAME(tmpstk) .space 0x2000 /* space for tmpstk - temporary stack */ HIDENAME(tmpstk): .globl _boothowto,_bootdev .globl _cpu,_cpu_vendor,_cpu_id,_bootinfo .globl _cpu_high, _cpu_feature _cpu: .long 0 /* are we 386, 386sx, or 486 */ _cpu_id: .long 0 /* stepping ID */ _cpu_high: .long 0 /* highest arg to CPUID */ _cpu_feature: .long 0 /* features */ _cpu_vendor: .space 20 /* CPU origin code */ _bootinfo: .space BOOTINFO_SIZE /* bootinfo that we can handle */ _KERNend: .long 0 /* phys addr end of kernel (just after bss) */ physfree: .long 0 /* phys addr of next free page */ #ifdef SMP cpu0pp: .long 0 /* phys addr cpu0 private pg */ cpu0pt: .long 0 /* phys addr cpu0 private pt */ .globl _cpu0prvpage,_cpu0prvpt _cpu0prvpage: .long 0 /* relocated version */ _cpu0prvpt: .long 0 /* relocated version */ #endif /* SMP */ .globl _IdlePTD _IdlePTD: .long 0 /* phys addr of kernel PTD */ #ifdef SMP .globl _KPTphys #endif _KPTphys: .long 0 /* phys addr of kernel page tables */ .globl _proc0paddr _proc0paddr: .long 0 /* address of proc 0 address space */ p0upa: .long 0 /* phys addr of proc0's UPAGES */ #ifdef BDE_DEBUGGER .globl _bdb_exists /* flag to indicate BDE debugger is present */ _bdb_exists: .long 0 #endif /********************************************************************** * * Some handy macros * */ #define R(foo) ((foo)-KERNBASE) #define ALLOCPAGES(foo) \ movl R(physfree), %esi ; \ movl $((foo)*PAGE_SIZE), %eax ; \ addl %esi, %eax ; \ movl %eax, R(physfree) ; \ movl %esi, %edi ; \ movl $((foo)*PAGE_SIZE),%ecx ; \ xorl %eax,%eax ; \ cld ; \ rep ; \ stosb /* * fillkpt * eax = page frame address * ebx = index into page table * ecx = how many pages to map * base = base address of page dir/table * prot = protection bits */ #define fillkpt(base, prot) \ shll $2,%ebx ; \ addl base,%ebx ; \ orl $PG_V,%eax ; \ orl prot,%eax ; \ 1: movl %eax,(%ebx) ; \ addl $PAGE_SIZE,%eax ; /* increment physical address */ \ addl $4,%ebx ; /* next pte */ \ loop 1b /* * fillkptphys(prot) * eax = physical address * ecx = how many pages to map * prot = protection bits */ #define fillkptphys(prot) \ movl %eax, %ebx ; \ shrl $PAGE_SHIFT, %ebx ; \ fillkpt(R(_KPTphys), prot) .text /********************************************************************** * * This is where the bootblocks start us, set the ball rolling... * */ NON_GPROF_ENTRY(btext) #ifdef PC98 jmp 1f .globl _pc98_system_parameter .org 0x400 _pc98_system_parameter: .space 0x240 /* BIOS parameter block */ 1: /* save SYSTEM PARAMETER for resume (NS/T or other) */ movl $0xa1000,%esi movl $0x100000,%edi movl $0x0630,%ecx cld rep movsb #else /* IBM-PC */ #ifdef BDE_DEBUGGER #ifdef BIOS_STEALS_3K cmpl $0x0375c339,0x95504 #else cmpl $0x0375c339,0x96104 /* XXX - debugger signature */ #endif jne 1f movb $1,R(_bdb_exists) 1: #endif /* Tell the bios to warmboot next time */ movw $0x1234,0x472 #endif /* PC98 */ /* Set up a real frame in case the double return in newboot is executed. */ pushl %ebp movl %esp, %ebp /* Don't trust what the BIOS gives for eflags. */ pushl $PSL_KERNEL popfl /* * Don't trust what the BIOS gives for %fs and %gs. Trust the bootstrap * to set %cs, %ds, %es and %ss. */ mov %ds, %ax mov %ax, %fs mov %ax, %gs call recover_bootinfo /* Get onto a stack that we can trust. */ /* * XXX this step is delayed in case recover_bootinfo needs to return via * the old stack, but it need not be, since recover_bootinfo actually * returns via the old frame. */ movl $R(HIDENAME(tmpstk)),%esp #ifdef PC98 testb $0x02,0x100620 /* pc98_machine_type & M_EPSON_PC98 */ jz 3f cmpb $0x0b,0x100624 /* epson_machine_id <= 0x0b */ ja 3f /* count up memory */ movl $0x100000,%eax /* next, talley remaining memory */ movl $0xFFF-0x100,%ecx 1: movl 0(%eax),%ebx /* save location to check */ movl $0xa55a5aa5,0(%eax) /* write test pattern */ cmpl $0xa55a5aa5,0(%eax) /* does not check yet for rollover */ jne 2f movl %ebx,0(%eax) /* restore memory */ addl $PAGE_SIZE,%eax loop 1b 2: subl $0x100000,%eax shrl $17,%eax movb %al,0x100401 3: #endif call identify_cpu /* clear bss */ /* * XXX this should be done a little earlier. * * XXX we don't check that there is memory for our bss and page tables * before using it. * * XXX the boot program somewhat bogusly clears the bss. We still have * to do it in case we were unzipped by kzipboot. Then the boot program * only clears kzipboot's bss. * * XXX the gdt and idt are still somewhere in the boot program. We * depend on the convention that the boot program is below 1MB and we * are above 1MB to keep the gdt and idt away from the bss and page * tables. The idt is only used if BDE_DEBUGGER is enabled. */ movl $R(_end),%ecx movl $R(_edata),%edi subl %edi,%ecx xorl %eax,%eax cld rep stosb #if NAPM > 0 /* * XXX it's not clear that APM can live in the current environonment. * Only pc-relative addressing works. */ call _apm_setup #endif call create_pagetables #ifdef BDE_DEBUGGER /* * Adjust as much as possible for paging before enabling paging so that the * adjustments can be traced. */ call bdb_prepare_paging #endif /* Now enable paging */ movl R(_IdlePTD), %eax movl %eax,%cr3 /* load ptd addr into mmu */ movl %cr0,%eax /* get control word */ orl $CR0_PE|CR0_PG,%eax /* enable paging */ movl %eax,%cr0 /* and let's page NOW! */ #ifdef BDE_DEBUGGER /* * Complete the adjustments for paging so that we can keep tracing through * initi386() after the low (physical) addresses for the gdt and idt become * invalid. */ call bdb_commit_paging #endif pushl $begin /* jump to high virtualized address */ ret /* now running relocated at KERNBASE where the system is linked to run */ begin: /* set up bootstrap stack */ movl _proc0paddr,%esp /* location of in-kernel pages */ addl $UPAGES*PAGE_SIZE,%esp /* bootstrap stack end location */ xorl %eax,%eax /* mark end of frames */ movl %eax,%ebp movl _proc0paddr,%eax movl _IdlePTD, %esi movl %esi,PCB_CR3(%eax) movl physfree, %esi pushl %esi /* value of first for init386(first) */ call _init386 /* wire 386 chip for unix operation */ popl %esi .globl __ucodesel,__udatasel pushl $0 /* unused */ pushl __udatasel /* ss */ pushl $0 /* esp - filled in by execve() */ pushl $PSL_USER /* eflags (IOPL 0, int enab) */ pushl __ucodesel /* cs */ pushl $0 /* eip - filled in by execve() */ subl $(12*4),%esp /* space for rest of registers */ pushl %esp /* call main with frame pointer */ call _main /* autoconfiguration, mountroot etc */ hlt /* never returns to here */ /* * When starting init, call this to configure the process for user * mode. This will be inherited by other processes. */ NON_GPROF_ENTRY(prepare_usermode) /* * Now we've run main() and determined what cpu-type we are, we can * enable write protection and alignment checking on i486 cpus and * above. */ #if defined(I486_CPU) || defined(I586_CPU) || defined(I686_CPU) cmpl $CPUCLASS_386,_cpu_class je 1f movl %cr0,%eax /* get control word */ orl $CR0_WP|CR0_AM,%eax /* enable i486 features */ movl %eax,%cr0 /* and do it */ 1: #endif /* * on return from main(), we are process 1 * set up address space and stack so that we can 'return' to user mode */ movl __ucodesel,%eax movl __udatasel,%ecx #if 0 movl %cx,%ds #endif movl %cx,%es movl %ax,%fs /* double map cs to fs */ movl %cx,%gs /* and ds to gs */ ret /* goto user! */ #define LCALL(x,y) .byte 0x9a ; .long y ; .word x /* * Signal trampoline, copied to top of user stack */ NON_GPROF_ENTRY(sigcode) call SIGF_HANDLER(%esp) lea SIGF_SC(%esp),%eax /* scp (the call may have clobbered the */ /* copy at 8(%esp)) */ pushl %eax pushl %eax /* junk to fake return address */ movl $SYS_sigreturn,%eax /* sigreturn() */ LCALL(0x7,0) /* enter kernel with args on stack */ hlt /* never gets here */ ALIGN_TEXT _esigcode: .data .globl _szsigcode _szsigcode: .long _esigcode-_sigcode .text /********************************************************************** * * Recover the bootinfo passed to us from the boot program * */ recover_bootinfo: /* * This code is called in different ways depending on what loaded * and started the kernel. This is used to detect how we get the * arguments from the other code and what we do with them. * * Old disk boot blocks: * (*btext)(howto, bootdev, cyloffset, esym); * [return address == 0, and can NOT be returned to] * [cyloffset was not supported by the FreeBSD boot code * and always passed in as 0] * [esym is also known as total in the boot code, and * was never properly supported by the FreeBSD boot code] * * Old diskless netboot code: * (*btext)(0,0,0,0,&nfsdiskless,0,0,0); * [return address != 0, and can NOT be returned to] * If we are being booted by this code it will NOT work, * so we are just going to halt if we find this case. * * New uniform boot code: * (*btext)(howto, bootdev, 0, 0, 0, &bootinfo) * [return address != 0, and can be returned to] * * There may seem to be a lot of wasted arguments in here, but * that is so the newer boot code can still load very old kernels * and old boot code can load new kernels. */ /* * The old style disk boot blocks fake a frame on the stack and * did an lret to get here. The frame on the stack has a return * address of 0. */ cmpl $0,4(%ebp) je olddiskboot /* * We have some form of return address, so this is either the * old diskless netboot code, or the new uniform code. That can * be detected by looking at the 5th argument, if it is 0 * we are being booted by the new uniform boot code. */ cmpl $0,24(%ebp) je newboot /* * Seems we have been loaded by the old diskless boot code, we * don't stand a chance of running as the diskless structure * changed considerably between the two, so just halt. */ hlt /* * We have been loaded by the new uniform boot code. * Let's check the bootinfo version, and if we do not understand * it we return to the loader with a status of 1 to indicate this error */ newboot: movl 28(%ebp),%ebx /* &bootinfo.version */ movl BI_VERSION(%ebx),%eax cmpl $1,%eax /* We only understand version 1 */ je 1f movl $1,%eax /* Return status */ leave /* * XXX this returns to our caller's caller (as is required) since * we didn't set up a frame and our caller did. */ ret 1: /* * If we have a kernelname copy it in */ movl BI_KERNELNAME(%ebx),%esi cmpl $0,%esi je 2f /* No kernelname */ movl $MAXPATHLEN,%ecx /* Brute force!!! */ movl $R(_kernelname),%edi cmpb $'/',(%esi) /* Make sure it starts with a slash */ je 1f movb $'/',(%edi) incl %edi decl %ecx 1: cld rep movsb 2: /* * Determine the size of the boot loader's copy of the bootinfo * struct. This is impossible to do properly because old versions * of the struct don't contain a size field and there are 2 old * versions with the same version number. */ movl $BI_ENDCOMMON,%ecx /* prepare for sizeless version */ testl $RB_BOOTINFO,8(%ebp) /* bi_size (and bootinfo) valid? */ je got_bi_size /* no, sizeless version */ movl BI_SIZE(%ebx),%ecx got_bi_size: /* * Copy the common part of the bootinfo struct */ movl %ebx,%esi movl $R(_bootinfo),%edi cmpl $BOOTINFO_SIZE,%ecx jbe got_common_bi_size movl $BOOTINFO_SIZE,%ecx got_common_bi_size: cld rep movsb #ifdef NFS #ifndef BOOTP_NFSV3 /* * If we have a nfs_diskless structure copy it in */ movl BI_NFS_DISKLESS(%ebx),%esi cmpl $0,%esi je olddiskboot movl $R(_nfs_diskless),%edi movl $NFSDISKLESS_SIZE,%ecx cld rep movsb movl $R(_nfs_diskless_valid),%edi movl $1,(%edi) #endif #endif /* * The old style disk boot. * (*btext)(howto, bootdev, cyloffset, esym); * Note that the newer boot code just falls into here to pick * up howto and bootdev, cyloffset and esym are no longer used */ olddiskboot: movl 8(%ebp),%eax movl %eax,R(_boothowto) movl 12(%ebp),%eax movl %eax,R(_bootdev) #if defined(USERCONFIG_BOOT) && defined(USERCONFIG) #ifdef PC98 movl $0x90200, %esi #else movl $0x10200, %esi #endif movl $R(_userconfig_from_boot),%edi movl $512,%ecx cld rep movsb #endif /* USERCONFIG_BOOT */ ret /********************************************************************** * * Identify the CPU and initialize anything special about it * */ identify_cpu: /* Try to toggle alignment check flag; does not exist on 386. */ pushfl popl %eax movl %eax,%ecx orl $PSL_AC,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_AC,%eax pushl %ecx popfl testl %eax,%eax jnz try486 /* NexGen CPU does not have aligment check flag. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jz trynexgen popfl movl $CPU_386,R(_cpu) jmp 3f trynexgen: popfl movl $CPU_NX586,R(_cpu) movl $0x4778654e,R(_cpu_vendor) # store vendor string movl $0x72446e65,R(_cpu_vendor+4) movl $0x6e657669,R(_cpu_vendor+8) movl $0,R(_cpu_vendor+12) jmp 3f try486: /* Try to toggle identification flag; does not exist on early 486s. */ pushfl popl %eax movl %eax,%ecx xorl $PSL_ID,%eax pushl %eax popfl pushfl popl %eax xorl %ecx,%eax andl $PSL_ID,%eax pushl %ecx popfl testl %eax,%eax jnz trycpuid movl $CPU_486,R(_cpu) /* * Check Cyrix CPU * Cyrix CPUs do not change the undefined flags following * execution of the divide instruction which divides 5 by 2. * * Note: CPUID is enabled on M2, so it passes another way. */ pushfl movl $0x5555, %eax xorl %edx, %edx movl $2, %ecx clc divl %ecx jnc trycyrix popfl jmp 3f /* You may use Intel CPU. */ trycyrix: popfl /* * IBM Bluelighting CPU also doesn't change the undefined flags. * Because IBM doesn't disclose the information for Bluelighting * CPU, we couldn't distinguish it from Cyrix's (including IBM * brand of Cyrix CPUs). */ movl $0x69727943,R(_cpu_vendor) # store vendor string movl $0x736e4978,R(_cpu_vendor+4) movl $0x64616574,R(_cpu_vendor+8) jmp 3f trycpuid: /* Use the `cpuid' instruction. */ xorl %eax,%eax .byte 0x0f,0xa2 # cpuid 0 movl %eax,R(_cpu_high) # highest capability movl %ebx,R(_cpu_vendor) # store vendor string movl %edx,R(_cpu_vendor+4) movl %ecx,R(_cpu_vendor+8) movb $0,R(_cpu_vendor+12) movl $1,%eax .byte 0x0f,0xa2 # cpuid 1 movl %eax,R(_cpu_id) # store cpu_id movl %edx,R(_cpu_feature) # store cpu_feature rorl $8,%eax # extract family type andl $15,%eax cmpl $5,%eax jae 1f /* less than Pentium; must be 486 */ movl $CPU_486,R(_cpu) jmp 3f 1: /* a Pentium? */ cmpl $5,%eax jne 2f movl $CPU_586,R(_cpu) jmp 3f 2: /* Greater than Pentium...call it a Pentium Pro */ movl $CPU_686,R(_cpu) 3: ret /********************************************************************** * * Create the first page directory and its page tables. * */ create_pagetables: testl $CPUID_PGE, R(_cpu_feature) jz 1f movl %cr4, %eax orl $CR4_PGE, %eax movl %eax, %cr4 1: /* Find end of kernel image (rounded up to a page boundary). */ movl $R(_end),%esi /* include symbols in "kernel image" if they are loaded and useful */ #ifdef DDB movl R(_bootinfo+BI_ESYMTAB),%edi testl %edi,%edi je over_symalloc movl %edi,%esi movl $KERNBASE,%edi addl %edi,R(_bootinfo+BI_SYMTAB) addl %edi,R(_bootinfo+BI_ESYMTAB) over_symalloc: #endif addl $PAGE_MASK,%esi andl $~PAGE_MASK,%esi movl %esi,R(_KERNend) /* save end of kernel */ movl %esi,R(physfree) /* next free page is at end of kernel */ /* Allocate Kernel Page Tables */ ALLOCPAGES(NKPT) movl %esi,R(_KPTphys) /* Allocate Page Table Directory */ ALLOCPAGES(1) movl %esi,R(_IdlePTD) /* Allocate UPAGES */ ALLOCPAGES(UPAGES) movl %esi,R(p0upa) addl $KERNBASE, %esi movl %esi, R(_proc0paddr) #ifdef SMP /* Allocate cpu0's private data page */ ALLOCPAGES(1) movl %esi,R(cpu0pp) addl $KERNBASE, %esi movl %esi, R(_cpu0prvpage) /* relocated to KVM space */ /* Allocate cpu0's private page table for mapping priv page, apic, etc */ ALLOCPAGES(1) movl %esi,R(cpu0pt) addl $KERNBASE, %esi movl %esi, R(_cpu0prvpt) /* relocated to KVM space */ #endif /* SMP */ /* Map read-only from zero to the end of the kernel text section */ xorl %eax, %eax #ifdef BDE_DEBUGGER /* If the debugger is present, actually map everything read-write. */ cmpl $0,R(_bdb_exists) jne map_read_write #endif xorl %edx,%edx + +#if !defined(SMP) testl $CPUID_PGE, R(_cpu_feature) jz 2f orl $PG_G,%edx +#endif 2: movl $R(_etext),%ecx addl $PAGE_MASK,%ecx shrl $PAGE_SHIFT,%ecx fillkptphys(%edx) /* Map read-write, data, bss and symbols */ movl $R(_etext),%eax addl $PAGE_MASK, %eax andl $~PAGE_MASK, %eax map_read_write: movl $PG_RW,%edx +#if !defined(SMP) testl $CPUID_PGE, R(_cpu_feature) jz 1f orl $PG_G,%edx +#endif 1: movl R(_KERNend),%ecx subl %eax,%ecx shrl $PAGE_SHIFT,%ecx fillkptphys(%edx) /* Map page directory. */ movl R(_IdlePTD), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map proc0's UPAGES in the physical way ... */ movl R(p0upa), %eax movl $UPAGES, %ecx fillkptphys($PG_RW) /* Map ISA hole */ movl $ISA_HOLE_START, %eax movl $ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx fillkptphys($PG_RW) #ifdef SMP /* Map cpu0's private page into global kmem (4K @ cpu0prvpage) */ movl R(cpu0pp), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map cpu0's private page table into global kmem FWIW */ movl R(cpu0pt), %eax movl $1, %ecx fillkptphys($PG_RW) /* Map the private page into the private page table into private space */ movl R(cpu0pp), %eax movl $0, %ebx /* pte offset = 0 */ movl $1, %ecx /* one private page coming right up */ fillkpt(R(cpu0pt), $PG_RW) /* Map the page table page into private space */ movl R(cpu0pt), %eax movl $1, %ebx /* pte offset = 1 */ movl $1, %ecx /* one private pt coming right up */ fillkpt(R(cpu0pt), $PG_RW) /* ... and put the page table table in the pde. */ movl R(cpu0pt), %eax movl $MPPTDI, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) #endif /* SMP */ /* install a pde for temporary double map of bottom of VA */ movl R(_KPTphys), %eax xorl %ebx, %ebx movl $1, %ecx fillkpt(R(_IdlePTD), $PG_RW) /* install pde's for pt's */ movl R(_KPTphys), %eax movl $KPTDI, %ebx movl $NKPT, %ecx fillkpt(R(_IdlePTD), $PG_RW) /* install a pde recursively mapping page directory as a page table */ movl R(_IdlePTD), %eax movl $PTDPTDI, %ebx movl $1,%ecx fillkpt(R(_IdlePTD), $PG_RW) ret #ifdef BDE_DEBUGGER bdb_prepare_paging: cmpl $0,R(_bdb_exists) je bdb_prepare_paging_exit subl $6,%esp /* * Copy and convert debugger entries from the bootstrap gdt and idt * to the kernel gdt and idt. Everything is still in low memory. * Tracing continues to work after paging is enabled because the * low memory addresses remain valid until everything is relocated. * However, tracing through the setidt() that initializes the trace * trap will crash. */ sgdt (%esp) movl 2(%esp),%esi /* base address of bootstrap gdt */ movl $R(_gdt),%edi movl %edi,2(%esp) /* prepare to load kernel gdt */ movl $8*18/4,%ecx cld rep /* copy gdt */ movsl movl $R(_gdt),-8+2(%edi) /* adjust gdt self-ptr */ movb $0x92,-8+5(%edi) lgdt (%esp) sidt (%esp) movl 2(%esp),%esi /* base address of current idt */ movl 8+4(%esi),%eax /* convert dbg descriptor to ... */ movw 8(%esi),%ax movl %eax,R(bdb_dbg_ljmp+1) /* ... immediate offset ... */ movl 8+2(%esi),%eax movw %ax,R(bdb_dbg_ljmp+5) /* ... and selector for ljmp */ movl 24+4(%esi),%eax /* same for bpt descriptor */ movw 24(%esi),%ax movl %eax,R(bdb_bpt_ljmp+1) movl 24+2(%esi),%eax movw %ax,R(bdb_bpt_ljmp+5) movl $R(_idt),%edi movl %edi,2(%esp) /* prepare to load kernel idt */ movl $8*4/4,%ecx cld rep /* copy idt */ movsl lidt (%esp) addl $6,%esp bdb_prepare_paging_exit: ret /* Relocate debugger gdt entries and gdt and idt pointers. */ bdb_commit_paging: cmpl $0,_bdb_exists je bdb_commit_paging_exit movl $_gdt+8*9,%eax /* adjust slots 9-17 */ movl $9,%ecx reloc_gdt: movb $KERNBASE>>24,7(%eax) /* top byte of base addresses, was 0, */ addl $8,%eax /* now KERNBASE>>24 */ loop reloc_gdt subl $6,%esp sgdt (%esp) addl $KERNBASE,2(%esp) lgdt (%esp) sidt (%esp) addl $KERNBASE,2(%esp) lidt (%esp) addl $6,%esp int $3 bdb_commit_paging_exit: ret #endif /* BDE_DEBUGGER */ Index: head/sys/i386/i386/mp_machdep.c =================================================================== --- head/sys/i386/i386/mp_machdep.c (revision 27483) +++ head/sys/i386/i386/mp_machdep.c (revision 27484) @@ -1,1830 +1,1834 @@ /* * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.14 1997/07/13 00:42:14 smp Exp smp $ + * $Id: mp_machdep.c,v 1.27 1997/07/13 01:22:43 fsmp Exp $ */ #include "opt_smp.h" #include /* for KERNBASE */ #include #include #include #include #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include #include #include #include #include #include #include #include /** TEST_DEFAULT_CONFIG, TEST_CPUSTOP _TEST1 */ #include #include #include /* cngetc() */ #if defined(APIC_IO) #include /* setidt() */ #include /* IPIs */ #include /* IPIs */ #endif /* APIC_IO */ #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define BIOS_BASE (0xf0000) #define BIOS_SIZE (0x10000) #define BIOS_COUNT (BIOS_SIZE/4) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) #define PROCENTRY_FLAG_EN 0x01 #define PROCENTRY_FLAG_BP 0x02 #define IOAPICENTRY_FLAG_EN 0x01 /* MP Floating Pointer Structure */ typedef struct MPFPS { char signature[4]; void *pap; u_char length; u_char spec_rev; u_char checksum; u_char mpfb1; u_char mpfb2; u_char mpfb3; u_char mpfb4; u_char mpfb5; } *mpfps_t; /* MP Configuration Table Header */ typedef struct MPCTH { char signature[4]; u_short base_table_length; u_char spec_rev; u_char checksum; u_char oem_id[8]; u_char product_id[12]; void *oem_table_pointer; u_short oem_table_size; u_short entry_count; void *apic_address; u_short extended_table_length; u_char extended_table_checksum; u_char reserved; } *mpcth_t; typedef struct PROCENTRY { u_char type; u_char apic_id; u_char apic_version; u_char cpu_flags; u_long cpu_signature; u_long feature_flags; u_long reserved1; u_long reserved2; } *proc_entry_ptr; typedef struct BUSENTRY { u_char type; u_char bus_id; char bus_type[6]; } *bus_entry_ptr; typedef struct IOAPICENTRY { u_char type; u_char apic_id; u_char apic_version; u_char apic_flags; void *apic_address; } *io_apic_entry_ptr; typedef struct INTENTRY { u_char type; u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } *int_entry_ptr; /* descriptions of MP basetable entries */ typedef struct BASETABLE_ENTRY { u_char type; u_char length; char name[16]; } basetable_entry; /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MP_START_POST 0x12 #define MP_ANNOUNCE_POST 0x13 #define MPTABLE_PASS1_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define MP_ENABLE_POST 0x16 #define START_ALL_APS_POST 0x17 #define INSTALL_AP_TRAMP_POST 0x18 #define START_AP_POST 0x19 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; /** XXX FIXME: what system files declare these??? */ extern struct region_descriptor r_gdt, r_idt; int mp_ncpus; /* # of CPUs, including BSP */ int mp_naps; /* # of Applications processors */ int mp_nbusses; /* # of busses */ int mp_napics; /* # of IO APICs */ int boot_cpu_id; /* designated BSP */ vm_offset_t cpu_apic_address; vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ u_int32_t cpu_apic_versions[NCPU]; u_int32_t io_apic_versions[NAPIC]; /* * APIC ID logical/physical mapping structures. * We oversize these to simplify boot-time config. */ int cpu_num_to_apic_id[NAPICID]; int io_num_to_apic_id[NAPICID]; int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; /* Boot of AP uses this PTD */ u_int *bootPTD; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t KPTphys; /* Virtual address of per-cpu common_tss */ extern struct i386tss common_tss; /* * Local data and functions. */ static int mp_capable; static u_int boot_address; static u_int base_memory; static int picmode; /* 0: virtual wire mode, 1: PIC mode */ static mpfps_t mpfps; static int search_for_sig(u_int32_t target, int count); static void mp_enable(u_int boot_addr); static int mptable_pass1(void); static int mptable_pass2(void); static void default_mp_table(int type); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); base_memory = basemem * 1024; /* convert to bytes */ boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ if ((base_memory - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } /* * Look for an Intel MP spec table (ie, SMP capable hardware). */ int mp_probe(void) { int x; u_long segment; u_int32_t target; POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { /* search first 1K of EBDA */ target = (u_int32_t) (segment << 4); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } else { /* last 1K of base memory, effective 'top of base' passed in */ target = (u_int32_t) (base_memory - 0x400); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } /* search the BIOS */ target = (u_int32_t) BIOS_BASE; if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) goto found; /* nothing found */ mpfps = (mpfps_t)0; mp_capable = 0; return 0; found: /* calculate needed resources */ mpfps = (mpfps_t)x; if (mptable_pass1()) panic("you must reconfigure your kernel"); /* flag fact that we are running multiple processors */ mp_capable = 1; return 1; } /* * Startup the SMP processors. */ void mp_start(void) { POSTCODE(MP_START_POST); /* look for MP capable motherboard */ if (mp_capable) mp_enable(boot_address); else panic("MP hardware not found!"); } /* * Print various information about the SMP system hardware and setup. */ void mp_announce(void) { int x; POSTCODE(MP_ANNOUNCE_POST); printf("FreeBSD/SMP: Multiprocessor motherboard\n"); printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); printf(", version: 0x%08x", cpu_apic_versions[0]); printf(", at 0x%08x\n", cpu_apic_address); for (x = 1; x <= mp_naps; ++x) { printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); printf(", version: 0x%08x", cpu_apic_versions[x]); printf(", at 0x%08x\n", cpu_apic_address); } #if defined(APIC_IO) for (x = 0; x < mp_napics; ++x) { printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); printf(", version: 0x%08x", io_apic_versions[x]); printf(", at 0x%08x\n", io_apic_address[x]); } #else printf(" Warning: APIC I/O disabled\n"); #endif /* APIC_IO */ } /* * AP cpu's call this to sync up protected mode. */ void init_secondary(void) { int gsel_tss, slot; r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; r_gdt.rd_base = (int) gdt; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); slot = NGDT + cpuid; gsel_tss = GSEL(slot, SEL_KPL); gdt[slot].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ PTD[0] = 0; + pmap_set_opt((unsigned *)PTD); + invltlb(); } #if defined(APIC_IO) /* * Final configuration of the BSP's local APIC: * - disable 'pic mode'. * - disable 'virtual wire mode'. * - enable NMI. */ void bsp_apic_configure(void) { u_char byte; u_int32_t temp; /* leave 'pic mode' if necessary */ if (picmode) { outb(0x22, 0x70); /* select IMCR */ byte = inb(0x23); /* current contents */ byte |= 0x01; /* mask external INTR */ outb(0x23, byte); /* disconnect 8259s/NMI */ } /* mask lint0 (the 8259 'virtual wire' connection) */ temp = lapic.lvt_lint0; temp |= APIC_LVT_M; /* set the mask */ lapic.lvt_lint0 = temp; /* setup lint1 to handle NMI */ temp = lapic.lvt_lint1; temp &= ~APIC_LVT_M; /* clear the mask */ lapic.lvt_lint1 = temp; if (bootverbose) apic_dump(); } #endif /* APIC_IO */ /******************************************************************* * local functions and data */ /* * start the SMP system */ static void mp_enable(u_int boot_addr) { int x; #if defined(APIC_IO) int apic; u_int ux; #endif /* APIC_IO */ POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); invltlb(); /* examine the MP table for needed info, uses physical addresses */ x = mptable_pass2(); *(int *)PTD = 0; invltlb(); /* can't process default configs till the CPU APIC is pmapped */ if (x) default_mp_table(x); #if defined(APIC_IO) /* fill the LOGICAL io_apic_versions table */ for (apic = 0; apic < mp_napics; ++apic) { ux = io_apic_read(apic, IOAPIC_VER); io_apic_versions[apic] = ux; } /* program each IO APIC in the system */ for (apic = 0; apic < mp_napics; ++apic) if (io_apic_setup(apic) < 0) panic("IO APIC setup failure"); /* install a 'Spurious INTerrupt' vector */ setidt(XSPURIOUSINT_OFFSET, Xspuriousint, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #if defined(TEST_CPUSTOP) /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_CPUSTOP */ #if defined(TEST_TEST1) /* install a 'Spurious INTerrupt' vector */ setidt(XTEST1_OFFSET, Xtest1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_TEST1 */ #endif /* APIC_IO */ /* start each Application Processor */ start_all_aps(boot_addr); /* * The init process might be started on a different CPU now, * and the boot CPU might not call prepare_usermode to get * cr0 correctly configured. Thus we initialize cr0 here. */ load_cr0(rcr0() | CR0_WP | CR0_AM); } /* * look for the MP spec signature */ /* string defined by the Intel MP Spec as identifying the MP table */ #define MP_SIG 0x5f504d5f /* _MP_ */ #define NEXT(X) ((X) += 4) static int search_for_sig(u_int32_t target, int count) { int x; u_int32_t *addr = (u_int32_t *) (KERNBASE + target); for (x = 0; x < count; NEXT(x)) if (addr[x] == MP_SIG) /* make array index a byte index */ return (target + (x * sizeof(u_int32_t))); return -1; } static basetable_entry basetable_entry_types[] = { {0, 20, "Processor"}, {1, 8, "Bus"}, {2, 8, "I/O APIC"}, {3, 8, "I/O INT"}, {4, 8, "Local INT"} }; typedef struct BUSDATA { u_char bus_id; enum busTypes bus_type; } bus_datum; typedef struct INTDATA { u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } io_int, local_int; typedef struct BUSTYPENAME { u_char type; char name[7]; } bus_type_name; static bus_type_name bus_type_table[] = { {CBUS, "CBUS"}, {CBUSII, "CBUSII"}, {EISA, "EISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {ISA, "ISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {PCI, "PCI"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {XPRESS, "XPRESS"}, {UNKNOWN_BUSTYPE, "---"} }; /* from MP spec v1.4, table 5-1 */ static int default_data[7][5] = { /* nbus, id0, type0, id1, type1 */ {1, 0, ISA, 255, 255}, {1, 0, EISA, 255, 255}, {1, 0, EISA, 255, 255}, {0, 255, 255, 255, 255},/* MCA not supported */ {2, 0, ISA, 1, PCI}, {2, 0, EISA, 1, PCI}, {0, 255, 255, 255, 255} /* MCA not supported */ }; /* the bus data */ bus_datum bus_data[NBUS]; /* the IO INT data, one entry per possible APIC INTerrupt */ io_int io_apic_ints[NINTR]; static int nintrs; static void fix_mp_table __P((void)); static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); /* * 1st pass on motherboard's Intel MP specification table. * * initializes: * mp_ncpus = 1 * * determines: * cpu_apic_address (common to all CPUs) * io_apic_address[N] * mp_naps * mp_nbusses * mp_napics * nintrs */ static int mptable_pass1(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int mustpanic; POSTCODE(MPTABLE_PASS1_POST); mustpanic = 0; /* clear various tables */ for (x = 0; x < NAPICID; ++x) { io_apic_address[x] = ~0; /* IO APIC address table */ } /* init everything to empty */ mp_naps = 0; mp_nbusses = 0; mp_napics = 0; nintrs = 0; /* check for use of 'default' configuration */ if (mpfps->mpfb1 != 0) { /* use default addresses */ cpu_apic_address = DEFAULT_APIC_BASE; io_apic_address[0] = DEFAULT_IO_APIC_BASE; /* fill in with defaults */ mp_naps = 2; /* includes BSP */ mp_nbusses = default_data[mpfps->mpfb1 - 1][0]; #if defined(APIC_IO) mp_napics = 1; nintrs = 16; #endif /* APIC_IO */ } else { if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); cpu_apic_address = (vm_offset_t) cth->apic_address; /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; while (count--) { switch (type = *(u_char *) position) { case 0: /* processor_entry */ if (((proc_entry_ptr)position)->cpu_flags & PROCENTRY_FLAG_EN) ++mp_naps; break; case 1: /* bus_entry */ ++mp_nbusses; break; case 2: /* io_apic_entry */ if (((io_apic_entry_ptr)position)->apic_flags & IOAPICENTRY_FLAG_EN) io_apic_address[mp_napics++] = (vm_offset_t)((io_apic_entry_ptr) position)->apic_address; break; case 3: /* int_entry */ ++nintrs; break; case 4: /* int_entry */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char*)position += basetable_entry_types[type].length; } } /* qualify the numbers */ if (mp_naps > NCPU) printf("Warning: only using %d of %d available CPUs!\n", NCPU, mp_naps); #if 0 /** XXX we consider this legal now (but should we?) */ mustpanic = 1; #endif if (mp_nbusses > NBUS) { printf("found %d busses, increase NBUS\n", mp_nbusses); mustpanic = 1; } if (mp_napics > NAPIC) { printf("found %d apics, increase NAPIC\n", mp_napics); mustpanic = 1; } if (nintrs > NINTR) { printf("found %d intrs, increase NINTR\n", nintrs); mustpanic = 1; } /* * Count the BSP. * This is also used as a counter while starting the APs. */ mp_ncpus = 1; --mp_naps; /* subtract the BSP */ return mustpanic; } /* * 2nd pass on motherboard's Intel MP specification table. * * sets: * boot_cpu_id * ID_TO_IO(N), phy APIC ID to log CPU/IO table * CPU_TO_ID(N), logical CPU to APIC ID table * IO_TO_ID(N), logical IO to APIC ID table * bus_data[N] * io_apic_ints[N] */ static int mptable_pass2(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int apic, bus, cpu, intr; POSTCODE(MPTABLE_PASS2_POST); /* clear various tables */ for (x = 0; x < NAPICID; ++x) { ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ } /* clear bus data table */ for (x = 0; x < NBUS; ++x) bus_data[x].bus_id = 0xff; /* clear IO APIC INT table */ for (x = 0; x < NINTR; ++x) io_apic_ints[x].int_type = 0xff; /* setup the cpu/apic mapping arrays */ boot_cpu_id = -1; /* record whether PIC or virtual-wire mode */ picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; /* check for use of 'default' configuration */ #if defined(TEST_DEFAULT_CONFIG) return TEST_DEFAULT_CONFIG; #else if (mpfps->mpfb1 != 0) return mpfps->mpfb1; /* return default configuration type */ #endif /* TEST_DEFAULT_CONFIG */ if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; apic = bus = intr = 0; cpu = 1; /* pre-count the BSP */ while (count--) { switch (type = *(u_char *) position) { case 0: if (processor_entry(position, cpu)) ++cpu; break; case 1: if (bus_entry(position, bus)) ++bus; break; case 2: if (io_apic_entry(position, apic)) ++apic; break; case 3: if (int_entry(position, intr)) ++intr; break; case 4: /* int_entry(position); */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char *) position += basetable_entry_types[type].length; } if (boot_cpu_id == -1) panic("NO BSP found!"); /* post scan cleanup */ fix_mp_table(); /* report fact that its NOT a default configuration */ return 0; } /* * parse an Intel MP specification table */ static void fix_mp_table(void) { int x; int id; int bus_0; int bus_pci; int num_pci_bus; /* * Fix mis-numbering of the PCI bus and its INT entries if the BIOS * did it wrong. The MP spec says that when more than 1 PCI bus * exists the BIOS must begin with bus entries for the PCI bus and use * actual PCI bus numbering. This implies that when only 1 PCI bus * exists the BIOS can choose to ignore this ordering, and indeed many * MP motherboards do ignore it. This causes a problem when the PCI * sub-system makes requests of the MP sub-system based on PCI bus * numbers. So here we look for the situation and renumber the * busses and associated INTs in an effort to "make it right". */ /* find bus 0, PCI bus, count the number of PCI busses */ for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_id == 0) { bus_0 = x; } if (bus_data[x].bus_type == PCI) { ++num_pci_bus; bus_pci = x; } } /* * bus_0 == slot of bus with ID of 0 * bus_pci == slot of last PCI bus encountered */ /* check the 1 PCI bus case for sanity */ if (num_pci_bus == 1) { /* if it is number 0 all is well */ if (bus_data[bus_pci].bus_id == 0) return; /* mis-numbered, swap with whichever bus uses slot 0 */ /* swap the bus entry types */ bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; /* swap each relavant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { io_apic_ints[x].src_bus_id = 0; } else if (io_apic_ints[x].src_bus_id == 0) { io_apic_ints[x].src_bus_id = id; } } } /* sanity check if more than 1 PCI bus */ else if (num_pci_bus > 1) { for (x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_type != PCI) continue; if (bus_data[x].bus_id >= num_pci_bus) panic("bad PCI bus numbering"); } } } static int processor_entry(proc_entry_ptr entry, int cpu) { /* check for usability */ if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) return 0; /* check for BSP flag */ if (entry->cpu_flags & PROCENTRY_FLAG_BP) { boot_cpu_id = entry->apic_id; CPU_TO_ID(0) = entry->apic_id; ID_TO_CPU(entry->apic_id) = 0; return 0; /* its already been counted */ } /* add another AP to list, if less than max number of CPUs */ else { CPU_TO_ID(cpu) = entry->apic_id; ID_TO_CPU(entry->apic_id) = cpu; return 1; } } static int bus_entry(bus_entry_ptr entry, int bus) { int x; char c, name[8]; /* encode the name into an index */ for (x = 0; x < 6; ++x) { if ((c = entry->bus_type[x]) == ' ') break; name[x] = c; } name[x] = '\0'; if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) panic("unknown bus type: '%s'", name); bus_data[bus].bus_id = entry->bus_id; bus_data[bus].bus_type = x; return 1; } static int io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; IO_TO_ID(apic) = entry->apic_id; ID_TO_IO(entry->apic_id) = apic; return 1; } static int lookup_bus_type(char *name) { int x; for (x = 0; x < MAX_BUSTYPE; ++x) if (strcmp(bus_type_table[x].name, name) == 0) return bus_type_table[x].type; return UNKNOWN_BUSTYPE; } static int int_entry(int_entry_ptr entry, int intr) { io_apic_ints[intr].int_type = entry->int_type; io_apic_ints[intr].int_flags = entry->int_flags; io_apic_ints[intr].src_bus_id = entry->src_bus_id; io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; } static int apic_int_is_bus_type(int intr, int bus_type) { int bus; for (bus = 0; bus < mp_nbusses; ++bus) if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) && ((int) bus_data[bus].bus_type == bus_type)) return 1; return 0; } /* * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { int isa_irq; int apic_pin; #if defined(SKIP_IRQ15_REDIRECT) if (isa_mask == (1 << 15)) { printf("skipping ISA IRQ15 redirect\n"); return isa_mask; } #endif /* SKIP_IRQ15_REDIRECT */ isa_irq = ffs(isa_mask); /* find its bit position */ if (isa_irq == 0) /* doesn't exist */ return 0; --isa_irq; /* make it zero based */ apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ if (apic_pin == -1) return 0; return (1 << apic_pin); /* convert pin# to a mask */ } /* * Determine which APIC pin an ISA/EISA INT is attached to. */ #define INTTYPE(I) (io_apic_ints[(I)].int_type) #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) int isa_apic_pin(int isa_irq) { int intr; #if defined(SMP_TIMER_NC) if (isa_irq == 0) return -1; #endif /* SMP_TIMER_NC */ for (intr = 0; intr < nintrs; ++intr) { /* check each record */ if (INTTYPE(intr) == 0) { /* standard INT */ if (SRCBUSIRQ(intr) == isa_irq) { if (apic_int_is_bus_type(intr, ISA) || apic_int_is_bus_type(intr, EISA)) return INTPIN(intr); /* found */ } } } return -1; /* NOT found */ } #undef SRCBUSIRQ /* * Determine which APIC pin a PCI INT is attached to. */ #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) int pci_apic_pin(int pciBus, int pciDevice, int pciInt) { int intr; --pciInt; /* zero based */ for (intr = 0; intr < nintrs; ++intr) /* check each record */ if ((INTTYPE(intr) == 0) /* standard INT */ && (SRCBUSID(intr) == pciBus) && (SRCBUSDEVICE(intr) == pciDevice) && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ if (apic_int_is_bus_type(intr, PCI)) return INTPIN(intr); /* exact match */ return -1; /* NOT found */ } #undef SRCBUSLINE #undef SRCBUSDEVICE #undef SRCBUSID #undef INTPIN #undef INTTYPE /* * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. * * XXX FIXME: * Exactly what this means is unclear at this point. It is a solution * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard * could route any of the ISA INTs to upper (>15) IRQ values. But most would * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an * option. */ int undirect_isa_irq(int rirq) { #if defined(READY) printf("Freeing redirected ISA irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); return 0; #endif /* READY */ } /* * Reprogram the MB chipset to NOT redirect a PCI INTerrupt */ int undirect_pci_irq(int rirq) { #if defined(READY) if (bootverbose) printf("Freeing redirected PCI irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else if (bootverbose) printf("Freeing (NOT implemented) redirected PCI irq %d.\n", rirq); return 0; #endif /* READY */ } /* * given a bus ID, return: * the bus type if found * -1 if NOT found */ int apic_bus_type(int id) { int x; for (x = 0; x < mp_nbusses; ++x) if (bus_data[x].bus_id == id) return bus_data[x].bus_type; return -1; } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus ID if found * -1 if NOT found */ int apic_src_bus_id(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_id); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus IRQ if found * -1 if NOT found */ int apic_src_bus_irq(int apic, int pin) { int x; for (x = 0; x < nintrs; x++) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_irq); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated INTerrupt type if found * -1 if NOT found */ int apic_int_type(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_type); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated trigger mode if found * -1 if NOT found */ int apic_trigger(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return ((io_apic_ints[x].int_flags >> 2) & 0x03); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated 'active' level if found * -1 if NOT found */ int apic_polarity(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_flags & 0x03); return -1; /* NOT found */ } /* * set data according to MP defaults * FIXME: probably not complete yet... */ static void default_mp_table(int type) { int ap_cpu_id; #if defined(APIC_IO) u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ #if 0 printf(" MP default config type: %d\n", type); switch (type) { case 1: printf(" bus: ISA, APIC: 82489DX\n"); break; case 2: printf(" bus: EISA, APIC: 82489DX\n"); break; case 3: printf(" bus: EISA, APIC: 82489DX\n"); break; case 4: printf(" bus: MCA, APIC: 82489DX\n"); break; case 5: printf(" bus: ISA+PCI, APIC: Integrated\n"); break; case 6: printf(" bus: EISA+PCI, APIC: Integrated\n"); break; case 7: printf(" bus: MCA+PCI, APIC: Integrated\n"); break; default: printf(" future type\n"); break; /* NOTREACHED */ } #endif /* 0 */ boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; /* BSP */ CPU_TO_ID(0) = boot_cpu_id; ID_TO_CPU(boot_cpu_id) = 0; /* one and only AP */ CPU_TO_ID(1) = ap_cpu_id; ID_TO_CPU(ap_cpu_id) = 1; #if defined(APIC_IO) /* one and only IO APIC */ io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; /* * sanity check, refer to MP spec section 3.6.6, last paragraph * necessary as some hardware isn't properly setting up the IO APIC */ #if defined(REALLY_ANAL_IOAPICID_VALUE) if (io_apic_id != 2) { #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ux &= ~APIC_ID_MASK; /* clear the ID field */ ux |= 0x02000000; /* set it to '2' */ io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ if ((ux & APIC_ID_MASK) != 0x02000000) panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; ID_TO_IO(io_apic_id) = 0; #endif /* APIC_IO */ /* fill out bus entries */ switch (type) { case 1: case 2: case 3: case 5: case 6: bus_data[0].bus_id = default_data[type - 1][1]; bus_data[0].bus_type = default_data[type - 1][2]; bus_data[1].bus_id = default_data[type - 1][3]; bus_data[1].bus_type = default_data[type - 1][4]; break; /* case 4: case 7: MCA NOT supported */ default: /* illegal/reserved */ panic("BAD default MP config: %d", type); /* NOTREACHED */ } #if defined(APIC_IO) /* general cases from MP v1.4, table 5-2 */ for (pin = 0; pin < 16; ++pin) { io_apic_ints[pin].int_type = 0; io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ io_apic_ints[pin].src_bus_id = 0; io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ io_apic_ints[pin].dst_apic_id = io_apic_id; io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ } /* special cases from MP v1.4, table 5-2 */ if (type == 2) { io_apic_ints[2].int_type = 0xff; /* N/C */ io_apic_ints[13].int_type = 0xff; /* N/C */ #if !defined(APIC_MIXED_MODE) /** FIXME: ??? */ panic("sorry, can't support type 2 default yet"); #endif /* APIC_MIXED_MODE */ } else io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ if (type == 7) io_apic_ints[0].int_type = 0xff; /* N/C */ else io_apic_ints[0].int_type = 3; /* vectored 8259 */ #endif /* APIC_IO */ } /* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { int x, i; u_char mpbiosreason; u_long mpbioswarmvec; pd_entry_t newptd; pt_entry_t newpt; int *newpp; POSTCODE(START_ALL_APS_POST); /** * NOTE: this needs further thought: * where does it get released? * should it be set to empy? * * get the initial mp_lock with a count of 1 for the BSP */ mp_lock = 1; /* this uses a LOGICAL cpu ID, ie BSP == 0 */ /* initialize BSP's local APIC */ apic_initialize(); /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* record BSP in CPU map */ all_cpus = 1; /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* HACK HACK HACK !!! */ /* alloc new page table directory */ newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* clone currently active one (ie: IdlePTD) */ bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ /* set up 0 -> 4MB P==V mapping for AP boot */ newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); /* store PTD for this AP */ bootPTD = (pd_entry_t)vtophys(newptd); /* alloc new page table page */ newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* set the new PTD's private page to point there */ newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt); /* install self referential entry */ newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd); /* get a new private data page */ newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ newpt[0] = PG_V | PG_RW | vtophys(newpp); /* wire the ptp into itself for access */ newpt[1] = PG_V | PG_RW | vtophys(newpt); /* and the local apic */ newpt[2] = SMP_prvpt[2]; /* and the IO apic mapping[s] */ for (i = 16; i < 32; i++) newpt[i] = SMP_prvpt[i]; /* prime data page for it to use */ newpp[0] = x; /* cpuid */ newpp[1] = 0; /* curproc */ newpp[2] = 0; /* curpcb */ newpp[3] = 0; /* npxproc */ newpp[4] = 0; /* runtime.tv_sec */ newpp[5] = 0; /* runtime.tv_usec */ newpp[6] = x << 24; /* cpu_lockid */ /* XXX NOTE: ABANDON bootPTD for now!!!! */ /* END REVOLTING HACKERY */ /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ /* record its version info */ cpu_apic_versions[x] = cpu_apic_versions[0]; all_cpus |= (1 << x); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ other_cpus = all_cpus & ~(1 << cpuid); /* fill in our (BSP) APIC version */ cpu_apic_versions[0] = lapic.version; /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); + + pmap_set_opt_bsp(); /* number of APs actually started */ return mp_ncpus - 1; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(u_int boot_addr) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_addr + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_addr + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; } /* * this function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It ain't pretty, * but it seems to work. */ static int start_ap(int logical_cpu, u_int boot_addr) { int physical_cpu; int vector; int cpus; u_long icr_lo, icr_hi; POSTCODE(START_AP_POST); /* get the PHYSICAL APIC ID# */ physical_cpu = CPU_TO_ID(logical_cpu); /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_ncpus; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* setup the address for the target AP */ icr_hi = lapic.icr_hi & ~APIC_ID_MASK; icr_hi |= (physical_cpu << 24); lapic.icr_hi = icr_hi; /* do an INIT IPI: assert RESET */ icr_lo = lapic.icr_lo & 0xfff00000; lapic.icr_lo = icr_lo | 0x0000c500; /* wait for pending status end */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* do an INIT IPI: deassert RESET */ lapic.icr_lo = icr_lo | 0x00008500; /* wait for pending status end */ u_sleep(10000); /* wait ~10mS */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* wait for it to start */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) if (mp_ncpus > cpus) return 1; /* return SUCCESS */ return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's * * XXX: Needs to handshake and wait for completion before proceding. */ void smp_invltlb(void) { #if defined(APIC_IO) if (smp_active && invltlb_ok) all_but_self_ipi(XINVLTLB_OFFSET); #endif /* APIC_IO */ } void invlpg(u_int addr) { __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); /* send a message to the other CPUs */ smp_invltlb(); } void invltlb(void) { u_long temp; /* * This should be implemented as load_cr3(rcr3()) when load_cr3() is * inlined. */ __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); /* send a message to the other CPUs */ smp_invltlb(); } #if defined(TEST_CPUSTOP) #if defined(DEBUG_CPUSTOP) void db_printf __P((const char *fmt, ...)); #endif /* DEBUG_CPUSTOP */ /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int stop_cpus( u_int map ) { if (!smp_active) return 0; /* send IPI to all CPUs in map */ #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d stopping CPUs: 0x%08x\n", cpuid, map); #endif /* DEBUG_CPUSTOP */ stopped_cpus = 0; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); #if defined(DEBUG_CPUSTOP) db_printf(" spin\n"); #endif /* DEBUG_CPUSTOP */ while (stopped_cpus != map) /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" spun\nstopped\n"); #endif /* DEBUG_CPUSTOP */ return 1; } /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus( u_int map ) { if (!smp_active) return 0; #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d restarting CPUs: 0x%08x (0x%08x)\n", cpuid, map, stopped_cpus); #endif /* DEBUG_CPUSTOP */ started_cpus = map; /* signal other cpus to restart */ while (started_cpus) /* wait for each to clear its bit */ /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" restarted\n"); #endif /* DEBUG_CPUSTOP */ return 1; } #endif /** TEST_CPUSTOP */ Index: head/sys/i386/i386/mptable.c =================================================================== --- head/sys/i386/i386/mptable.c (revision 27483) +++ head/sys/i386/i386/mptable.c (revision 27484) @@ -1,1830 +1,1834 @@ /* * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.14 1997/07/13 00:42:14 smp Exp smp $ + * $Id: mp_machdep.c,v 1.27 1997/07/13 01:22:43 fsmp Exp $ */ #include "opt_smp.h" #include /* for KERNBASE */ #include #include #include #include #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include #include #include #include #include #include #include #include /** TEST_DEFAULT_CONFIG, TEST_CPUSTOP _TEST1 */ #include #include #include /* cngetc() */ #if defined(APIC_IO) #include /* setidt() */ #include /* IPIs */ #include /* IPIs */ #endif /* APIC_IO */ #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define BIOS_BASE (0xf0000) #define BIOS_SIZE (0x10000) #define BIOS_COUNT (BIOS_SIZE/4) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) #define PROCENTRY_FLAG_EN 0x01 #define PROCENTRY_FLAG_BP 0x02 #define IOAPICENTRY_FLAG_EN 0x01 /* MP Floating Pointer Structure */ typedef struct MPFPS { char signature[4]; void *pap; u_char length; u_char spec_rev; u_char checksum; u_char mpfb1; u_char mpfb2; u_char mpfb3; u_char mpfb4; u_char mpfb5; } *mpfps_t; /* MP Configuration Table Header */ typedef struct MPCTH { char signature[4]; u_short base_table_length; u_char spec_rev; u_char checksum; u_char oem_id[8]; u_char product_id[12]; void *oem_table_pointer; u_short oem_table_size; u_short entry_count; void *apic_address; u_short extended_table_length; u_char extended_table_checksum; u_char reserved; } *mpcth_t; typedef struct PROCENTRY { u_char type; u_char apic_id; u_char apic_version; u_char cpu_flags; u_long cpu_signature; u_long feature_flags; u_long reserved1; u_long reserved2; } *proc_entry_ptr; typedef struct BUSENTRY { u_char type; u_char bus_id; char bus_type[6]; } *bus_entry_ptr; typedef struct IOAPICENTRY { u_char type; u_char apic_id; u_char apic_version; u_char apic_flags; void *apic_address; } *io_apic_entry_ptr; typedef struct INTENTRY { u_char type; u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } *int_entry_ptr; /* descriptions of MP basetable entries */ typedef struct BASETABLE_ENTRY { u_char type; u_char length; char name[16]; } basetable_entry; /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MP_START_POST 0x12 #define MP_ANNOUNCE_POST 0x13 #define MPTABLE_PASS1_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define MP_ENABLE_POST 0x16 #define START_ALL_APS_POST 0x17 #define INSTALL_AP_TRAMP_POST 0x18 #define START_AP_POST 0x19 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; /** XXX FIXME: what system files declare these??? */ extern struct region_descriptor r_gdt, r_idt; int mp_ncpus; /* # of CPUs, including BSP */ int mp_naps; /* # of Applications processors */ int mp_nbusses; /* # of busses */ int mp_napics; /* # of IO APICs */ int boot_cpu_id; /* designated BSP */ vm_offset_t cpu_apic_address; vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ u_int32_t cpu_apic_versions[NCPU]; u_int32_t io_apic_versions[NAPIC]; /* * APIC ID logical/physical mapping structures. * We oversize these to simplify boot-time config. */ int cpu_num_to_apic_id[NAPICID]; int io_num_to_apic_id[NAPICID]; int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; /* Boot of AP uses this PTD */ u_int *bootPTD; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t KPTphys; /* Virtual address of per-cpu common_tss */ extern struct i386tss common_tss; /* * Local data and functions. */ static int mp_capable; static u_int boot_address; static u_int base_memory; static int picmode; /* 0: virtual wire mode, 1: PIC mode */ static mpfps_t mpfps; static int search_for_sig(u_int32_t target, int count); static void mp_enable(u_int boot_addr); static int mptable_pass1(void); static int mptable_pass2(void); static void default_mp_table(int type); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); base_memory = basemem * 1024; /* convert to bytes */ boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ if ((base_memory - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } /* * Look for an Intel MP spec table (ie, SMP capable hardware). */ int mp_probe(void) { int x; u_long segment; u_int32_t target; POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { /* search first 1K of EBDA */ target = (u_int32_t) (segment << 4); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } else { /* last 1K of base memory, effective 'top of base' passed in */ target = (u_int32_t) (base_memory - 0x400); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } /* search the BIOS */ target = (u_int32_t) BIOS_BASE; if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) goto found; /* nothing found */ mpfps = (mpfps_t)0; mp_capable = 0; return 0; found: /* calculate needed resources */ mpfps = (mpfps_t)x; if (mptable_pass1()) panic("you must reconfigure your kernel"); /* flag fact that we are running multiple processors */ mp_capable = 1; return 1; } /* * Startup the SMP processors. */ void mp_start(void) { POSTCODE(MP_START_POST); /* look for MP capable motherboard */ if (mp_capable) mp_enable(boot_address); else panic("MP hardware not found!"); } /* * Print various information about the SMP system hardware and setup. */ void mp_announce(void) { int x; POSTCODE(MP_ANNOUNCE_POST); printf("FreeBSD/SMP: Multiprocessor motherboard\n"); printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); printf(", version: 0x%08x", cpu_apic_versions[0]); printf(", at 0x%08x\n", cpu_apic_address); for (x = 1; x <= mp_naps; ++x) { printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); printf(", version: 0x%08x", cpu_apic_versions[x]); printf(", at 0x%08x\n", cpu_apic_address); } #if defined(APIC_IO) for (x = 0; x < mp_napics; ++x) { printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); printf(", version: 0x%08x", io_apic_versions[x]); printf(", at 0x%08x\n", io_apic_address[x]); } #else printf(" Warning: APIC I/O disabled\n"); #endif /* APIC_IO */ } /* * AP cpu's call this to sync up protected mode. */ void init_secondary(void) { int gsel_tss, slot; r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; r_gdt.rd_base = (int) gdt; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); slot = NGDT + cpuid; gsel_tss = GSEL(slot, SEL_KPL); gdt[slot].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ PTD[0] = 0; + pmap_set_opt((unsigned *)PTD); + invltlb(); } #if defined(APIC_IO) /* * Final configuration of the BSP's local APIC: * - disable 'pic mode'. * - disable 'virtual wire mode'. * - enable NMI. */ void bsp_apic_configure(void) { u_char byte; u_int32_t temp; /* leave 'pic mode' if necessary */ if (picmode) { outb(0x22, 0x70); /* select IMCR */ byte = inb(0x23); /* current contents */ byte |= 0x01; /* mask external INTR */ outb(0x23, byte); /* disconnect 8259s/NMI */ } /* mask lint0 (the 8259 'virtual wire' connection) */ temp = lapic.lvt_lint0; temp |= APIC_LVT_M; /* set the mask */ lapic.lvt_lint0 = temp; /* setup lint1 to handle NMI */ temp = lapic.lvt_lint1; temp &= ~APIC_LVT_M; /* clear the mask */ lapic.lvt_lint1 = temp; if (bootverbose) apic_dump(); } #endif /* APIC_IO */ /******************************************************************* * local functions and data */ /* * start the SMP system */ static void mp_enable(u_int boot_addr) { int x; #if defined(APIC_IO) int apic; u_int ux; #endif /* APIC_IO */ POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); invltlb(); /* examine the MP table for needed info, uses physical addresses */ x = mptable_pass2(); *(int *)PTD = 0; invltlb(); /* can't process default configs till the CPU APIC is pmapped */ if (x) default_mp_table(x); #if defined(APIC_IO) /* fill the LOGICAL io_apic_versions table */ for (apic = 0; apic < mp_napics; ++apic) { ux = io_apic_read(apic, IOAPIC_VER); io_apic_versions[apic] = ux; } /* program each IO APIC in the system */ for (apic = 0; apic < mp_napics; ++apic) if (io_apic_setup(apic) < 0) panic("IO APIC setup failure"); /* install a 'Spurious INTerrupt' vector */ setidt(XSPURIOUSINT_OFFSET, Xspuriousint, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #if defined(TEST_CPUSTOP) /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_CPUSTOP */ #if defined(TEST_TEST1) /* install a 'Spurious INTerrupt' vector */ setidt(XTEST1_OFFSET, Xtest1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_TEST1 */ #endif /* APIC_IO */ /* start each Application Processor */ start_all_aps(boot_addr); /* * The init process might be started on a different CPU now, * and the boot CPU might not call prepare_usermode to get * cr0 correctly configured. Thus we initialize cr0 here. */ load_cr0(rcr0() | CR0_WP | CR0_AM); } /* * look for the MP spec signature */ /* string defined by the Intel MP Spec as identifying the MP table */ #define MP_SIG 0x5f504d5f /* _MP_ */ #define NEXT(X) ((X) += 4) static int search_for_sig(u_int32_t target, int count) { int x; u_int32_t *addr = (u_int32_t *) (KERNBASE + target); for (x = 0; x < count; NEXT(x)) if (addr[x] == MP_SIG) /* make array index a byte index */ return (target + (x * sizeof(u_int32_t))); return -1; } static basetable_entry basetable_entry_types[] = { {0, 20, "Processor"}, {1, 8, "Bus"}, {2, 8, "I/O APIC"}, {3, 8, "I/O INT"}, {4, 8, "Local INT"} }; typedef struct BUSDATA { u_char bus_id; enum busTypes bus_type; } bus_datum; typedef struct INTDATA { u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } io_int, local_int; typedef struct BUSTYPENAME { u_char type; char name[7]; } bus_type_name; static bus_type_name bus_type_table[] = { {CBUS, "CBUS"}, {CBUSII, "CBUSII"}, {EISA, "EISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {ISA, "ISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {PCI, "PCI"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {XPRESS, "XPRESS"}, {UNKNOWN_BUSTYPE, "---"} }; /* from MP spec v1.4, table 5-1 */ static int default_data[7][5] = { /* nbus, id0, type0, id1, type1 */ {1, 0, ISA, 255, 255}, {1, 0, EISA, 255, 255}, {1, 0, EISA, 255, 255}, {0, 255, 255, 255, 255},/* MCA not supported */ {2, 0, ISA, 1, PCI}, {2, 0, EISA, 1, PCI}, {0, 255, 255, 255, 255} /* MCA not supported */ }; /* the bus data */ bus_datum bus_data[NBUS]; /* the IO INT data, one entry per possible APIC INTerrupt */ io_int io_apic_ints[NINTR]; static int nintrs; static void fix_mp_table __P((void)); static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); /* * 1st pass on motherboard's Intel MP specification table. * * initializes: * mp_ncpus = 1 * * determines: * cpu_apic_address (common to all CPUs) * io_apic_address[N] * mp_naps * mp_nbusses * mp_napics * nintrs */ static int mptable_pass1(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int mustpanic; POSTCODE(MPTABLE_PASS1_POST); mustpanic = 0; /* clear various tables */ for (x = 0; x < NAPICID; ++x) { io_apic_address[x] = ~0; /* IO APIC address table */ } /* init everything to empty */ mp_naps = 0; mp_nbusses = 0; mp_napics = 0; nintrs = 0; /* check for use of 'default' configuration */ if (mpfps->mpfb1 != 0) { /* use default addresses */ cpu_apic_address = DEFAULT_APIC_BASE; io_apic_address[0] = DEFAULT_IO_APIC_BASE; /* fill in with defaults */ mp_naps = 2; /* includes BSP */ mp_nbusses = default_data[mpfps->mpfb1 - 1][0]; #if defined(APIC_IO) mp_napics = 1; nintrs = 16; #endif /* APIC_IO */ } else { if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); cpu_apic_address = (vm_offset_t) cth->apic_address; /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; while (count--) { switch (type = *(u_char *) position) { case 0: /* processor_entry */ if (((proc_entry_ptr)position)->cpu_flags & PROCENTRY_FLAG_EN) ++mp_naps; break; case 1: /* bus_entry */ ++mp_nbusses; break; case 2: /* io_apic_entry */ if (((io_apic_entry_ptr)position)->apic_flags & IOAPICENTRY_FLAG_EN) io_apic_address[mp_napics++] = (vm_offset_t)((io_apic_entry_ptr) position)->apic_address; break; case 3: /* int_entry */ ++nintrs; break; case 4: /* int_entry */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char*)position += basetable_entry_types[type].length; } } /* qualify the numbers */ if (mp_naps > NCPU) printf("Warning: only using %d of %d available CPUs!\n", NCPU, mp_naps); #if 0 /** XXX we consider this legal now (but should we?) */ mustpanic = 1; #endif if (mp_nbusses > NBUS) { printf("found %d busses, increase NBUS\n", mp_nbusses); mustpanic = 1; } if (mp_napics > NAPIC) { printf("found %d apics, increase NAPIC\n", mp_napics); mustpanic = 1; } if (nintrs > NINTR) { printf("found %d intrs, increase NINTR\n", nintrs); mustpanic = 1; } /* * Count the BSP. * This is also used as a counter while starting the APs. */ mp_ncpus = 1; --mp_naps; /* subtract the BSP */ return mustpanic; } /* * 2nd pass on motherboard's Intel MP specification table. * * sets: * boot_cpu_id * ID_TO_IO(N), phy APIC ID to log CPU/IO table * CPU_TO_ID(N), logical CPU to APIC ID table * IO_TO_ID(N), logical IO to APIC ID table * bus_data[N] * io_apic_ints[N] */ static int mptable_pass2(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int apic, bus, cpu, intr; POSTCODE(MPTABLE_PASS2_POST); /* clear various tables */ for (x = 0; x < NAPICID; ++x) { ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ } /* clear bus data table */ for (x = 0; x < NBUS; ++x) bus_data[x].bus_id = 0xff; /* clear IO APIC INT table */ for (x = 0; x < NINTR; ++x) io_apic_ints[x].int_type = 0xff; /* setup the cpu/apic mapping arrays */ boot_cpu_id = -1; /* record whether PIC or virtual-wire mode */ picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; /* check for use of 'default' configuration */ #if defined(TEST_DEFAULT_CONFIG) return TEST_DEFAULT_CONFIG; #else if (mpfps->mpfb1 != 0) return mpfps->mpfb1; /* return default configuration type */ #endif /* TEST_DEFAULT_CONFIG */ if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; apic = bus = intr = 0; cpu = 1; /* pre-count the BSP */ while (count--) { switch (type = *(u_char *) position) { case 0: if (processor_entry(position, cpu)) ++cpu; break; case 1: if (bus_entry(position, bus)) ++bus; break; case 2: if (io_apic_entry(position, apic)) ++apic; break; case 3: if (int_entry(position, intr)) ++intr; break; case 4: /* int_entry(position); */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char *) position += basetable_entry_types[type].length; } if (boot_cpu_id == -1) panic("NO BSP found!"); /* post scan cleanup */ fix_mp_table(); /* report fact that its NOT a default configuration */ return 0; } /* * parse an Intel MP specification table */ static void fix_mp_table(void) { int x; int id; int bus_0; int bus_pci; int num_pci_bus; /* * Fix mis-numbering of the PCI bus and its INT entries if the BIOS * did it wrong. The MP spec says that when more than 1 PCI bus * exists the BIOS must begin with bus entries for the PCI bus and use * actual PCI bus numbering. This implies that when only 1 PCI bus * exists the BIOS can choose to ignore this ordering, and indeed many * MP motherboards do ignore it. This causes a problem when the PCI * sub-system makes requests of the MP sub-system based on PCI bus * numbers. So here we look for the situation and renumber the * busses and associated INTs in an effort to "make it right". */ /* find bus 0, PCI bus, count the number of PCI busses */ for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_id == 0) { bus_0 = x; } if (bus_data[x].bus_type == PCI) { ++num_pci_bus; bus_pci = x; } } /* * bus_0 == slot of bus with ID of 0 * bus_pci == slot of last PCI bus encountered */ /* check the 1 PCI bus case for sanity */ if (num_pci_bus == 1) { /* if it is number 0 all is well */ if (bus_data[bus_pci].bus_id == 0) return; /* mis-numbered, swap with whichever bus uses slot 0 */ /* swap the bus entry types */ bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; /* swap each relavant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { io_apic_ints[x].src_bus_id = 0; } else if (io_apic_ints[x].src_bus_id == 0) { io_apic_ints[x].src_bus_id = id; } } } /* sanity check if more than 1 PCI bus */ else if (num_pci_bus > 1) { for (x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_type != PCI) continue; if (bus_data[x].bus_id >= num_pci_bus) panic("bad PCI bus numbering"); } } } static int processor_entry(proc_entry_ptr entry, int cpu) { /* check for usability */ if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) return 0; /* check for BSP flag */ if (entry->cpu_flags & PROCENTRY_FLAG_BP) { boot_cpu_id = entry->apic_id; CPU_TO_ID(0) = entry->apic_id; ID_TO_CPU(entry->apic_id) = 0; return 0; /* its already been counted */ } /* add another AP to list, if less than max number of CPUs */ else { CPU_TO_ID(cpu) = entry->apic_id; ID_TO_CPU(entry->apic_id) = cpu; return 1; } } static int bus_entry(bus_entry_ptr entry, int bus) { int x; char c, name[8]; /* encode the name into an index */ for (x = 0; x < 6; ++x) { if ((c = entry->bus_type[x]) == ' ') break; name[x] = c; } name[x] = '\0'; if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) panic("unknown bus type: '%s'", name); bus_data[bus].bus_id = entry->bus_id; bus_data[bus].bus_type = x; return 1; } static int io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; IO_TO_ID(apic) = entry->apic_id; ID_TO_IO(entry->apic_id) = apic; return 1; } static int lookup_bus_type(char *name) { int x; for (x = 0; x < MAX_BUSTYPE; ++x) if (strcmp(bus_type_table[x].name, name) == 0) return bus_type_table[x].type; return UNKNOWN_BUSTYPE; } static int int_entry(int_entry_ptr entry, int intr) { io_apic_ints[intr].int_type = entry->int_type; io_apic_ints[intr].int_flags = entry->int_flags; io_apic_ints[intr].src_bus_id = entry->src_bus_id; io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; } static int apic_int_is_bus_type(int intr, int bus_type) { int bus; for (bus = 0; bus < mp_nbusses; ++bus) if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) && ((int) bus_data[bus].bus_type == bus_type)) return 1; return 0; } /* * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { int isa_irq; int apic_pin; #if defined(SKIP_IRQ15_REDIRECT) if (isa_mask == (1 << 15)) { printf("skipping ISA IRQ15 redirect\n"); return isa_mask; } #endif /* SKIP_IRQ15_REDIRECT */ isa_irq = ffs(isa_mask); /* find its bit position */ if (isa_irq == 0) /* doesn't exist */ return 0; --isa_irq; /* make it zero based */ apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ if (apic_pin == -1) return 0; return (1 << apic_pin); /* convert pin# to a mask */ } /* * Determine which APIC pin an ISA/EISA INT is attached to. */ #define INTTYPE(I) (io_apic_ints[(I)].int_type) #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) int isa_apic_pin(int isa_irq) { int intr; #if defined(SMP_TIMER_NC) if (isa_irq == 0) return -1; #endif /* SMP_TIMER_NC */ for (intr = 0; intr < nintrs; ++intr) { /* check each record */ if (INTTYPE(intr) == 0) { /* standard INT */ if (SRCBUSIRQ(intr) == isa_irq) { if (apic_int_is_bus_type(intr, ISA) || apic_int_is_bus_type(intr, EISA)) return INTPIN(intr); /* found */ } } } return -1; /* NOT found */ } #undef SRCBUSIRQ /* * Determine which APIC pin a PCI INT is attached to. */ #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) int pci_apic_pin(int pciBus, int pciDevice, int pciInt) { int intr; --pciInt; /* zero based */ for (intr = 0; intr < nintrs; ++intr) /* check each record */ if ((INTTYPE(intr) == 0) /* standard INT */ && (SRCBUSID(intr) == pciBus) && (SRCBUSDEVICE(intr) == pciDevice) && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ if (apic_int_is_bus_type(intr, PCI)) return INTPIN(intr); /* exact match */ return -1; /* NOT found */ } #undef SRCBUSLINE #undef SRCBUSDEVICE #undef SRCBUSID #undef INTPIN #undef INTTYPE /* * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. * * XXX FIXME: * Exactly what this means is unclear at this point. It is a solution * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard * could route any of the ISA INTs to upper (>15) IRQ values. But most would * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an * option. */ int undirect_isa_irq(int rirq) { #if defined(READY) printf("Freeing redirected ISA irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); return 0; #endif /* READY */ } /* * Reprogram the MB chipset to NOT redirect a PCI INTerrupt */ int undirect_pci_irq(int rirq) { #if defined(READY) if (bootverbose) printf("Freeing redirected PCI irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else if (bootverbose) printf("Freeing (NOT implemented) redirected PCI irq %d.\n", rirq); return 0; #endif /* READY */ } /* * given a bus ID, return: * the bus type if found * -1 if NOT found */ int apic_bus_type(int id) { int x; for (x = 0; x < mp_nbusses; ++x) if (bus_data[x].bus_id == id) return bus_data[x].bus_type; return -1; } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus ID if found * -1 if NOT found */ int apic_src_bus_id(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_id); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus IRQ if found * -1 if NOT found */ int apic_src_bus_irq(int apic, int pin) { int x; for (x = 0; x < nintrs; x++) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_irq); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated INTerrupt type if found * -1 if NOT found */ int apic_int_type(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_type); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated trigger mode if found * -1 if NOT found */ int apic_trigger(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return ((io_apic_ints[x].int_flags >> 2) & 0x03); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated 'active' level if found * -1 if NOT found */ int apic_polarity(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_flags & 0x03); return -1; /* NOT found */ } /* * set data according to MP defaults * FIXME: probably not complete yet... */ static void default_mp_table(int type) { int ap_cpu_id; #if defined(APIC_IO) u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ #if 0 printf(" MP default config type: %d\n", type); switch (type) { case 1: printf(" bus: ISA, APIC: 82489DX\n"); break; case 2: printf(" bus: EISA, APIC: 82489DX\n"); break; case 3: printf(" bus: EISA, APIC: 82489DX\n"); break; case 4: printf(" bus: MCA, APIC: 82489DX\n"); break; case 5: printf(" bus: ISA+PCI, APIC: Integrated\n"); break; case 6: printf(" bus: EISA+PCI, APIC: Integrated\n"); break; case 7: printf(" bus: MCA+PCI, APIC: Integrated\n"); break; default: printf(" future type\n"); break; /* NOTREACHED */ } #endif /* 0 */ boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; /* BSP */ CPU_TO_ID(0) = boot_cpu_id; ID_TO_CPU(boot_cpu_id) = 0; /* one and only AP */ CPU_TO_ID(1) = ap_cpu_id; ID_TO_CPU(ap_cpu_id) = 1; #if defined(APIC_IO) /* one and only IO APIC */ io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; /* * sanity check, refer to MP spec section 3.6.6, last paragraph * necessary as some hardware isn't properly setting up the IO APIC */ #if defined(REALLY_ANAL_IOAPICID_VALUE) if (io_apic_id != 2) { #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ux &= ~APIC_ID_MASK; /* clear the ID field */ ux |= 0x02000000; /* set it to '2' */ io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ if ((ux & APIC_ID_MASK) != 0x02000000) panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; ID_TO_IO(io_apic_id) = 0; #endif /* APIC_IO */ /* fill out bus entries */ switch (type) { case 1: case 2: case 3: case 5: case 6: bus_data[0].bus_id = default_data[type - 1][1]; bus_data[0].bus_type = default_data[type - 1][2]; bus_data[1].bus_id = default_data[type - 1][3]; bus_data[1].bus_type = default_data[type - 1][4]; break; /* case 4: case 7: MCA NOT supported */ default: /* illegal/reserved */ panic("BAD default MP config: %d", type); /* NOTREACHED */ } #if defined(APIC_IO) /* general cases from MP v1.4, table 5-2 */ for (pin = 0; pin < 16; ++pin) { io_apic_ints[pin].int_type = 0; io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ io_apic_ints[pin].src_bus_id = 0; io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ io_apic_ints[pin].dst_apic_id = io_apic_id; io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ } /* special cases from MP v1.4, table 5-2 */ if (type == 2) { io_apic_ints[2].int_type = 0xff; /* N/C */ io_apic_ints[13].int_type = 0xff; /* N/C */ #if !defined(APIC_MIXED_MODE) /** FIXME: ??? */ panic("sorry, can't support type 2 default yet"); #endif /* APIC_MIXED_MODE */ } else io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ if (type == 7) io_apic_ints[0].int_type = 0xff; /* N/C */ else io_apic_ints[0].int_type = 3; /* vectored 8259 */ #endif /* APIC_IO */ } /* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { int x, i; u_char mpbiosreason; u_long mpbioswarmvec; pd_entry_t newptd; pt_entry_t newpt; int *newpp; POSTCODE(START_ALL_APS_POST); /** * NOTE: this needs further thought: * where does it get released? * should it be set to empy? * * get the initial mp_lock with a count of 1 for the BSP */ mp_lock = 1; /* this uses a LOGICAL cpu ID, ie BSP == 0 */ /* initialize BSP's local APIC */ apic_initialize(); /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* record BSP in CPU map */ all_cpus = 1; /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* HACK HACK HACK !!! */ /* alloc new page table directory */ newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* clone currently active one (ie: IdlePTD) */ bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ /* set up 0 -> 4MB P==V mapping for AP boot */ newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); /* store PTD for this AP */ bootPTD = (pd_entry_t)vtophys(newptd); /* alloc new page table page */ newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* set the new PTD's private page to point there */ newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt); /* install self referential entry */ newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd); /* get a new private data page */ newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ newpt[0] = PG_V | PG_RW | vtophys(newpp); /* wire the ptp into itself for access */ newpt[1] = PG_V | PG_RW | vtophys(newpt); /* and the local apic */ newpt[2] = SMP_prvpt[2]; /* and the IO apic mapping[s] */ for (i = 16; i < 32; i++) newpt[i] = SMP_prvpt[i]; /* prime data page for it to use */ newpp[0] = x; /* cpuid */ newpp[1] = 0; /* curproc */ newpp[2] = 0; /* curpcb */ newpp[3] = 0; /* npxproc */ newpp[4] = 0; /* runtime.tv_sec */ newpp[5] = 0; /* runtime.tv_usec */ newpp[6] = x << 24; /* cpu_lockid */ /* XXX NOTE: ABANDON bootPTD for now!!!! */ /* END REVOLTING HACKERY */ /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ /* record its version info */ cpu_apic_versions[x] = cpu_apic_versions[0]; all_cpus |= (1 << x); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ other_cpus = all_cpus & ~(1 << cpuid); /* fill in our (BSP) APIC version */ cpu_apic_versions[0] = lapic.version; /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); + + pmap_set_opt_bsp(); /* number of APs actually started */ return mp_ncpus - 1; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(u_int boot_addr) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_addr + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_addr + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; } /* * this function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It ain't pretty, * but it seems to work. */ static int start_ap(int logical_cpu, u_int boot_addr) { int physical_cpu; int vector; int cpus; u_long icr_lo, icr_hi; POSTCODE(START_AP_POST); /* get the PHYSICAL APIC ID# */ physical_cpu = CPU_TO_ID(logical_cpu); /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_ncpus; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* setup the address for the target AP */ icr_hi = lapic.icr_hi & ~APIC_ID_MASK; icr_hi |= (physical_cpu << 24); lapic.icr_hi = icr_hi; /* do an INIT IPI: assert RESET */ icr_lo = lapic.icr_lo & 0xfff00000; lapic.icr_lo = icr_lo | 0x0000c500; /* wait for pending status end */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* do an INIT IPI: deassert RESET */ lapic.icr_lo = icr_lo | 0x00008500; /* wait for pending status end */ u_sleep(10000); /* wait ~10mS */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* wait for it to start */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) if (mp_ncpus > cpus) return 1; /* return SUCCESS */ return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's * * XXX: Needs to handshake and wait for completion before proceding. */ void smp_invltlb(void) { #if defined(APIC_IO) if (smp_active && invltlb_ok) all_but_self_ipi(XINVLTLB_OFFSET); #endif /* APIC_IO */ } void invlpg(u_int addr) { __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); /* send a message to the other CPUs */ smp_invltlb(); } void invltlb(void) { u_long temp; /* * This should be implemented as load_cr3(rcr3()) when load_cr3() is * inlined. */ __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); /* send a message to the other CPUs */ smp_invltlb(); } #if defined(TEST_CPUSTOP) #if defined(DEBUG_CPUSTOP) void db_printf __P((const char *fmt, ...)); #endif /* DEBUG_CPUSTOP */ /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int stop_cpus( u_int map ) { if (!smp_active) return 0; /* send IPI to all CPUs in map */ #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d stopping CPUs: 0x%08x\n", cpuid, map); #endif /* DEBUG_CPUSTOP */ stopped_cpus = 0; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); #if defined(DEBUG_CPUSTOP) db_printf(" spin\n"); #endif /* DEBUG_CPUSTOP */ while (stopped_cpus != map) /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" spun\nstopped\n"); #endif /* DEBUG_CPUSTOP */ return 1; } /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus( u_int map ) { if (!smp_active) return 0; #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d restarting CPUs: 0x%08x (0x%08x)\n", cpuid, map, stopped_cpus); #endif /* DEBUG_CPUSTOP */ started_cpus = map; /* signal other cpus to restart */ while (started_cpus) /* wait for each to clear its bit */ /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" restarted\n"); #endif /* DEBUG_CPUSTOP */ return 1; } #endif /** TEST_CPUSTOP */ Index: head/sys/i386/i386/pmap.c =================================================================== --- head/sys/i386/i386/pmap.c (revision 27483) +++ head/sys/i386/i386/pmap.c (revision 27484) @@ -1,3357 +1,3357 @@ /* * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.147 1997/06/25 20:07:50 tegge Exp $ + * $Id: pmap.c,v 1.148 1997/07/17 04:33:38 dyson Exp $ */ /* * Manages physical address maps. * * In addition to hardware address maps, this * module is called upon to provide software-use-only * maps which may or may not be stored in the same * form as hardware maps. These pseudo-maps are * used to store intermediate results from copy * operations to and from address spaces. * * Since the information managed by this module is * also stored by the logical address mapping module, * this module may throw away valid virtual-to-physical * mappings at almost any time. However, invalidations * of virtual-to-physical mappings must be done as * requested. * * In order to cope with hardware architectures which * make virtual-to-physical map invalidates expensive, * this module may delay invalidate or reduced protection * operations until such time as they are actually * necessary. This module is given full information as * to which processors are currently using which maps, * and to when physical maps must be made correct. */ #include "opt_cpu.h" #define PMAP_LOCK 1 #define PMAP_PVLIST 1 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(SMP) || defined(APIC_IO) #include #include #include #endif /* SMP || APIC_IO */ #define PMAP_KEEP_PDIRS #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif #if defined(DIAGNOSTIC) #define PMAP_DIAGNOSTIC #endif #if !defined(PMAP_DIAGNOSTIC) #define PMAP_INLINE __inline #else #define PMAP_INLINE #endif #define PTPHINT static void init_pv_entries __P((int)); /* * Get PDEs and PTEs for user/kernel address space */ #define pmap_pde(m, v) (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT])) #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT]) #define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0) #define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0) #define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0) #define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0) #define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) #define pmap_pte_set_w(pte, v) ((v)?(*(int *)pte |= PG_W):(*(int *)pte &= ~PG_W)) #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) /* * Given a map and a machine independent protection code, * convert to a vax protection code. */ #define pte_prot(m, p) (protection_codes[p]) static int protection_codes[8]; #define pa_index(pa) atop((pa) - vm_first_phys) #define pa_to_pvh(pa) (&pv_table[pa_index(pa)]) static struct pmap kernel_pmap_store; pmap_t kernel_pmap; vm_offset_t avail_start; /* PA of first available physical page */ vm_offset_t avail_end; /* PA of last available physical page */ vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ static vm_offset_t vm_first_phys; static int pgeflag; /* PG_G or-in */ static int pseflag; /* PG_PS or-in */ static int nkpt; static vm_page_t nkpg; vm_offset_t kernel_vm_end; extern vm_offset_t clean_sva, clean_eva; #define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2) /* * Data for the pv entry allocation mechanism */ static int pv_freelistcnt; TAILQ_HEAD (,pv_entry) pv_freelist = {0}; static vm_offset_t pvva; static int npvvapg; /* * All those kernel PT submaps that BSD is so fond of */ pt_entry_t *CMAP1 = 0; static pt_entry_t *CMAP2, *ptmmap; static pv_table_t *pv_table; caddr_t CADDR1 = 0, ptvmmap = 0; static caddr_t CADDR2; static pt_entry_t *msgbufmap; struct msgbuf *msgbufp=0; pt_entry_t *PMAP1 = 0; unsigned *PADDR1 = 0; static PMAP_INLINE void free_pv_entry __P((pv_entry_t pv)); static unsigned * get_ptbase __P((pmap_t pmap)); static pv_entry_t get_pv_entry __P((void)); static void i386_protection_init __P((void)); static void pmap_alloc_pv_entry __P((void)); static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); static PMAP_INLINE int pmap_is_managed __P((vm_offset_t pa)); static void pmap_remove_all __P((vm_offset_t pa)); static vm_page_t pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_page_t mpte)); static int pmap_remove_pte __P((struct pmap *pmap, unsigned *ptq, vm_offset_t sva)); static void pmap_remove_page __P((struct pmap *pmap, vm_offset_t va)); static int pmap_remove_entry __P((struct pmap *pmap, pv_table_t *pv, vm_offset_t va)); static boolean_t pmap_testbit __P((vm_offset_t pa, int bit)); static void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_offset_t pa)); static vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va)); static int pmap_release_free_page __P((pmap_t pmap, vm_page_t p)); static vm_page_t _pmap_allocpte __P((pmap_t pmap, unsigned ptepindex)); static unsigned * pmap_pte_quick __P((pmap_t pmap, vm_offset_t va)); static vm_page_t pmap_page_alloc __P((vm_object_t object, vm_pindex_t pindex)); static vm_page_t pmap_page_lookup __P((vm_object_t object, vm_pindex_t pindex)); static int pmap_unuse_pt __P((pmap_t, vm_offset_t, vm_page_t)); vm_offset_t pmap_kmem_choose(vm_offset_t addr) ; #define PDSTACKMAX 6 static vm_offset_t pdstack[PDSTACKMAX]; static int pdstackptr; unsigned pdir4mb; /* * Routine: pmap_pte * Function: * Extract the page table entry associated * with the given map/virtual_address pair. */ PMAP_INLINE unsigned * pmap_pte(pmap, va) register pmap_t pmap; vm_offset_t va; { if (pmap && *pmap_pde(pmap, va)) { return get_ptbase(pmap) + i386_btop(va); } return (0); } /* * Move the kernel virtual free pointer to the next * 4MB. This is used to help improve performance * by using a large (4MB) page for much of the kernel * (.text, .data, .bss) */ vm_offset_t pmap_kmem_choose(vm_offset_t addr) { vm_offset_t newaddr = addr; if (cpu_feature & CPUID_PSE) { newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1); } return newaddr; } /* * Bootstrap the system enough to run with virtual memory. * * On the i386 this is called after mapping has already been enabled * and just syncs the pmap module with what has already been done. * [We can't call it easily with mapping off since the kernel is not * mapped with PA == VA, hence we would have to relocate every address * from the linked base (virtual) address "KERNBASE" to the actual * (physical) address starting relative to 0] */ void pmap_bootstrap(firstaddr, loadaddr) vm_offset_t firstaddr; vm_offset_t loadaddr; { vm_offset_t va; pt_entry_t *pte; int i, j; avail_start = firstaddr; /* * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too * large. It should instead be correctly calculated in locore.s and * not based on 'first' (which is a physical address, not a virtual * address, for the start of unused physical memory). The kernel * page tables are NOT double mapped and thus should not be included * in this calculation. */ virtual_avail = (vm_offset_t) KERNBASE + firstaddr; virtual_avail = pmap_kmem_choose(virtual_avail); virtual_end = VM_MAX_KERNEL_ADDRESS; /* * Initialize protection array. */ i386_protection_init(); /* * The kernel's pmap is statically allocated so we don't have to use * pmap_create, which is unlikely to work correctly at this part of * the boot sequence (XXX and which no longer exists). */ kernel_pmap = &kernel_pmap_store; kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD); kernel_pmap->pm_count = 1; #if PMAP_PVLIST TAILQ_INIT(&kernel_pmap->pm_pvlist); #endif nkpt = NKPT; /* * Reserve some special page table entries/VA space for temporary * mapping of pages. */ #define SYSMAP(c, p, v, n) \ v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); va = virtual_avail; pte = (pt_entry_t *) pmap_pte(kernel_pmap, va); /* * CMAP1/CMAP2 are used for zeroing and copying pages. */ SYSMAP(caddr_t, CMAP1, CADDR1, 1) SYSMAP(caddr_t, CMAP2, CADDR2, 1) /* * ptvmmap is used for reading arbitrary physical pages via /dev/mem. * XXX ptmmap is not used. */ SYSMAP(caddr_t, ptmmap, ptvmmap, 1) /* * msgbufp is used to map the system message buffer. * XXX msgbufmap is not used. */ SYSMAP(struct msgbuf *, msgbufmap, msgbufp, atop(round_page(sizeof(struct msgbuf)))) /* * ptemap is used for pmap_pte_quick */ SYSMAP(unsigned *, PMAP1, PADDR1, 1); virtual_avail = va; *(int *) CMAP1 = *(int *) CMAP2 = 0; *(int *) PTD = 0; #ifdef SMP if (cpu_apic_address == 0) panic("pmap_bootstrap: no local apic!"); /* 0 = private page */ /* 1 = page table page */ /* 2 = local apic */ /* 16-31 = io apics */ SMP_prvpt[2] = PG_V | PG_RW | ((u_long)cpu_apic_address & PG_FRAME); for (i = 0; i < mp_napics; i++) { for (j = 0; j < 16; j++) { /* same page frame as a previous IO apic? */ if (((u_long)SMP_prvpt[j + 16] & PG_FRAME) == ((u_long)io_apic_address[0] & PG_FRAME)) { ioapic[i] = (ioapic_t *)&SMP_ioapic[j * PAGE_SIZE]; break; } /* use this slot if available */ if (((u_long)SMP_prvpt[j + 16] & PG_FRAME) == 0) { SMP_prvpt[j + 16] = PG_V | PG_RW | ((u_long)io_apic_address[i] & PG_FRAME); ioapic[i] = (ioapic_t *)&SMP_ioapic[j * PAGE_SIZE]; break; } } if (j == 16) panic("no space to map IO apic %d!", i); } #endif invltlb(); pgeflag = 0; #if !defined(SMP) - if (cpu_feature & CPUID_PGE) + if (cpu_feature & CPUID_PGE) { pgeflag = PG_G; -#endif /* !SMP */ + } +#endif /* * Initialize the 4MB page size flag */ pseflag = 0; /* * The 4MB page version of the initial * kernel page mapping. */ pdir4mb = 0; +#if !defined(DISABLE_PSE) if (cpu_feature & CPUID_PSE) { unsigned ptditmp; /* * Enable the PSE mode */ load_cr4(rcr4() | CR4_PSE); /* * Note that we have enabled PSE mode */ pseflag = PG_PS; ptditmp = (unsigned) kernel_pmap->pm_pdir[KPTDI]; ptditmp &= ~(NBPDR - 1); ptditmp |= PG_V | PG_RW | PG_PS | PG_U | pgeflag; pdir4mb = ptditmp; /* * We can do the mapping here for the single processor * case. We simply ignore the old page table page from * now on. */ #if !defined(SMP) PTD[KPTDI] = (pd_entry_t) ptditmp; kernel_pmap->pm_pdir[KPTDI] = (pd_entry_t) ptditmp; invltlb(); #endif } +#endif } -#if defined(SMP) /* * Set 4mb pdir for mp startup, and global flags */ void pmap_set_opt(unsigned *pdir) { int i; if (pseflag && (cpu_feature & CPUID_PSE)) { load_cr4(rcr4() | CR4_PSE); if (pdir4mb) { (unsigned) pdir[KPTDI] = pdir4mb; } } - if (cpu_feature & CPUID_PGE) { + if (pgeflag && (cpu_feature & CPUID_PGE)) { load_cr4(rcr4() | CR4_PGE); for(i = KPTDI; i < KPTDI + nkpt; i++) { if (pdir[i]) { pdir[i] |= PG_G; } } } } /* * Setup the PTD for the boot processor */ void pmap_set_opt_bsp(void) { pmap_set_opt((unsigned *)kernel_pmap->pm_pdir); pmap_set_opt((unsigned *)PTD); invltlb(); } -#endif /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. * pmap_init has been enhanced to support in a fairly consistant * way, discontiguous physical memory. */ void pmap_init(phys_start, phys_end) vm_offset_t phys_start, phys_end; { vm_offset_t addr; vm_size_t s; int i, npg; /* * calculate the number of pv_entries needed */ vm_first_phys = phys_avail[0]; for (i = 0; phys_avail[i + 1]; i += 2); npg = (phys_avail[(i - 2) + 1] - vm_first_phys) / PAGE_SIZE; /* * Allocate memory for random pmap data structures. Includes the * pv_head_table. */ s = (vm_size_t) (sizeof(pv_table_t) * npg); s = round_page(s); addr = (vm_offset_t) kmem_alloc(kernel_map, s); pv_table = (pv_table_t *) addr; for(i = 0; i < npg; i++) { vm_offset_t pa; TAILQ_INIT(&pv_table[i].pv_list); pv_table[i].pv_list_count = 0; pa = vm_first_phys + i * PAGE_SIZE; pv_table[i].pv_vm_page = PHYS_TO_VM_PAGE(pa); } TAILQ_INIT(&pv_freelist); /* * init the pv free list */ init_pv_entries(npg); /* * Now it is safe to enable pv_table recording. */ pmap_initialized = TRUE; } /* * Used to map a range of physical addresses into kernel * virtual address space. * * For now, VM is already on, we only need to map the * specified memory. */ vm_offset_t pmap_map(virt, start, end, prot) vm_offset_t virt; vm_offset_t start; vm_offset_t end; int prot; { while (start < end) { pmap_enter(kernel_pmap, virt, start, prot, FALSE); virt += PAGE_SIZE; start += PAGE_SIZE; } return (virt); } /*************************************************** * Low level helper routines..... ***************************************************/ #if defined(PMAP_DIAGNOSTIC) /* * This code checks for non-writeable/modified pages. * This should be an invalid condition. */ static int pmap_nw_modified(pt_entry_t ptea) { int pte; pte = (int) ptea; if ((pte & (PG_M|PG_RW)) == PG_M) return 1; else return 0; } #endif /* * this routine defines the region(s) of memory that should * not be tested for the modified bit. */ static PMAP_INLINE int pmap_track_modified( vm_offset_t va) { if ((va < clean_sva) || (va >= clean_eva)) return 1; else return 0; } static PMAP_INLINE void invltlb_1pg( vm_offset_t va) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) { invltlb(); } else #endif { invlpg(va); } } static PMAP_INLINE void invltlb_2pg( vm_offset_t va1, vm_offset_t va2) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) { invltlb(); } else #endif { invlpg(va1); invlpg(va2); } } static unsigned * get_ptbase(pmap) pmap_t pmap; { unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; /* are we current address space or kernel? */ if (pmap == kernel_pmap || frame == (((unsigned) PTDpde) & PG_FRAME)) { return (unsigned *) PTmap; } /* otherwise, we are alternate address space */ if (frame != (((unsigned) APTDpde) & PG_FRAME)) { APTDpde = (pd_entry_t) (frame | PG_RW | PG_V); invltlb(); } return (unsigned *) APTmap; } /* * Super fast pmap_pte routine best used when scanning * the pv lists. This eliminates many coarse-grained * invltlb calls. Note that many of the pv list * scans are across different pmaps. It is very wasteful * to do an entire invltlb for checking a single mapping. */ static unsigned * pmap_pte_quick(pmap, va) register pmap_t pmap; vm_offset_t va; { unsigned pde, newpf; if (pde = (unsigned) pmap->pm_pdir[va >> PDRSHIFT]) { unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; unsigned index = i386_btop(va); /* are we current address space or kernel? */ if ((pmap == kernel_pmap) || (frame == (((unsigned) PTDpde) & PG_FRAME))) { return (unsigned *) PTmap + index; } newpf = pde & PG_FRAME; if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) { * (unsigned *) PMAP1 = newpf | PG_RW | PG_V; invltlb_1pg((vm_offset_t) PADDR1); } return PADDR1 + ((unsigned) index & (NPTEPG - 1)); } return (0); } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_offset_t pmap_extract(pmap, va) register pmap_t pmap; vm_offset_t va; { vm_offset_t rtval; vm_offset_t pdirindex; pdirindex = va >> PDRSHIFT; if (pmap) { unsigned *pte; if (((rtval = (unsigned) pmap->pm_pdir[pdirindex]) & PG_PS) != 0) { rtval &= ~(NBPDR - 1); rtval |= va & (NBPDR - 1); return rtval; } pte = get_ptbase(pmap) + i386_btop(va); rtval = ((*pte & PG_FRAME) | (va & PAGE_MASK)); return rtval; } return 0; } /* * determine if a page is managed (memory vs. device) */ static PMAP_INLINE int pmap_is_managed(pa) vm_offset_t pa; { int i; if (!pmap_initialized) return 0; for (i = 0; phys_avail[i + 1]; i += 2) { if (pa < phys_avail[i + 1] && pa >= phys_avail[i]) return 1; } return 0; } /*************************************************** * Low level mapping routines..... ***************************************************/ /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. */ void pmap_qenter(va, m, count) vm_offset_t va; vm_page_t *m; int count; { int i; register unsigned *pte; for (i = 0; i < count; i++) { vm_offset_t tva = va + i * PAGE_SIZE; unsigned npte = VM_PAGE_TO_PHYS(m[i]) | PG_RW | PG_V | pgeflag; unsigned opte; pte = (unsigned *)vtopte(tva); opte = *pte; *pte = npte; if (opte) invltlb_1pg(tva); } } /* * this routine jerks page mappings from the * kernel -- it is meant only for temporary mappings. */ void pmap_qremove(va, count) vm_offset_t va; int count; { int i; register unsigned *pte; for (i = 0; i < count; i++) { pte = (unsigned *)vtopte(va); *pte = 0; invltlb_1pg(va); va += PAGE_SIZE; } } /* * add a wired page to the kva * note that in order for the mapping to take effect -- you * should do a invltlb after doing the pmap_kenter... */ PMAP_INLINE void pmap_kenter(va, pa) vm_offset_t va; register vm_offset_t pa; { register unsigned *pte; unsigned npte, opte; npte = pa | PG_RW | PG_V | pgeflag; pte = (unsigned *)vtopte(va); opte = *pte; *pte = npte; if (opte) invltlb_1pg(va); } /* * remove a page from the kernel pagetables */ PMAP_INLINE void pmap_kremove(va) vm_offset_t va; { register unsigned *pte; pte = (unsigned *)vtopte(va); *pte = 0; invltlb_1pg(va); } static vm_page_t pmap_page_alloc(object, pindex) vm_object_t object; vm_pindex_t pindex; { vm_page_t m; m = vm_page_alloc(object, pindex, VM_ALLOC_ZERO); if (m == NULL) { VM_WAIT; } return m; } static vm_page_t pmap_page_lookup(object, pindex) vm_object_t object; vm_pindex_t pindex; { vm_page_t m; retry: m = vm_page_lookup(object, pindex); if (m) { if (m->flags & PG_BUSY) { m->flags |= PG_WANTED; tsleep(m, PVM, "pplookp", 0); goto retry; } } return m; } /* * Create the UPAGES for a new process. * This routine directly affects the fork perf for a process. */ void pmap_new_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; struct user *up; unsigned *ptek; /* * allocate object for the upages */ upobj = vm_object_allocate( OBJT_DEFAULT, UPAGES); p->p_upages_obj = upobj; /* get a kernel virtual address for the UPAGES for this proc */ up = (struct user *) kmem_alloc_pageable(u_map, UPAGES * PAGE_SIZE); if (up == NULL) panic("pmap_new_proc: u_map allocation failed"); ptek = (unsigned *) vtopte((vm_offset_t) up); for(i=0;iwire_count++; ++cnt.v_wire_count; /* * Enter the page into the kernel address space. */ *(ptek + i) = VM_PAGE_TO_PHYS(m) | PG_RW | PG_V | pgeflag; m->flags &= ~(PG_ZERO|PG_BUSY); m->flags |= PG_MAPPED|PG_WRITEABLE; m->valid = VM_PAGE_BITS_ALL; } p->p_addr = up; } /* * Dispose the UPAGES for a process that has exited. * This routine directly impacts the exit perf of a process. */ void pmap_dispose_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; unsigned *ptek; ptek = (unsigned *) vtopte((vm_offset_t) p->p_addr); upobj = p->p_upages_obj; for(i=0;ip_addr + i * PAGE_SIZE); vm_page_unwire(m); vm_page_free(m); } vm_object_deallocate(upobj); kmem_free(u_map, (vm_offset_t)p->p_addr, ctob(UPAGES)); } /* * Allow the UPAGES for a process to be prejudicially paged out. */ void pmap_swapout_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; upobj = p->p_upages_obj; /* * let the upages be paged */ for(i=0;idirty = VM_PAGE_BITS_ALL; vm_page_unwire(m); vm_page_deactivate(m); pmap_kremove( (vm_offset_t) p->p_addr + PAGE_SIZE * i); } } /* * Bring the UPAGES for a specified process back in. */ void pmap_swapin_proc(p) struct proc *p; { int i; vm_object_t upobj; vm_page_t m; unsigned *pte; upobj = p->p_upages_obj; for(i=0;iflags & PG_BUSY) || m->busy) { m->flags |= PG_WANTED; tsleep(m, PVM, "swinuw",0); goto retry; } m->flags |= PG_BUSY; } vm_page_wire(m); splx(s); pmap_kenter(((vm_offset_t) p->p_addr) + i * PAGE_SIZE, VM_PAGE_TO_PHYS(m)); if (m->valid != VM_PAGE_BITS_ALL) { int rv; rv = vm_pager_get_pages(upobj, &m, 1, 0); if (rv != VM_PAGER_OK) panic("pmap_swapin_proc: cannot get upages for proc: %d\n", p->p_pid); m->valid = VM_PAGE_BITS_ALL; } PAGE_WAKEUP(m); m->flags |= PG_MAPPED|PG_WRITEABLE; } } /*************************************************** * Page table page management routines..... ***************************************************/ /* * This routine unholds page table pages, and if the hold count * drops to zero, then it decrements the wire count. */ static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { int s; if (m->flags & PG_BUSY) { s = splvm(); while (m->flags & PG_BUSY) { m->flags |= PG_WANTED; tsleep(m, PVM, "pmuwpt", 0); } splx(s); } if (m->hold_count == 0) { vm_offset_t pteva; /* * unmap the page table page */ pmap->pm_pdir[m->pindex] = 0; --pmap->pm_stats.resident_count; if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) == (((unsigned) PTDpde) & PG_FRAME)) { /* * Do a invltlb to make the invalidated mapping * take effect immediately. */ pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex); invltlb_1pg(pteva); } #if defined(PTPHINT) if (pmap->pm_ptphint == m) pmap->pm_ptphint = NULL; #endif /* * If the page is finally unwired, simply free it. */ --m->wire_count; if (m->wire_count == 0) { if (m->flags & PG_WANTED) { m->flags &= ~PG_WANTED; wakeup(m); } vm_page_free_zero(m); --cnt.v_wire_count; } return 1; } return 0; } __inline static int pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { vm_page_unhold(m); if (m->hold_count == 0) return _pmap_unwire_pte_hold(pmap, m); else return 0; } /* * After removing a page table entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ static int pmap_unuse_pt(pmap, va, mpte) pmap_t pmap; vm_offset_t va; vm_page_t mpte; { unsigned ptepindex; if (va >= UPT_MIN_ADDRESS) return 0; if (mpte == NULL) { ptepindex = (va >> PDRSHIFT); #if defined(PTPHINT) if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == ptepindex)) { mpte = pmap->pm_ptphint; } else { mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); pmap->pm_ptphint = mpte; } #else mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); #endif } return pmap_unwire_pte_hold(pmap, mpte); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ void pmap_pinit(pmap) register struct pmap *pmap; { vm_page_t ptdpg; /* * No need to allocate page table space yet but we do need a valid * page directory table. */ if (pdstackptr > 0) { --pdstackptr; pmap->pm_pdir = (pd_entry_t *)pdstack[pdstackptr]; } else { pmap->pm_pdir = (pd_entry_t *)kmem_alloc_pageable(kernel_map, PAGE_SIZE); } /* * allocate object for the ptes */ pmap->pm_pteobj = vm_object_allocate( OBJT_DEFAULT, PTDPTDI + 1); /* * allocate the page directory page */ retry: ptdpg = pmap_page_alloc( pmap->pm_pteobj, PTDPTDI); if (ptdpg == NULL) goto retry; ptdpg->wire_count = 1; ++cnt.v_wire_count; ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */ ptdpg->valid = VM_PAGE_BITS_ALL; pmap_kenter((vm_offset_t) pmap->pm_pdir, VM_PAGE_TO_PHYS(ptdpg)); if ((ptdpg->flags & PG_ZERO) == 0) bzero(pmap->pm_pdir, PAGE_SIZE); /* wire in kernel global address entries */ /* XXX copies current process, does not fill in MPPTDI */ bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * PTESIZE); /* install self-referential address mapping entry */ *(unsigned *) (pmap->pm_pdir + PTDPTDI) = VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW; pmap->pm_flags = 0; pmap->pm_count = 1; pmap->pm_ptphint = NULL; #if PMAP_PVLIST TAILQ_INIT(&pmap->pm_pvlist); #endif } static int pmap_release_free_page(pmap, p) struct pmap *pmap; vm_page_t p; { int s; unsigned *pde = (unsigned *) pmap->pm_pdir; /* * This code optimizes the case of freeing non-busy * page-table pages. Those pages are zero now, and * might as well be placed directly into the zero queue. */ s = splvm(); if (p->flags & PG_BUSY) { p->flags |= PG_WANTED; tsleep(p, PVM, "pmaprl", 0); splx(s); return 0; } if (p->flags & PG_WANTED) { p->flags &= ~PG_WANTED; wakeup(p); } /* * Remove the page table page from the processes address space. */ pde[p->pindex] = 0; --pmap->pm_stats.resident_count; if (p->hold_count) { panic("pmap_release: freeing held page table page"); } /* * Page directory pages need to have the kernel * stuff cleared, so they can go into the zero queue also. */ if (p->pindex == PTDPTDI) { bzero(pde + KPTDI, nkpt * PTESIZE); #ifdef SMP pde[MPPTDI] = 0; #endif pde[APTDPTDI] = 0; pmap_kremove((vm_offset_t) pmap->pm_pdir); } #if defined(PTPHINT) if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == p->pindex)) pmap->pm_ptphint = NULL; #endif vm_page_free_zero(p); splx(s); return 1; } /* * this routine is called if the page table page is not * mapped correctly. */ static vm_page_t _pmap_allocpte(pmap, ptepindex) pmap_t pmap; unsigned ptepindex; { vm_offset_t pteva, ptepa; vm_page_t m; int needszero = 0; /* * Find or fabricate a new pagetable page */ retry: m = vm_page_lookup(pmap->pm_pteobj, ptepindex); if (m == NULL) { m = pmap_page_alloc(pmap->pm_pteobj, ptepindex); if (m == NULL) goto retry; if ((m->flags & PG_ZERO) == 0) needszero = 1; m->flags &= ~(PG_ZERO|PG_BUSY); m->valid = VM_PAGE_BITS_ALL; } else { if ((m->flags & PG_BUSY) || m->busy) { m->flags |= PG_WANTED; tsleep(m, PVM, "ptewai", 0); goto retry; } } if (m->queue != PQ_NONE) { int s = splvm(); vm_page_unqueue(m); splx(s); } if (m->wire_count == 0) ++cnt.v_wire_count; ++m->wire_count; /* * Increment the hold count for the page table page * (denoting a new mapping.) */ ++m->hold_count; /* * Map the pagetable page into the process address space, if * it isn't already there. */ pmap->pm_stats.resident_count++; ptepa = VM_PAGE_TO_PHYS(m); pmap->pm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V); #if defined(PTPHINT) /* * Set the page table hint */ pmap->pm_ptphint = m; #endif /* * Try to use the new mapping, but if we cannot, then * do it with the routine that maps the page explicitly. */ if (needszero) { if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) == (((unsigned) PTDpde) & PG_FRAME)) { pteva = UPT_MIN_ADDRESS + i386_ptob(ptepindex); bzero((caddr_t) pteva, PAGE_SIZE); } else { pmap_zero_page(ptepa); } } m->valid = VM_PAGE_BITS_ALL; m->flags |= PG_MAPPED; return m; } static vm_page_t pmap_allocpte(pmap, va) pmap_t pmap; vm_offset_t va; { unsigned ptepindex; vm_offset_t ptepa; vm_page_t m; /* * Calculate pagetable page index */ ptepindex = va >> PDRSHIFT; /* * Get the page directory entry */ ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex]; /* * This supports switching from a 4MB page to a * normal 4K page. */ if (ptepa & PG_PS) { pmap->pm_pdir[ptepindex] = 0; ptepa = 0; invltlb(); } /* * If the page table page is mapped, we just increment the * hold count, and activate it. */ if (ptepa) { #if defined(PTPHINT) /* * In order to get the page table page, try the * hint first. */ if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == ptepindex)) { m = pmap->pm_ptphint; } else { m = pmap_page_lookup( pmap->pm_pteobj, ptepindex); pmap->pm_ptphint = m; } #else m = pmap_page_lookup( pmap->pm_pteobj, ptepindex); #endif ++m->hold_count; return m; } /* * Here if the pte page isn't mapped, or if it has been deallocated. */ return _pmap_allocpte(pmap, ptepindex); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap) register struct pmap *pmap; { vm_page_t p,n,ptdpg; vm_object_t object = pmap->pm_pteobj; #if defined(DIAGNOSTIC) if (object->ref_count != 1) panic("pmap_release: pteobj reference count != 1"); #endif ptdpg = NULL; retry: for (p = TAILQ_FIRST(&object->memq); p != NULL; p = n) { n = TAILQ_NEXT(p, listq); if (p->pindex == PTDPTDI) { ptdpg = p; continue; } if (!pmap_release_free_page(pmap, p)) goto retry; } if (ptdpg && !pmap_release_free_page(pmap, ptdpg)) goto retry; vm_object_deallocate(object); if (pdstackptr < PDSTACKMAX) { pdstack[pdstackptr] = (vm_offset_t) pmap->pm_pdir; ++pdstackptr; } else { kmem_free(kernel_map, (vm_offset_t) pmap->pm_pdir, PAGE_SIZE); } pmap->pm_pdir = 0; } /* * grow the number of kernel page table entries, if needed */ void pmap_growkernel(vm_offset_t addr) { struct proc *p; struct pmap *pmap; int s; s = splhigh(); if (kernel_vm_end == 0) { kernel_vm_end = KERNBASE; nkpt = 0; while (pdir_pde(PTD, kernel_vm_end)) { kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); ++nkpt; } } addr = (addr + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); while (kernel_vm_end < addr) { if (pdir_pde(PTD, kernel_vm_end)) { kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); continue; } ++nkpt; if (!nkpg) { vm_offset_t ptpkva = (vm_offset_t) vtopte(addr); /* * This index is bogus, but out of the way */ vm_pindex_t ptpidx = (ptpkva >> PAGE_SHIFT); nkpg = vm_page_alloc(kernel_object, ptpidx, VM_ALLOC_SYSTEM); if (!nkpg) panic("pmap_growkernel: no memory to grow kernel"); vm_page_wire(nkpg); vm_page_remove(nkpg); pmap_zero_page(VM_PAGE_TO_PHYS(nkpg)); } pdir_pde(PTD, kernel_vm_end) = (pd_entry_t) (VM_PAGE_TO_PHYS(nkpg) | PG_V | PG_RW | pgeflag); nkpg = NULL; for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { if (p->p_vmspace) { pmap = &p->p_vmspace->vm_pmap; *pmap_pde(pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); } } *pmap_pde(kernel_pmap, kernel_vm_end) = pdir_pde(PTD, kernel_vm_end); kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); } splx(s); } /* * Retire the given physical map from service. * Should only be called if the map contains * no valid mappings. */ void pmap_destroy(pmap) register pmap_t pmap; { int count; if (pmap == NULL) return; count = --pmap->pm_count; if (count == 0) { pmap_release(pmap); free((caddr_t) pmap, M_VMPMAP); } } /* * Add a reference to the specified pmap. */ void pmap_reference(pmap) pmap_t pmap; { if (pmap != NULL) { pmap->pm_count++; } } /*************************************************** * page management routines. ***************************************************/ /* * free the pv_entry back to the free list */ static PMAP_INLINE void free_pv_entry(pv) pv_entry_t pv; { ++pv_freelistcnt; TAILQ_INSERT_HEAD(&pv_freelist, pv, pv_list); } /* * get a new pv_entry, allocating a block from the system * when needed. * the memory allocation is performed bypassing the malloc code * because of the possibility of allocations at interrupt time. */ static pv_entry_t get_pv_entry() { pv_entry_t tmp; /* * get more pv_entry pages if needed */ if (pv_freelistcnt < PV_FREELIST_MIN || !TAILQ_FIRST(&pv_freelist)) { pmap_alloc_pv_entry(); } /* * get a pv_entry off of the free list */ --pv_freelistcnt; tmp = TAILQ_FIRST(&pv_freelist); TAILQ_REMOVE(&pv_freelist, tmp, pv_list); return tmp; } /* * This *strange* allocation routine eliminates the possibility of a malloc * failure (*FATAL*) for a pv_entry_t data structure. * also -- this code is MUCH MUCH faster than the malloc equiv... * We really need to do the slab allocator thingie here. */ static void pmap_alloc_pv_entry() { /* * do we have any pre-allocated map-pages left? */ if (npvvapg) { vm_page_t m; /* * allocate a physical page out of the vm system */ m = vm_page_alloc(kernel_object, OFF_TO_IDX(pvva - vm_map_min(kernel_map)), VM_ALLOC_INTERRUPT); if (m) { int newentries; int i; pv_entry_t entry; newentries = (PAGE_SIZE / sizeof(struct pv_entry)); /* * wire the page */ vm_page_wire(m); m->flags &= ~PG_BUSY; /* * let the kernel see it */ pmap_kenter(pvva, VM_PAGE_TO_PHYS(m)); entry = (pv_entry_t) pvva; /* * update the allocation pointers */ pvva += PAGE_SIZE; --npvvapg; /* * free the entries into the free list */ for (i = 0; i < newentries; i++) { free_pv_entry(entry); entry++; } } } if (!TAILQ_FIRST(&pv_freelist)) panic("get_pv_entry: cannot get a pv_entry_t"); } /* * init the pv_entry allocation system */ void init_pv_entries(npg) int npg; { /* * Allocate enough kvm space for one entry per page, and * each process having PMAP_SHPGPERPROC pages shared with other * processes. (The system can panic if this is too small, but also * can fail on bootup if this is too big.) * XXX The pv management mechanism needs to be fixed so that systems * with lots of shared mappings amongst lots of processes will still * work. The fix will likely be that once we run out of pv entries * we will free other entries (and the associated mappings), with * some policy yet to be determined. */ npvvapg = ((PMAP_SHPGPERPROC * maxproc + npg) * sizeof(struct pv_entry) + PAGE_SIZE - 1) / PAGE_SIZE; pvva = kmem_alloc_pageable(kernel_map, npvvapg * PAGE_SIZE); /* * get the first batch of entries */ pmap_alloc_pv_entry(); } /* * If it is the first entry on the list, it is actually * in the header and we must copy the following entry up * to the header. Otherwise we must search the list for * the entry. In either case we free the now unused entry. */ static int pmap_remove_entry(pmap, ppv, va) struct pmap *pmap; pv_table_t *ppv; vm_offset_t va; { pv_entry_t pv; int rtval; int s; s = splvm(); #if PMAP_PVLIST if (ppv->pv_list_count < pmap->pm_stats.resident_count) { #endif for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { if (pmap == pv->pv_pmap && va == pv->pv_va) break; } #if PMAP_PVLIST } else { for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = TAILQ_NEXT(pv, pv_plist)) { if (va == pv->pv_va) break; } } #endif rtval = 0; if (pv) { rtval = pmap_unuse_pt(pmap, va, pv->pv_ptem); TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); --ppv->pv_list_count; if (TAILQ_FIRST(&ppv->pv_list) == NULL) { ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE); } #if PMAP_PVLIST TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); #endif free_pv_entry(pv); } splx(s); return rtval; } /* * Create a pv entry for page at pa for * (pmap, va). */ static void pmap_insert_entry(pmap, va, mpte, pa) pmap_t pmap; vm_offset_t va; vm_page_t mpte; vm_offset_t pa; { int s; pv_entry_t pv; pv_table_t *ppv; s = splvm(); pv = get_pv_entry(); pv->pv_va = va; pv->pv_pmap = pmap; pv->pv_ptem = mpte; #if PMAP_PVLIST TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); #endif ppv = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&ppv->pv_list, pv, pv_list); ++ppv->pv_list_count; splx(s); } /* * pmap_remove_pte: do the things to unmap a page in a process */ static int pmap_remove_pte(pmap, ptq, va) struct pmap *pmap; unsigned *ptq; vm_offset_t va; { unsigned oldpte; pv_table_t *ppv; oldpte = *ptq; *ptq = 0; if (oldpte & PG_W) pmap->pm_stats.wired_count -= 1; /* * Machines that don't support invlpg, also don't support * PG_G. */ if (oldpte & PG_G) invlpg(va); pmap->pm_stats.resident_count -= 1; if (oldpte & PG_MANAGED) { ppv = pa_to_pvh(oldpte); if (oldpte & PG_M) { #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) oldpte)) { printf("pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, (int) oldpte); } #endif if (pmap_track_modified(va)) ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } return pmap_remove_entry(pmap, ppv, va); } else { return pmap_unuse_pt(pmap, va, NULL); } return 0; } /* * Remove a single page from a process address space */ static void pmap_remove_page(pmap, va) struct pmap *pmap; register vm_offset_t va; { register unsigned *ptq; /* * if there is no pte for this address, just skip it!!! */ if (*pmap_pde(pmap, va) == 0) { return; } /* * get a local va for mappings for this pmap. */ ptq = get_ptbase(pmap) + i386_btop(va); if (*ptq) { (void) pmap_remove_pte(pmap, ptq, va); invltlb_1pg(va); } return; } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ void pmap_remove(pmap, sva, eva) struct pmap *pmap; register vm_offset_t sva; register vm_offset_t eva; { register unsigned *ptbase; vm_offset_t pdnxt; vm_offset_t ptpaddr; vm_offset_t sindex, eindex; int anyvalid; if (pmap == NULL) return; if (pmap->pm_stats.resident_count == 0) return; /* * special handling of removing one page. a very * common operation and easy to short circuit some * code. */ if (((sva + PAGE_SIZE) == eva) && (((unsigned) pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) { pmap_remove_page(pmap, sva); return; } anyvalid = 0; /* * Get a local virtual address for the mappings that are being * worked with. */ ptbase = get_ptbase(pmap); sindex = i386_btop(sva); eindex = i386_btop(eva); for (; sindex < eindex; sindex = pdnxt) { unsigned pdirindex; /* * Calculate index for next page table. */ pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); if (pmap->pm_stats.resident_count == 0) break; pdirindex = sindex / NPDEPG; if (((ptpaddr = (unsigned) pmap->pm_pdir[pdirindex]) & PG_PS) != 0) { pmap->pm_pdir[pdirindex] = 0; pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; anyvalid++; continue; } /* * Weed out invalid mappings. Note: we assume that the page * directory table is always allocated, and in kernel virtual. */ if (ptpaddr == 0) continue; /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the * range being removed. */ if (pdnxt > eindex) { pdnxt = eindex; } for ( ;sindex != pdnxt; sindex++) { vm_offset_t va; if (ptbase[sindex] == 0) { continue; } va = i386_ptob(sindex); anyvalid++; if (pmap_remove_pte(pmap, ptbase + sindex, va)) break; } } if (anyvalid) { invltlb(); } } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ static void pmap_remove_all(pa) vm_offset_t pa; { register pv_entry_t pv; pv_table_t *ppv; register unsigned *pte, tpte; int nmodify; int update_needed; int s; nmodify = 0; update_needed = 0; #if defined(PMAP_DIAGNOSTIC) /* * XXX this makes pmap_page_protect(NONE) illegal for non-managed * pages! */ if (!pmap_is_managed(pa)) { panic("pmap_page_protect: illegal for unmanaged page, va: 0x%lx", pa); } #endif s = splvm(); ppv = pa_to_pvh(pa); while ((pv = TAILQ_FIRST(&ppv->pv_list)) != NULL) { pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); pv->pv_pmap->pm_stats.resident_count--; tpte = *pte; *pte = 0; if (tpte & PG_W) pv->pv_pmap->pm_stats.wired_count--; /* * Update the vm_page_t clean and reference bits. */ if (tpte & PG_M) { #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) tpte)) { printf("pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n", pv->pv_va, tpte); } #endif if (pmap_track_modified(pv->pv_va)) ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } if (!update_needed && ((!curproc || (&curproc->p_vmspace->vm_pmap == pv->pv_pmap)) || (pv->pv_pmap == kernel_pmap))) { update_needed = 1; } #if PMAP_PVLIST TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); #endif TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); --ppv->pv_list_count; pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); free_pv_entry(pv); } ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE); if (update_needed) invltlb(); splx(s); return; } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap, sva, eva, prot) register pmap_t pmap; vm_offset_t sva, eva; vm_prot_t prot; { register unsigned *ptbase; vm_offset_t pdnxt; vm_offset_t ptpaddr; vm_offset_t sindex, eindex; int anychanged; if (pmap == NULL) return; if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; } anychanged = 0; ptbase = get_ptbase(pmap); sindex = i386_btop(sva); eindex = i386_btop(eva); for (; sindex < eindex; sindex = pdnxt) { unsigned pdirindex; pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); pdirindex = sindex / NPDEPG; if (((ptpaddr = (unsigned) pmap->pm_pdir[pdirindex]) & PG_PS) != 0) { (unsigned) pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW); pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; anychanged++; continue; } /* * Weed out invalid mappings. Note: we assume that the page * directory table is always allocated, and in kernel virtual. */ if (ptpaddr == 0) continue; if (pdnxt > eindex) { pdnxt = eindex; } for (; sindex != pdnxt; sindex++) { unsigned pbits = ptbase[sindex]; if (prot & VM_PROT_WRITE) { if ((pbits & (PG_RW|PG_V)) == PG_V) { if (pbits & PG_MANAGED) { vm_page_t m = PHYS_TO_VM_PAGE(pbits); m->flags |= PG_WRITEABLE; m->object->flags |= OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY; } ptbase[sindex] = pbits | PG_RW; anychanged = 1; } } else if (pbits & PG_RW) { if (pbits & PG_M) { vm_offset_t sva = i386_ptob(sindex); if ((pbits & PG_MANAGED) && pmap_track_modified(sva)) { vm_page_t m = PHYS_TO_VM_PAGE(pbits); m->dirty = VM_PAGE_BITS_ALL; } } ptbase[sindex] = pbits & ~(PG_M|PG_RW); anychanged = 1; } } } if (anychanged) invltlb(); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ void pmap_enter(pmap, va, pa, prot, wired) register pmap_t pmap; vm_offset_t va; register vm_offset_t pa; vm_prot_t prot; boolean_t wired; { register unsigned *pte; vm_offset_t opa; vm_offset_t origpte, newpte; vm_page_t mpte; if (pmap == NULL) return; va &= PG_FRAME; #ifdef PMAP_DIAGNOSTIC if (va > VM_MAX_KERNEL_ADDRESS) panic("pmap_enter: toobig"); if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS)) panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)", va); #endif mpte = NULL; /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < UPT_MIN_ADDRESS) mpte = pmap_allocpte(pmap, va); pte = pmap_pte(pmap, va); /* * Page Directory table entry not valid, we need a new PT page */ if (pte == NULL) { panic("pmap_enter: invalid page directory, pdir=%p, va=0x%lx\n", pmap->pm_pdir[PTDPTDI], va); } origpte = *(vm_offset_t *)pte; pa &= PG_FRAME; opa = origpte & PG_FRAME; if (origpte & PG_PS) panic("pmap_enter: attempted pmap_enter on 4MB page"); /* * Mapping has not changed, must be protection or wiring change. */ if (origpte && (opa == pa)) { /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there * are valid mappings in them. Hence, if a user page is wired, * the PT page will be also. */ if (wired && ((origpte & PG_W) == 0)) pmap->pm_stats.wired_count++; else if (!wired && (origpte & PG_W)) pmap->pm_stats.wired_count--; #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) origpte)) { printf("pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n", va, origpte); } #endif /* * We might be turning off write access to the page, * so we go ahead and sense modify status. */ if (origpte & PG_MANAGED) { vm_page_t m; if (origpte & PG_M) { if (pmap_track_modified(va)) { m = PHYS_TO_VM_PAGE(pa); m->dirty = VM_PAGE_BITS_ALL; } } pa |= PG_MANAGED; } if (mpte) --mpte->hold_count; goto validate; } /* * Mapping has changed, invalidate old range and fall through to * handle validating new mapping. */ if (opa) { int err; err = pmap_remove_pte(pmap, pte, va); if (err) panic("pmap_enter: pte vanished, va: 0x%x", va); } /* * Enter on the PV list if part of our managed memory Note that we * raise IPL while manipulating pv_table since pmap_enter can be * called at interrupt time. */ if (pmap_is_managed(pa)) { pmap_insert_entry(pmap, va, mpte, pa); pa |= PG_MANAGED; } /* * Increment counters */ pmap->pm_stats.resident_count++; if (wired) pmap->pm_stats.wired_count++; validate: /* * Now validate mapping with desired protection/wiring. */ newpte = (vm_offset_t) (pa | pte_prot(pmap, prot) | PG_V); if (wired) newpte |= PG_W; if (va < UPT_MIN_ADDRESS) newpte |= PG_U; if (pmap == kernel_pmap) newpte |= pgeflag; /* * if the mapping or permission bits are different, we need * to update the pte. */ if ((origpte & ~(PG_M|PG_A)) != newpte) { *pte = newpte; if (origpte) invltlb_1pg(va); } } /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No page table pages. * 5. Tlbflush is deferred to calling procedure. * 6. Page IS managed. * but is *MUCH* faster than pmap_enter... */ static vm_page_t pmap_enter_quick(pmap, va, pa, mpte) register pmap_t pmap; vm_offset_t va; register vm_offset_t pa; vm_page_t mpte; { register unsigned *pte; /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < UPT_MIN_ADDRESS) { unsigned ptepindex; vm_offset_t ptepa; /* * Calculate pagetable page index */ ptepindex = va >> PDRSHIFT; if (mpte && (mpte->pindex == ptepindex)) { ++mpte->hold_count; } else { retry: /* * Get the page directory entry */ ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex]; /* * If the page table page is mapped, we just increment * the hold count, and activate it. */ if (ptepa) { if (ptepa & PG_PS) panic("pmap_enter_quick: unexpected mapping into 4MB page"); #if defined(PTPHINT) if (pmap->pm_ptphint && (pmap->pm_ptphint->pindex == ptepindex)) { mpte = pmap->pm_ptphint; } else { mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); pmap->pm_ptphint = mpte; } #else mpte = pmap_page_lookup( pmap->pm_pteobj, ptepindex); #endif if (mpte == NULL) goto retry; ++mpte->hold_count; } else { mpte = _pmap_allocpte(pmap, ptepindex); } } } else { mpte = NULL; } /* * This call to vtopte makes the assumption that we are * entering the page into the current pmap. In order to support * quick entry into any pmap, one would likely use pmap_pte_quick. * But that isn't as quick as vtopte. */ pte = (unsigned *)vtopte(va); if (*pte) { if (mpte) pmap_unwire_pte_hold(pmap, mpte); return 0; } /* * Enter on the PV list if part of our managed memory Note that we * raise IPL while manipulating pv_table since pmap_enter can be * called at interrupt time. */ pmap_insert_entry(pmap, va, mpte, pa); /* * Increment counters */ pmap->pm_stats.resident_count++; /* * Now validate mapping with RO protection */ *pte = pa | PG_V | PG_U | PG_MANAGED; return mpte; } #define MAX_INIT_PT (96) /* * pmap_object_init_pt preloads the ptes for a given object * into the specified pmap. This eliminates the blast of soft * faults on process startup and immediately after an mmap. */ void pmap_object_init_pt(pmap, addr, object, pindex, size, limit) pmap_t pmap; vm_offset_t addr; vm_object_t object; vm_pindex_t pindex; vm_size_t size; int limit; { vm_offset_t tmpidx; int psize; vm_page_t p, mpte; int objpgs; if (!pmap) return; /* * This code maps large physical mmap regions into the * processor address space. Note that some shortcuts * are taken, but the code works. */ if (pseflag && (object->type == OBJT_DEVICE) && + ((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0) ) { int i; int s; vm_page_t m[1]; unsigned int ptepindex; int npdes; vm_offset_t ptepa; if (pmap->pm_pdir[ptepindex = (addr >> PDRSHIFT)]) return; s = splhigh(); retry: p = vm_page_lookup(object, pindex); if (p && (p->flags & PG_BUSY)) { tsleep(p, PVM, "init4p", 0); goto retry; } splx(s); if (p == NULL) { p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL); if (p == NULL) return; m[0] = p; if (vm_pager_get_pages(object, m, 1, 0) != VM_PAGER_OK) { PAGE_WAKEUP(p); vm_page_free(p); return; } p = vm_page_lookup(object, pindex); PAGE_WAKEUP(p); } ptepa = (vm_offset_t) VM_PAGE_TO_PHYS(p); if (ptepa & (NBPDR - 1)) { return; } p->valid = VM_PAGE_BITS_ALL; pmap->pm_stats.resident_count += size >> PAGE_SHIFT; npdes = size >> PDRSHIFT; for(i=0;ipm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_PS); ptepa += NBPDR; ptepindex += 1; } p->flags |= PG_MAPPED; -#if 0 invltlb(); -#endif return; } psize = i386_btop(size); if ((object->type != OBJT_VNODE) || (limit && (psize > MAX_INIT_PT) && (object->resident_page_count > MAX_INIT_PT))) { return; } if (psize + pindex > object->size) psize = object->size - pindex; mpte = NULL; /* * if we are processing a major portion of the object, then scan the * entire thing. */ if (psize > (object->size >> 2)) { objpgs = psize; for (p = TAILQ_FIRST(&object->memq); ((objpgs > 0) && (p != NULL)); p = TAILQ_NEXT(p, listq)) { tmpidx = p->pindex; if (tmpidx < pindex) { continue; } tmpidx -= pindex; if (tmpidx >= psize) { continue; } if (((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((p->queue - p->pc) == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; mpte = pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), VM_PAGE_TO_PHYS(p), mpte); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } objpgs -= 1; } } else { /* * else lookup the pages one-by-one. */ for (tmpidx = 0; tmpidx < psize; tmpidx += 1) { p = vm_page_lookup(object, tmpidx + pindex); if (p && ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((p->queue - p->pc) == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; mpte = pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), VM_PAGE_TO_PHYS(p), mpte); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } } } return; } /* * pmap_prefault provides a quick way of clustering * pagefaults into a processes address space. It is a "cousin" * of pmap_object_init_pt, except it runs at page fault time instead * of mmap time. */ #define PFBAK 2 #define PFFOR 2 #define PAGEORDER_SIZE (PFBAK+PFFOR) static int pmap_prefault_pageorder[] = { -PAGE_SIZE, PAGE_SIZE, -2 * PAGE_SIZE, 2 * PAGE_SIZE }; void pmap_prefault(pmap, addra, entry, object) pmap_t pmap; vm_offset_t addra; vm_map_entry_t entry; vm_object_t object; { int i; vm_offset_t starta; vm_offset_t addr; vm_pindex_t pindex; vm_page_t m, mpte; if (entry->object.vm_object != object) return; if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) return; starta = addra - PFBAK * PAGE_SIZE; if (starta < entry->start) { starta = entry->start; } else if (starta > addra) { starta = 0; } mpte = NULL; for (i = 0; i < PAGEORDER_SIZE; i++) { vm_object_t lobject; unsigned *pte; addr = addra + pmap_prefault_pageorder[i]; if (addr < starta || addr >= entry->end) continue; if ((*pmap_pde(pmap, addr)) == NULL) continue; pte = (unsigned *) vtopte(addr); if (*pte) continue; pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT; lobject = object; for (m = vm_page_lookup(lobject, pindex); (!m && (lobject->type == OBJT_DEFAULT) && (lobject->backing_object)); lobject = lobject->backing_object) { if (lobject->backing_object_offset & PAGE_MASK) break; pindex += (lobject->backing_object_offset >> PAGE_SHIFT); m = vm_page_lookup(lobject->backing_object, pindex); } /* * give-up when a page is not in memory */ if (m == NULL) break; if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (m->busy == 0) && (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((m->queue - m->pc) == PQ_CACHE) { vm_page_deactivate(m); } m->flags |= PG_BUSY; mpte = pmap_enter_quick(pmap, addr, VM_PAGE_TO_PHYS(m), mpte); m->flags |= PG_MAPPED; PAGE_WAKEUP(m); } } } /* * Routine: pmap_change_wiring * Function: Change the wiring attribute for a map/virtual-address * pair. * In/out conditions: * The mapping must already exist in the pmap. */ void pmap_change_wiring(pmap, va, wired) register pmap_t pmap; vm_offset_t va; boolean_t wired; { register unsigned *pte; if (pmap == NULL) return; pte = pmap_pte(pmap, va); if (wired && !pmap_pte_w(pte)) pmap->pm_stats.wired_count++; else if (!wired && pmap_pte_w(pte)) pmap->pm_stats.wired_count--; /* * Wiring is not a hardware characteristic so there is no need to * invalidate TLB. */ pmap_pte_set_w(pte, wired); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) pmap_t dst_pmap, src_pmap; vm_offset_t dst_addr; vm_size_t len; vm_offset_t src_addr; { vm_offset_t addr; vm_offset_t end_addr = src_addr + len; vm_offset_t pdnxt; unsigned src_frame, dst_frame; if (dst_addr != src_addr) return; src_frame = ((unsigned) src_pmap->pm_pdir[PTDPTDI]) & PG_FRAME; if (src_frame != (((unsigned) PTDpde) & PG_FRAME)) { return; } dst_frame = ((unsigned) dst_pmap->pm_pdir[PTDPTDI]) & PG_FRAME; if (dst_frame != (((unsigned) APTDpde) & PG_FRAME)) { APTDpde = (pd_entry_t) (dst_frame | PG_RW | PG_V); invltlb(); } for(addr = src_addr; addr < end_addr; addr = pdnxt) { unsigned *src_pte, *dst_pte; vm_page_t dstmpte, srcmpte; vm_offset_t srcptepaddr; unsigned ptepindex; if (addr >= UPT_MIN_ADDRESS) panic("pmap_copy: invalid to pmap_copy page tables\n"); pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1)); ptepindex = addr >> PDRSHIFT; srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[ptepindex]; if (srcptepaddr == 0) continue; if (srcptepaddr & PG_PS) { if (dst_pmap->pm_pdir[ptepindex] == 0) { dst_pmap->pm_pdir[ptepindex] = (pd_entry_t) srcptepaddr; dst_pmap->pm_stats.resident_count += NBPDR; } continue; } srcmpte = vm_page_lookup(src_pmap->pm_pteobj, ptepindex); if ((srcmpte->hold_count == 0) || (srcmpte->flags & PG_BUSY)) continue; if (pdnxt > end_addr) pdnxt = end_addr; src_pte = (unsigned *) vtopte(addr); dst_pte = (unsigned *) avtopte(addr); while (addr < pdnxt) { unsigned ptetemp; ptetemp = *src_pte; /* * we only virtual copy managed pages */ if ((ptetemp & PG_MANAGED) != 0) { /* * We have to check after allocpte for the * pte still being around... allocpte can * block. */ dstmpte = pmap_allocpte(dst_pmap, addr); if ((*dst_pte == 0) && (ptetemp = *src_pte)) { /* * Clear the modified and * accessed (referenced) bits * during the copy. */ *dst_pte = ptetemp & ~(PG_M|PG_A); dst_pmap->pm_stats.resident_count++; pmap_insert_entry(dst_pmap, addr, dstmpte, (ptetemp & PG_FRAME)); } else { pmap_unwire_pte_hold(dst_pmap, dstmpte); } if (dstmpte->hold_count >= srcmpte->hold_count) break; } addr += PAGE_SIZE; ++src_pte; ++dst_pte; } } } /* * Routine: pmap_kernel * Function: * Returns the physical map handle for the kernel. */ pmap_t pmap_kernel() { return (kernel_pmap); } /* * pmap_zero_page zeros the specified (machine independent) * page by mapping the page into virtual memory and using * bzero to clear its contents, one machine dependent page * at a time. */ void pmap_zero_page(phys) vm_offset_t phys; { if (*(int *) CMAP2) panic("pmap_zero_page: CMAP busy"); *(int *) CMAP2 = PG_V | PG_RW | (phys & PG_FRAME); bzero(CADDR2, PAGE_SIZE); *(int *) CMAP2 = 0; invltlb_1pg((vm_offset_t) CADDR2); } /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ void pmap_copy_page(src, dst) vm_offset_t src; vm_offset_t dst; { if (*(int *) CMAP1 || *(int *) CMAP2) panic("pmap_copy_page: CMAP busy"); *(int *) CMAP1 = PG_V | PG_RW | (src & PG_FRAME); *(int *) CMAP2 = PG_V | PG_RW | (dst & PG_FRAME); bcopy(CADDR1, CADDR2, PAGE_SIZE); *(int *) CMAP1 = 0; *(int *) CMAP2 = 0; invltlb_2pg( (vm_offset_t) CADDR1, (vm_offset_t) CADDR2); } /* * Routine: pmap_pageable * Function: * Make the specified pages (by pmap, offset) * pageable (or not) as requested. * * A page which is not pageable may not take * a fault; therefore, its page table entry * must remain valid for the duration. * * This routine is merely advisory; pmap_enter * will specify that these pages are to be wired * down (or not) as appropriate. */ void pmap_pageable(pmap, sva, eva, pageable) pmap_t pmap; vm_offset_t sva, eva; boolean_t pageable; { } /* * this routine returns true if a physical page resides * in the given pmap. */ boolean_t pmap_page_exists(pmap, pa) pmap_t pmap; vm_offset_t pa; { register pv_entry_t pv; pv_table_t *ppv; int s; if (!pmap_is_managed(pa)) return FALSE; s = splvm(); ppv = pa_to_pvh(pa); /* * Not found, check current mappings returning immediately if found. */ for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { if (pv->pv_pmap == pmap) { splx(s); return TRUE; } } splx(s); return (FALSE); } #define PMAP_REMOVE_PAGES_CURPROC_ONLY /* * Remove all pages from specified address space * this aids process exit speeds. Also, this code * is special cased for current process only, but * can have the more generic (and slightly slower) * mode enabled. This is much faster than pmap_remove * in the case of running down an entire address space. */ void pmap_remove_pages(pmap, sva, eva) pmap_t pmap; vm_offset_t sva, eva; { unsigned *pte, tpte; pv_table_t *ppv; pv_entry_t pv, npv; int s; #if PMAP_PVLIST #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) { printf("warning: pmap_remove_pages called with non-current pmap\n"); return; } #endif s = splvm(); for(pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { if (pv->pv_va >= eva || pv->pv_va < sva) { npv = TAILQ_NEXT(pv, pv_plist); continue; } #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY pte = (unsigned *)vtopte(pv->pv_va); #else pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); #endif tpte = *pte; /* * We cannot remove wired pages from a process' mapping at this time */ if (tpte & PG_W) { npv = TAILQ_NEXT(pv, pv_plist); continue; } *pte = 0; ppv = pa_to_pvh(tpte); pv->pv_pmap->pm_stats.resident_count--; /* * Update the vm_page_t clean and reference bits. */ if (tpte & PG_M) { ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } npv = TAILQ_NEXT(pv, pv_plist); TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); --ppv->pv_list_count; TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); if (TAILQ_FIRST(&ppv->pv_list) == NULL) { ppv->pv_vm_page->flags &= ~(PG_MAPPED|PG_WRITEABLE); } pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); free_pv_entry(pv); } splx(s); invltlb(); #endif } /* * pmap_testbit tests bits in pte's * note that the testbit/changebit routines are inline, * and a lot of things compile-time evaluate. */ static boolean_t pmap_testbit(pa, bit) register vm_offset_t pa; int bit; { register pv_entry_t pv; pv_table_t *ppv; unsigned *pte; int s; if (!pmap_is_managed(pa)) return FALSE; ppv = pa_to_pvh(pa); if (TAILQ_FIRST(&ppv->pv_list) == NULL) return FALSE; s = splvm(); for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { /* * if the bit being tested is the modified bit, then * mark clean_map and ptes as never * modified. */ if (bit & (PG_A|PG_M)) { if (!pmap_track_modified(pv->pv_va)) continue; } #if defined(PMAP_DIAGNOSTIC) if (!pv->pv_pmap) { printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); continue; } #endif pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (*pte & bit) { splx(s); return TRUE; } } splx(s); return (FALSE); } /* * this routine is used to modify bits in ptes */ static void pmap_changebit(pa, bit, setem) vm_offset_t pa; int bit; boolean_t setem; { register pv_entry_t pv; pv_table_t *ppv; register unsigned *pte; int changed; int s; if (!pmap_is_managed(pa)) return; s = splvm(); changed = 0; ppv = pa_to_pvh(pa); /* * Loop over all current mappings setting/clearing as appropos If * setting RO do we need to clear the VAC? */ for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { /* * don't write protect pager mappings */ if (!setem && (bit == PG_RW)) { if (!pmap_track_modified(pv->pv_va)) continue; } #if defined(PMAP_DIAGNOSTIC) if (!pv->pv_pmap) { printf("Null pmap (cb) at va: 0x%lx\n", pv->pv_va); continue; } #endif pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (setem) { *(int *)pte |= bit; changed = 1; } else { vm_offset_t pbits = *(vm_offset_t *)pte; if (pbits & bit) { changed = 1; if (bit == PG_RW) { if (pbits & PG_M) { ppv->pv_vm_page->dirty = VM_PAGE_BITS_ALL; } *(int *)pte = pbits & ~(PG_M|PG_RW); } else { *(int *)pte = pbits & ~bit; } } } } splx(s); if (changed) invltlb(); } /* * pmap_page_protect: * * Lower the permission for all mappings to a given page. */ void pmap_page_protect(phys, prot) vm_offset_t phys; vm_prot_t prot; { if ((prot & VM_PROT_WRITE) == 0) { if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { pmap_changebit(phys, PG_RW, FALSE); } else { pmap_remove_all(phys); } } } vm_offset_t pmap_phys_address(ppn) int ppn; { return (i386_ptob(ppn)); } /* * pmap_ts_referenced: * * Return the count of reference bits for a page, clearing all of them. * */ int pmap_ts_referenced(vm_offset_t pa) { register pv_entry_t pv; pv_table_t *ppv; unsigned *pte; int s; int rtval = 0; if (!pmap_is_managed(pa)) return FALSE; s = splvm(); ppv = pa_to_pvh(pa); if (TAILQ_FIRST(&ppv->pv_list) == NULL) { splx(s); return 0; } /* * Not found, check current mappings returning immediately if found. */ for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { /* * if the bit being tested is the modified bit, then * mark clean_map and ptes as never * modified. */ if (!pmap_track_modified(pv->pv_va)) continue; pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (pte == NULL) { continue; } if (*pte & PG_A) { rtval++; *pte &= ~PG_A; } } splx(s); if (rtval) { invltlb(); } return (rtval); } /* * pmap_is_modified: * * Return whether or not the specified physical page was modified * in any physical maps. */ boolean_t pmap_is_modified(vm_offset_t pa) { return pmap_testbit((pa), PG_M); } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_offset_t pa) { pmap_changebit((pa), PG_M, FALSE); } /* * pmap_clear_reference: * * Clear the reference bit on the specified physical page. */ void pmap_clear_reference(vm_offset_t pa) { pmap_changebit((pa), PG_A, FALSE); } /* * Miscellaneous support routines follow */ static void i386_protection_init() { register int *kp, prot; kp = protection_codes; for (prot = 0; prot < 8; prot++) { switch (prot) { case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: /* * Read access is also 0. There isn't any execute bit, * so just make it readable. */ case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: *kp++ = 0; break; case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: *kp++ = PG_RW; break; } } } /* * Map a set of physical memory pages into the kernel virtual * address space. Return a pointer to where it is mapped. This * routine is intended to be used for mapping device memory, * NOT real memory. The non-cacheable bits are set on each * mapped page. */ void * pmap_mapdev(pa, size) vm_offset_t pa; vm_size_t size; { vm_offset_t va, tmpva; unsigned *pte; size = roundup(size, PAGE_SIZE); va = kmem_alloc_pageable(kernel_map, size); if (!va) panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); pa = pa & PG_FRAME; for (tmpva = va; size > 0;) { pte = (unsigned *)vtopte(tmpva); *pte = pa | PG_RW | PG_V | pgeflag; size -= PAGE_SIZE; tmpva += PAGE_SIZE; pa += PAGE_SIZE; } invltlb(); return ((void *) va); } /* * perform the pmap work for mincore */ int pmap_mincore(pmap, addr) pmap_t pmap; vm_offset_t addr; { unsigned *ptep, pte; int val = 0; ptep = pmap_pte(pmap, addr); if (ptep == 0) { return 0; } if (pte = *ptep) { vm_offset_t pa; val = MINCORE_INCORE; pa = pte & PG_FRAME; /* * Modified by us */ if (pte & PG_M) val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER; /* * Modified by someone */ else if (PHYS_TO_VM_PAGE(pa)->dirty || pmap_is_modified(pa)) val |= MINCORE_MODIFIED_OTHER; /* * Referenced by us */ if (pte & PG_U) val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER; /* * Referenced by someone */ else if ((PHYS_TO_VM_PAGE(pa)->flags & PG_REFERENCED) || pmap_ts_referenced(pa)) { val |= MINCORE_REFERENCED_OTHER; PHYS_TO_VM_PAGE(pa)->flags |= PG_REFERENCED; } } return val; } void pmap_activate(struct proc *p) { load_cr3(p->p_addr->u_pcb.pcb_cr3 = vtophys(p->p_vmspace->vm_pmap.pm_pdir)); } vm_offset_t pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) { if ((size < NBPDR) || (obj->type != OBJT_DEVICE)) { return addr; } addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1); return addr; } #if defined(PMAP_DEBUG) pmap_pid_dump(int pid) { pmap_t pmap; struct proc *p; int npte = 0; int index; for (p = allproc.lh_first; p != NULL; p = p->p_list.le_next) { if (p->p_pid != pid) continue; if (p->p_vmspace) { int i,j; index = 0; pmap = &p->p_vmspace->vm_pmap; for(i=0;i<1024;i++) { pd_entry_t *pde; unsigned *pte; unsigned base = i << PDRSHIFT; pde = &pmap->pm_pdir[i]; if (pde && pmap_pde_v(pde)) { for(j=0;j<1024;j++) { unsigned va = base + (j << PAGE_SHIFT); if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) { if (index) { index = 0; printf("\n"); } return npte; } pte = pmap_pte_quick( pmap, va); if (pte && pmap_pte_v(pte)) { vm_offset_t pa; vm_page_t m; pa = *(int *)pte; m = PHYS_TO_VM_PAGE((pa & PG_FRAME)); printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x", va, pa, m->hold_count, m->wire_count, m->flags); npte++; index++; if (index >= 2) { index = 0; printf("\n"); } else { printf(" "); } } } } } } } return npte; } #endif #if defined(DEBUG) static void pads __P((pmap_t pm)); static void pmap_pvdump __P((vm_offset_t pa)); /* print address space of pmap*/ static void pads(pm) pmap_t pm; { unsigned va, i, j; unsigned *ptep; if (pm == kernel_pmap) return; for (i = 0; i < 1024; i++) if (pm->pm_pdir[i]) for (j = 0; j < 1024; j++) { va = (i << PDRSHIFT) + (j << PAGE_SHIFT); if (pm == kernel_pmap && va < KERNBASE) continue; if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) continue; ptep = pmap_pte_quick(pm, va); if (pmap_pte_v(ptep)) printf("%x:%x ", va, *(int *) ptep); }; } static void pmap_pvdump(pa) vm_offset_t pa; { pv_table_t *ppv; register pv_entry_t pv; printf("pa %x", pa); ppv = pa_to_pvh(pa); for (pv = TAILQ_FIRST(&ppv->pv_list); pv; pv = TAILQ_NEXT(pv, pv_list)) { #ifdef used_to_be printf(" -> pmap %x, va %x, flags %x", pv->pv_pmap, pv->pv_va, pv->pv_flags); #endif printf(" -> pmap %x, va %x", pv->pv_pmap, pv->pv_va); pads(pv->pv_pmap); } printf(" "); } #endif Index: head/sys/i386/include/mptable.h =================================================================== --- head/sys/i386/include/mptable.h (revision 27483) +++ head/sys/i386/include/mptable.h (revision 27484) @@ -1,1830 +1,1834 @@ /* * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.14 1997/07/13 00:42:14 smp Exp smp $ + * $Id: mp_machdep.c,v 1.27 1997/07/13 01:22:43 fsmp Exp $ */ #include "opt_smp.h" #include /* for KERNBASE */ #include #include #include #include #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include #include #include #include #include #include #include #include /** TEST_DEFAULT_CONFIG, TEST_CPUSTOP _TEST1 */ #include #include #include /* cngetc() */ #if defined(APIC_IO) #include /* setidt() */ #include /* IPIs */ #include /* IPIs */ #endif /* APIC_IO */ #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define BIOS_BASE (0xf0000) #define BIOS_SIZE (0x10000) #define BIOS_COUNT (BIOS_SIZE/4) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) #define PROCENTRY_FLAG_EN 0x01 #define PROCENTRY_FLAG_BP 0x02 #define IOAPICENTRY_FLAG_EN 0x01 /* MP Floating Pointer Structure */ typedef struct MPFPS { char signature[4]; void *pap; u_char length; u_char spec_rev; u_char checksum; u_char mpfb1; u_char mpfb2; u_char mpfb3; u_char mpfb4; u_char mpfb5; } *mpfps_t; /* MP Configuration Table Header */ typedef struct MPCTH { char signature[4]; u_short base_table_length; u_char spec_rev; u_char checksum; u_char oem_id[8]; u_char product_id[12]; void *oem_table_pointer; u_short oem_table_size; u_short entry_count; void *apic_address; u_short extended_table_length; u_char extended_table_checksum; u_char reserved; } *mpcth_t; typedef struct PROCENTRY { u_char type; u_char apic_id; u_char apic_version; u_char cpu_flags; u_long cpu_signature; u_long feature_flags; u_long reserved1; u_long reserved2; } *proc_entry_ptr; typedef struct BUSENTRY { u_char type; u_char bus_id; char bus_type[6]; } *bus_entry_ptr; typedef struct IOAPICENTRY { u_char type; u_char apic_id; u_char apic_version; u_char apic_flags; void *apic_address; } *io_apic_entry_ptr; typedef struct INTENTRY { u_char type; u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } *int_entry_ptr; /* descriptions of MP basetable entries */ typedef struct BASETABLE_ENTRY { u_char type; u_char length; char name[16]; } basetable_entry; /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MP_START_POST 0x12 #define MP_ANNOUNCE_POST 0x13 #define MPTABLE_PASS1_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define MP_ENABLE_POST 0x16 #define START_ALL_APS_POST 0x17 #define INSTALL_AP_TRAMP_POST 0x18 #define START_AP_POST 0x19 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; /** XXX FIXME: what system files declare these??? */ extern struct region_descriptor r_gdt, r_idt; int mp_ncpus; /* # of CPUs, including BSP */ int mp_naps; /* # of Applications processors */ int mp_nbusses; /* # of busses */ int mp_napics; /* # of IO APICs */ int boot_cpu_id; /* designated BSP */ vm_offset_t cpu_apic_address; vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ u_int32_t cpu_apic_versions[NCPU]; u_int32_t io_apic_versions[NAPIC]; /* * APIC ID logical/physical mapping structures. * We oversize these to simplify boot-time config. */ int cpu_num_to_apic_id[NAPICID]; int io_num_to_apic_id[NAPICID]; int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; /* Boot of AP uses this PTD */ u_int *bootPTD; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t KPTphys; /* Virtual address of per-cpu common_tss */ extern struct i386tss common_tss; /* * Local data and functions. */ static int mp_capable; static u_int boot_address; static u_int base_memory; static int picmode; /* 0: virtual wire mode, 1: PIC mode */ static mpfps_t mpfps; static int search_for_sig(u_int32_t target, int count); static void mp_enable(u_int boot_addr); static int mptable_pass1(void); static int mptable_pass2(void); static void default_mp_table(int type); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); base_memory = basemem * 1024; /* convert to bytes */ boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ if ((base_memory - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } /* * Look for an Intel MP spec table (ie, SMP capable hardware). */ int mp_probe(void) { int x; u_long segment; u_int32_t target; POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { /* search first 1K of EBDA */ target = (u_int32_t) (segment << 4); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } else { /* last 1K of base memory, effective 'top of base' passed in */ target = (u_int32_t) (base_memory - 0x400); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } /* search the BIOS */ target = (u_int32_t) BIOS_BASE; if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) goto found; /* nothing found */ mpfps = (mpfps_t)0; mp_capable = 0; return 0; found: /* calculate needed resources */ mpfps = (mpfps_t)x; if (mptable_pass1()) panic("you must reconfigure your kernel"); /* flag fact that we are running multiple processors */ mp_capable = 1; return 1; } /* * Startup the SMP processors. */ void mp_start(void) { POSTCODE(MP_START_POST); /* look for MP capable motherboard */ if (mp_capable) mp_enable(boot_address); else panic("MP hardware not found!"); } /* * Print various information about the SMP system hardware and setup. */ void mp_announce(void) { int x; POSTCODE(MP_ANNOUNCE_POST); printf("FreeBSD/SMP: Multiprocessor motherboard\n"); printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); printf(", version: 0x%08x", cpu_apic_versions[0]); printf(", at 0x%08x\n", cpu_apic_address); for (x = 1; x <= mp_naps; ++x) { printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); printf(", version: 0x%08x", cpu_apic_versions[x]); printf(", at 0x%08x\n", cpu_apic_address); } #if defined(APIC_IO) for (x = 0; x < mp_napics; ++x) { printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); printf(", version: 0x%08x", io_apic_versions[x]); printf(", at 0x%08x\n", io_apic_address[x]); } #else printf(" Warning: APIC I/O disabled\n"); #endif /* APIC_IO */ } /* * AP cpu's call this to sync up protected mode. */ void init_secondary(void) { int gsel_tss, slot; r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; r_gdt.rd_base = (int) gdt; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); slot = NGDT + cpuid; gsel_tss = GSEL(slot, SEL_KPL); gdt[slot].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ PTD[0] = 0; + pmap_set_opt((unsigned *)PTD); + invltlb(); } #if defined(APIC_IO) /* * Final configuration of the BSP's local APIC: * - disable 'pic mode'. * - disable 'virtual wire mode'. * - enable NMI. */ void bsp_apic_configure(void) { u_char byte; u_int32_t temp; /* leave 'pic mode' if necessary */ if (picmode) { outb(0x22, 0x70); /* select IMCR */ byte = inb(0x23); /* current contents */ byte |= 0x01; /* mask external INTR */ outb(0x23, byte); /* disconnect 8259s/NMI */ } /* mask lint0 (the 8259 'virtual wire' connection) */ temp = lapic.lvt_lint0; temp |= APIC_LVT_M; /* set the mask */ lapic.lvt_lint0 = temp; /* setup lint1 to handle NMI */ temp = lapic.lvt_lint1; temp &= ~APIC_LVT_M; /* clear the mask */ lapic.lvt_lint1 = temp; if (bootverbose) apic_dump(); } #endif /* APIC_IO */ /******************************************************************* * local functions and data */ /* * start the SMP system */ static void mp_enable(u_int boot_addr) { int x; #if defined(APIC_IO) int apic; u_int ux; #endif /* APIC_IO */ POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); invltlb(); /* examine the MP table for needed info, uses physical addresses */ x = mptable_pass2(); *(int *)PTD = 0; invltlb(); /* can't process default configs till the CPU APIC is pmapped */ if (x) default_mp_table(x); #if defined(APIC_IO) /* fill the LOGICAL io_apic_versions table */ for (apic = 0; apic < mp_napics; ++apic) { ux = io_apic_read(apic, IOAPIC_VER); io_apic_versions[apic] = ux; } /* program each IO APIC in the system */ for (apic = 0; apic < mp_napics; ++apic) if (io_apic_setup(apic) < 0) panic("IO APIC setup failure"); /* install a 'Spurious INTerrupt' vector */ setidt(XSPURIOUSINT_OFFSET, Xspuriousint, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #if defined(TEST_CPUSTOP) /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_CPUSTOP */ #if defined(TEST_TEST1) /* install a 'Spurious INTerrupt' vector */ setidt(XTEST1_OFFSET, Xtest1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_TEST1 */ #endif /* APIC_IO */ /* start each Application Processor */ start_all_aps(boot_addr); /* * The init process might be started on a different CPU now, * and the boot CPU might not call prepare_usermode to get * cr0 correctly configured. Thus we initialize cr0 here. */ load_cr0(rcr0() | CR0_WP | CR0_AM); } /* * look for the MP spec signature */ /* string defined by the Intel MP Spec as identifying the MP table */ #define MP_SIG 0x5f504d5f /* _MP_ */ #define NEXT(X) ((X) += 4) static int search_for_sig(u_int32_t target, int count) { int x; u_int32_t *addr = (u_int32_t *) (KERNBASE + target); for (x = 0; x < count; NEXT(x)) if (addr[x] == MP_SIG) /* make array index a byte index */ return (target + (x * sizeof(u_int32_t))); return -1; } static basetable_entry basetable_entry_types[] = { {0, 20, "Processor"}, {1, 8, "Bus"}, {2, 8, "I/O APIC"}, {3, 8, "I/O INT"}, {4, 8, "Local INT"} }; typedef struct BUSDATA { u_char bus_id; enum busTypes bus_type; } bus_datum; typedef struct INTDATA { u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } io_int, local_int; typedef struct BUSTYPENAME { u_char type; char name[7]; } bus_type_name; static bus_type_name bus_type_table[] = { {CBUS, "CBUS"}, {CBUSII, "CBUSII"}, {EISA, "EISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {ISA, "ISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {PCI, "PCI"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {XPRESS, "XPRESS"}, {UNKNOWN_BUSTYPE, "---"} }; /* from MP spec v1.4, table 5-1 */ static int default_data[7][5] = { /* nbus, id0, type0, id1, type1 */ {1, 0, ISA, 255, 255}, {1, 0, EISA, 255, 255}, {1, 0, EISA, 255, 255}, {0, 255, 255, 255, 255},/* MCA not supported */ {2, 0, ISA, 1, PCI}, {2, 0, EISA, 1, PCI}, {0, 255, 255, 255, 255} /* MCA not supported */ }; /* the bus data */ bus_datum bus_data[NBUS]; /* the IO INT data, one entry per possible APIC INTerrupt */ io_int io_apic_ints[NINTR]; static int nintrs; static void fix_mp_table __P((void)); static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); /* * 1st pass on motherboard's Intel MP specification table. * * initializes: * mp_ncpus = 1 * * determines: * cpu_apic_address (common to all CPUs) * io_apic_address[N] * mp_naps * mp_nbusses * mp_napics * nintrs */ static int mptable_pass1(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int mustpanic; POSTCODE(MPTABLE_PASS1_POST); mustpanic = 0; /* clear various tables */ for (x = 0; x < NAPICID; ++x) { io_apic_address[x] = ~0; /* IO APIC address table */ } /* init everything to empty */ mp_naps = 0; mp_nbusses = 0; mp_napics = 0; nintrs = 0; /* check for use of 'default' configuration */ if (mpfps->mpfb1 != 0) { /* use default addresses */ cpu_apic_address = DEFAULT_APIC_BASE; io_apic_address[0] = DEFAULT_IO_APIC_BASE; /* fill in with defaults */ mp_naps = 2; /* includes BSP */ mp_nbusses = default_data[mpfps->mpfb1 - 1][0]; #if defined(APIC_IO) mp_napics = 1; nintrs = 16; #endif /* APIC_IO */ } else { if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); cpu_apic_address = (vm_offset_t) cth->apic_address; /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; while (count--) { switch (type = *(u_char *) position) { case 0: /* processor_entry */ if (((proc_entry_ptr)position)->cpu_flags & PROCENTRY_FLAG_EN) ++mp_naps; break; case 1: /* bus_entry */ ++mp_nbusses; break; case 2: /* io_apic_entry */ if (((io_apic_entry_ptr)position)->apic_flags & IOAPICENTRY_FLAG_EN) io_apic_address[mp_napics++] = (vm_offset_t)((io_apic_entry_ptr) position)->apic_address; break; case 3: /* int_entry */ ++nintrs; break; case 4: /* int_entry */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char*)position += basetable_entry_types[type].length; } } /* qualify the numbers */ if (mp_naps > NCPU) printf("Warning: only using %d of %d available CPUs!\n", NCPU, mp_naps); #if 0 /** XXX we consider this legal now (but should we?) */ mustpanic = 1; #endif if (mp_nbusses > NBUS) { printf("found %d busses, increase NBUS\n", mp_nbusses); mustpanic = 1; } if (mp_napics > NAPIC) { printf("found %d apics, increase NAPIC\n", mp_napics); mustpanic = 1; } if (nintrs > NINTR) { printf("found %d intrs, increase NINTR\n", nintrs); mustpanic = 1; } /* * Count the BSP. * This is also used as a counter while starting the APs. */ mp_ncpus = 1; --mp_naps; /* subtract the BSP */ return mustpanic; } /* * 2nd pass on motherboard's Intel MP specification table. * * sets: * boot_cpu_id * ID_TO_IO(N), phy APIC ID to log CPU/IO table * CPU_TO_ID(N), logical CPU to APIC ID table * IO_TO_ID(N), logical IO to APIC ID table * bus_data[N] * io_apic_ints[N] */ static int mptable_pass2(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int apic, bus, cpu, intr; POSTCODE(MPTABLE_PASS2_POST); /* clear various tables */ for (x = 0; x < NAPICID; ++x) { ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ } /* clear bus data table */ for (x = 0; x < NBUS; ++x) bus_data[x].bus_id = 0xff; /* clear IO APIC INT table */ for (x = 0; x < NINTR; ++x) io_apic_ints[x].int_type = 0xff; /* setup the cpu/apic mapping arrays */ boot_cpu_id = -1; /* record whether PIC or virtual-wire mode */ picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; /* check for use of 'default' configuration */ #if defined(TEST_DEFAULT_CONFIG) return TEST_DEFAULT_CONFIG; #else if (mpfps->mpfb1 != 0) return mpfps->mpfb1; /* return default configuration type */ #endif /* TEST_DEFAULT_CONFIG */ if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; apic = bus = intr = 0; cpu = 1; /* pre-count the BSP */ while (count--) { switch (type = *(u_char *) position) { case 0: if (processor_entry(position, cpu)) ++cpu; break; case 1: if (bus_entry(position, bus)) ++bus; break; case 2: if (io_apic_entry(position, apic)) ++apic; break; case 3: if (int_entry(position, intr)) ++intr; break; case 4: /* int_entry(position); */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char *) position += basetable_entry_types[type].length; } if (boot_cpu_id == -1) panic("NO BSP found!"); /* post scan cleanup */ fix_mp_table(); /* report fact that its NOT a default configuration */ return 0; } /* * parse an Intel MP specification table */ static void fix_mp_table(void) { int x; int id; int bus_0; int bus_pci; int num_pci_bus; /* * Fix mis-numbering of the PCI bus and its INT entries if the BIOS * did it wrong. The MP spec says that when more than 1 PCI bus * exists the BIOS must begin with bus entries for the PCI bus and use * actual PCI bus numbering. This implies that when only 1 PCI bus * exists the BIOS can choose to ignore this ordering, and indeed many * MP motherboards do ignore it. This causes a problem when the PCI * sub-system makes requests of the MP sub-system based on PCI bus * numbers. So here we look for the situation and renumber the * busses and associated INTs in an effort to "make it right". */ /* find bus 0, PCI bus, count the number of PCI busses */ for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_id == 0) { bus_0 = x; } if (bus_data[x].bus_type == PCI) { ++num_pci_bus; bus_pci = x; } } /* * bus_0 == slot of bus with ID of 0 * bus_pci == slot of last PCI bus encountered */ /* check the 1 PCI bus case for sanity */ if (num_pci_bus == 1) { /* if it is number 0 all is well */ if (bus_data[bus_pci].bus_id == 0) return; /* mis-numbered, swap with whichever bus uses slot 0 */ /* swap the bus entry types */ bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; /* swap each relavant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { io_apic_ints[x].src_bus_id = 0; } else if (io_apic_ints[x].src_bus_id == 0) { io_apic_ints[x].src_bus_id = id; } } } /* sanity check if more than 1 PCI bus */ else if (num_pci_bus > 1) { for (x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_type != PCI) continue; if (bus_data[x].bus_id >= num_pci_bus) panic("bad PCI bus numbering"); } } } static int processor_entry(proc_entry_ptr entry, int cpu) { /* check for usability */ if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) return 0; /* check for BSP flag */ if (entry->cpu_flags & PROCENTRY_FLAG_BP) { boot_cpu_id = entry->apic_id; CPU_TO_ID(0) = entry->apic_id; ID_TO_CPU(entry->apic_id) = 0; return 0; /* its already been counted */ } /* add another AP to list, if less than max number of CPUs */ else { CPU_TO_ID(cpu) = entry->apic_id; ID_TO_CPU(entry->apic_id) = cpu; return 1; } } static int bus_entry(bus_entry_ptr entry, int bus) { int x; char c, name[8]; /* encode the name into an index */ for (x = 0; x < 6; ++x) { if ((c = entry->bus_type[x]) == ' ') break; name[x] = c; } name[x] = '\0'; if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) panic("unknown bus type: '%s'", name); bus_data[bus].bus_id = entry->bus_id; bus_data[bus].bus_type = x; return 1; } static int io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; IO_TO_ID(apic) = entry->apic_id; ID_TO_IO(entry->apic_id) = apic; return 1; } static int lookup_bus_type(char *name) { int x; for (x = 0; x < MAX_BUSTYPE; ++x) if (strcmp(bus_type_table[x].name, name) == 0) return bus_type_table[x].type; return UNKNOWN_BUSTYPE; } static int int_entry(int_entry_ptr entry, int intr) { io_apic_ints[intr].int_type = entry->int_type; io_apic_ints[intr].int_flags = entry->int_flags; io_apic_ints[intr].src_bus_id = entry->src_bus_id; io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; } static int apic_int_is_bus_type(int intr, int bus_type) { int bus; for (bus = 0; bus < mp_nbusses; ++bus) if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) && ((int) bus_data[bus].bus_type == bus_type)) return 1; return 0; } /* * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { int isa_irq; int apic_pin; #if defined(SKIP_IRQ15_REDIRECT) if (isa_mask == (1 << 15)) { printf("skipping ISA IRQ15 redirect\n"); return isa_mask; } #endif /* SKIP_IRQ15_REDIRECT */ isa_irq = ffs(isa_mask); /* find its bit position */ if (isa_irq == 0) /* doesn't exist */ return 0; --isa_irq; /* make it zero based */ apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ if (apic_pin == -1) return 0; return (1 << apic_pin); /* convert pin# to a mask */ } /* * Determine which APIC pin an ISA/EISA INT is attached to. */ #define INTTYPE(I) (io_apic_ints[(I)].int_type) #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) int isa_apic_pin(int isa_irq) { int intr; #if defined(SMP_TIMER_NC) if (isa_irq == 0) return -1; #endif /* SMP_TIMER_NC */ for (intr = 0; intr < nintrs; ++intr) { /* check each record */ if (INTTYPE(intr) == 0) { /* standard INT */ if (SRCBUSIRQ(intr) == isa_irq) { if (apic_int_is_bus_type(intr, ISA) || apic_int_is_bus_type(intr, EISA)) return INTPIN(intr); /* found */ } } } return -1; /* NOT found */ } #undef SRCBUSIRQ /* * Determine which APIC pin a PCI INT is attached to. */ #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) int pci_apic_pin(int pciBus, int pciDevice, int pciInt) { int intr; --pciInt; /* zero based */ for (intr = 0; intr < nintrs; ++intr) /* check each record */ if ((INTTYPE(intr) == 0) /* standard INT */ && (SRCBUSID(intr) == pciBus) && (SRCBUSDEVICE(intr) == pciDevice) && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ if (apic_int_is_bus_type(intr, PCI)) return INTPIN(intr); /* exact match */ return -1; /* NOT found */ } #undef SRCBUSLINE #undef SRCBUSDEVICE #undef SRCBUSID #undef INTPIN #undef INTTYPE /* * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. * * XXX FIXME: * Exactly what this means is unclear at this point. It is a solution * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard * could route any of the ISA INTs to upper (>15) IRQ values. But most would * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an * option. */ int undirect_isa_irq(int rirq) { #if defined(READY) printf("Freeing redirected ISA irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); return 0; #endif /* READY */ } /* * Reprogram the MB chipset to NOT redirect a PCI INTerrupt */ int undirect_pci_irq(int rirq) { #if defined(READY) if (bootverbose) printf("Freeing redirected PCI irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else if (bootverbose) printf("Freeing (NOT implemented) redirected PCI irq %d.\n", rirq); return 0; #endif /* READY */ } /* * given a bus ID, return: * the bus type if found * -1 if NOT found */ int apic_bus_type(int id) { int x; for (x = 0; x < mp_nbusses; ++x) if (bus_data[x].bus_id == id) return bus_data[x].bus_type; return -1; } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus ID if found * -1 if NOT found */ int apic_src_bus_id(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_id); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus IRQ if found * -1 if NOT found */ int apic_src_bus_irq(int apic, int pin) { int x; for (x = 0; x < nintrs; x++) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_irq); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated INTerrupt type if found * -1 if NOT found */ int apic_int_type(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_type); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated trigger mode if found * -1 if NOT found */ int apic_trigger(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return ((io_apic_ints[x].int_flags >> 2) & 0x03); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated 'active' level if found * -1 if NOT found */ int apic_polarity(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_flags & 0x03); return -1; /* NOT found */ } /* * set data according to MP defaults * FIXME: probably not complete yet... */ static void default_mp_table(int type) { int ap_cpu_id; #if defined(APIC_IO) u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ #if 0 printf(" MP default config type: %d\n", type); switch (type) { case 1: printf(" bus: ISA, APIC: 82489DX\n"); break; case 2: printf(" bus: EISA, APIC: 82489DX\n"); break; case 3: printf(" bus: EISA, APIC: 82489DX\n"); break; case 4: printf(" bus: MCA, APIC: 82489DX\n"); break; case 5: printf(" bus: ISA+PCI, APIC: Integrated\n"); break; case 6: printf(" bus: EISA+PCI, APIC: Integrated\n"); break; case 7: printf(" bus: MCA+PCI, APIC: Integrated\n"); break; default: printf(" future type\n"); break; /* NOTREACHED */ } #endif /* 0 */ boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; /* BSP */ CPU_TO_ID(0) = boot_cpu_id; ID_TO_CPU(boot_cpu_id) = 0; /* one and only AP */ CPU_TO_ID(1) = ap_cpu_id; ID_TO_CPU(ap_cpu_id) = 1; #if defined(APIC_IO) /* one and only IO APIC */ io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; /* * sanity check, refer to MP spec section 3.6.6, last paragraph * necessary as some hardware isn't properly setting up the IO APIC */ #if defined(REALLY_ANAL_IOAPICID_VALUE) if (io_apic_id != 2) { #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ux &= ~APIC_ID_MASK; /* clear the ID field */ ux |= 0x02000000; /* set it to '2' */ io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ if ((ux & APIC_ID_MASK) != 0x02000000) panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; ID_TO_IO(io_apic_id) = 0; #endif /* APIC_IO */ /* fill out bus entries */ switch (type) { case 1: case 2: case 3: case 5: case 6: bus_data[0].bus_id = default_data[type - 1][1]; bus_data[0].bus_type = default_data[type - 1][2]; bus_data[1].bus_id = default_data[type - 1][3]; bus_data[1].bus_type = default_data[type - 1][4]; break; /* case 4: case 7: MCA NOT supported */ default: /* illegal/reserved */ panic("BAD default MP config: %d", type); /* NOTREACHED */ } #if defined(APIC_IO) /* general cases from MP v1.4, table 5-2 */ for (pin = 0; pin < 16; ++pin) { io_apic_ints[pin].int_type = 0; io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ io_apic_ints[pin].src_bus_id = 0; io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ io_apic_ints[pin].dst_apic_id = io_apic_id; io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ } /* special cases from MP v1.4, table 5-2 */ if (type == 2) { io_apic_ints[2].int_type = 0xff; /* N/C */ io_apic_ints[13].int_type = 0xff; /* N/C */ #if !defined(APIC_MIXED_MODE) /** FIXME: ??? */ panic("sorry, can't support type 2 default yet"); #endif /* APIC_MIXED_MODE */ } else io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ if (type == 7) io_apic_ints[0].int_type = 0xff; /* N/C */ else io_apic_ints[0].int_type = 3; /* vectored 8259 */ #endif /* APIC_IO */ } /* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { int x, i; u_char mpbiosreason; u_long mpbioswarmvec; pd_entry_t newptd; pt_entry_t newpt; int *newpp; POSTCODE(START_ALL_APS_POST); /** * NOTE: this needs further thought: * where does it get released? * should it be set to empy? * * get the initial mp_lock with a count of 1 for the BSP */ mp_lock = 1; /* this uses a LOGICAL cpu ID, ie BSP == 0 */ /* initialize BSP's local APIC */ apic_initialize(); /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* record BSP in CPU map */ all_cpus = 1; /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* HACK HACK HACK !!! */ /* alloc new page table directory */ newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* clone currently active one (ie: IdlePTD) */ bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ /* set up 0 -> 4MB P==V mapping for AP boot */ newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); /* store PTD for this AP */ bootPTD = (pd_entry_t)vtophys(newptd); /* alloc new page table page */ newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* set the new PTD's private page to point there */ newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt); /* install self referential entry */ newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd); /* get a new private data page */ newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ newpt[0] = PG_V | PG_RW | vtophys(newpp); /* wire the ptp into itself for access */ newpt[1] = PG_V | PG_RW | vtophys(newpt); /* and the local apic */ newpt[2] = SMP_prvpt[2]; /* and the IO apic mapping[s] */ for (i = 16; i < 32; i++) newpt[i] = SMP_prvpt[i]; /* prime data page for it to use */ newpp[0] = x; /* cpuid */ newpp[1] = 0; /* curproc */ newpp[2] = 0; /* curpcb */ newpp[3] = 0; /* npxproc */ newpp[4] = 0; /* runtime.tv_sec */ newpp[5] = 0; /* runtime.tv_usec */ newpp[6] = x << 24; /* cpu_lockid */ /* XXX NOTE: ABANDON bootPTD for now!!!! */ /* END REVOLTING HACKERY */ /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ /* record its version info */ cpu_apic_versions[x] = cpu_apic_versions[0]; all_cpus |= (1 << x); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ other_cpus = all_cpus & ~(1 << cpuid); /* fill in our (BSP) APIC version */ cpu_apic_versions[0] = lapic.version; /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); + + pmap_set_opt_bsp(); /* number of APs actually started */ return mp_ncpus - 1; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(u_int boot_addr) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_addr + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_addr + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; } /* * this function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It ain't pretty, * but it seems to work. */ static int start_ap(int logical_cpu, u_int boot_addr) { int physical_cpu; int vector; int cpus; u_long icr_lo, icr_hi; POSTCODE(START_AP_POST); /* get the PHYSICAL APIC ID# */ physical_cpu = CPU_TO_ID(logical_cpu); /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_ncpus; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* setup the address for the target AP */ icr_hi = lapic.icr_hi & ~APIC_ID_MASK; icr_hi |= (physical_cpu << 24); lapic.icr_hi = icr_hi; /* do an INIT IPI: assert RESET */ icr_lo = lapic.icr_lo & 0xfff00000; lapic.icr_lo = icr_lo | 0x0000c500; /* wait for pending status end */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* do an INIT IPI: deassert RESET */ lapic.icr_lo = icr_lo | 0x00008500; /* wait for pending status end */ u_sleep(10000); /* wait ~10mS */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* wait for it to start */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) if (mp_ncpus > cpus) return 1; /* return SUCCESS */ return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's * * XXX: Needs to handshake and wait for completion before proceding. */ void smp_invltlb(void) { #if defined(APIC_IO) if (smp_active && invltlb_ok) all_but_self_ipi(XINVLTLB_OFFSET); #endif /* APIC_IO */ } void invlpg(u_int addr) { __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); /* send a message to the other CPUs */ smp_invltlb(); } void invltlb(void) { u_long temp; /* * This should be implemented as load_cr3(rcr3()) when load_cr3() is * inlined. */ __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); /* send a message to the other CPUs */ smp_invltlb(); } #if defined(TEST_CPUSTOP) #if defined(DEBUG_CPUSTOP) void db_printf __P((const char *fmt, ...)); #endif /* DEBUG_CPUSTOP */ /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int stop_cpus( u_int map ) { if (!smp_active) return 0; /* send IPI to all CPUs in map */ #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d stopping CPUs: 0x%08x\n", cpuid, map); #endif /* DEBUG_CPUSTOP */ stopped_cpus = 0; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); #if defined(DEBUG_CPUSTOP) db_printf(" spin\n"); #endif /* DEBUG_CPUSTOP */ while (stopped_cpus != map) /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" spun\nstopped\n"); #endif /* DEBUG_CPUSTOP */ return 1; } /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus( u_int map ) { if (!smp_active) return 0; #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d restarting CPUs: 0x%08x (0x%08x)\n", cpuid, map, stopped_cpus); #endif /* DEBUG_CPUSTOP */ started_cpus = map; /* signal other cpus to restart */ while (started_cpus) /* wait for each to clear its bit */ /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" restarted\n"); #endif /* DEBUG_CPUSTOP */ return 1; } #endif /** TEST_CPUSTOP */ Index: head/sys/kern/subr_smp.c =================================================================== --- head/sys/kern/subr_smp.c (revision 27483) +++ head/sys/kern/subr_smp.c (revision 27484) @@ -1,1830 +1,1834 @@ /* * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.14 1997/07/13 00:42:14 smp Exp smp $ + * $Id: mp_machdep.c,v 1.27 1997/07/13 01:22:43 fsmp Exp $ */ #include "opt_smp.h" #include /* for KERNBASE */ #include #include #include #include #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include /* for KERNBASE */ #include #include #include #include #include #include #include #include /** TEST_DEFAULT_CONFIG, TEST_CPUSTOP _TEST1 */ #include #include #include /* cngetc() */ #if defined(APIC_IO) #include /* setidt() */ #include /* IPIs */ #include /* IPIs */ #endif /* APIC_IO */ #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define BIOS_BASE (0xf0000) #define BIOS_SIZE (0x10000) #define BIOS_COUNT (BIOS_SIZE/4) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) #define PROCENTRY_FLAG_EN 0x01 #define PROCENTRY_FLAG_BP 0x02 #define IOAPICENTRY_FLAG_EN 0x01 /* MP Floating Pointer Structure */ typedef struct MPFPS { char signature[4]; void *pap; u_char length; u_char spec_rev; u_char checksum; u_char mpfb1; u_char mpfb2; u_char mpfb3; u_char mpfb4; u_char mpfb5; } *mpfps_t; /* MP Configuration Table Header */ typedef struct MPCTH { char signature[4]; u_short base_table_length; u_char spec_rev; u_char checksum; u_char oem_id[8]; u_char product_id[12]; void *oem_table_pointer; u_short oem_table_size; u_short entry_count; void *apic_address; u_short extended_table_length; u_char extended_table_checksum; u_char reserved; } *mpcth_t; typedef struct PROCENTRY { u_char type; u_char apic_id; u_char apic_version; u_char cpu_flags; u_long cpu_signature; u_long feature_flags; u_long reserved1; u_long reserved2; } *proc_entry_ptr; typedef struct BUSENTRY { u_char type; u_char bus_id; char bus_type[6]; } *bus_entry_ptr; typedef struct IOAPICENTRY { u_char type; u_char apic_id; u_char apic_version; u_char apic_flags; void *apic_address; } *io_apic_entry_ptr; typedef struct INTENTRY { u_char type; u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } *int_entry_ptr; /* descriptions of MP basetable entries */ typedef struct BASETABLE_ENTRY { u_char type; u_char length; char name[16]; } basetable_entry; /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MP_START_POST 0x12 #define MP_ANNOUNCE_POST 0x13 #define MPTABLE_PASS1_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define MP_ENABLE_POST 0x16 #define START_ALL_APS_POST 0x17 #define INSTALL_AP_TRAMP_POST 0x18 #define START_AP_POST 0x19 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; /** XXX FIXME: what system files declare these??? */ extern struct region_descriptor r_gdt, r_idt; int mp_ncpus; /* # of CPUs, including BSP */ int mp_naps; /* # of Applications processors */ int mp_nbusses; /* # of busses */ int mp_napics; /* # of IO APICs */ int boot_cpu_id; /* designated BSP */ vm_offset_t cpu_apic_address; vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ u_int32_t cpu_apic_versions[NCPU]; u_int32_t io_apic_versions[NAPIC]; /* * APIC ID logical/physical mapping structures. * We oversize these to simplify boot-time config. */ int cpu_num_to_apic_id[NAPICID]; int io_num_to_apic_id[NAPICID]; int apic_id_to_logical[NAPICID]; /* Bitmap of all available CPUs */ u_int all_cpus; /* Boot of AP uses this PTD */ u_int *bootPTD; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t KPTphys; /* Virtual address of per-cpu common_tss */ extern struct i386tss common_tss; /* * Local data and functions. */ static int mp_capable; static u_int boot_address; static u_int base_memory; static int picmode; /* 0: virtual wire mode, 1: PIC mode */ static mpfps_t mpfps; static int search_for_sig(u_int32_t target, int count); static void mp_enable(u_int boot_addr); static int mptable_pass1(void); static int mptable_pass2(void); static void default_mp_table(int type); static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); base_memory = basemem * 1024; /* convert to bytes */ boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ if ((base_memory - boot_address) < bootMP_size) boot_address -= 4096; /* not enough, lower by 4k */ return boot_address; } /* * Look for an Intel MP spec table (ie, SMP capable hardware). */ int mp_probe(void) { int x; u_long segment; u_int32_t target; POSTCODE(MP_PROBE_POST); /* see if EBDA exists */ if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { /* search first 1K of EBDA */ target = (u_int32_t) (segment << 4); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } else { /* last 1K of base memory, effective 'top of base' passed in */ target = (u_int32_t) (base_memory - 0x400); if ((x = search_for_sig(target, 1024 / 4)) >= 0) goto found; } /* search the BIOS */ target = (u_int32_t) BIOS_BASE; if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) goto found; /* nothing found */ mpfps = (mpfps_t)0; mp_capable = 0; return 0; found: /* calculate needed resources */ mpfps = (mpfps_t)x; if (mptable_pass1()) panic("you must reconfigure your kernel"); /* flag fact that we are running multiple processors */ mp_capable = 1; return 1; } /* * Startup the SMP processors. */ void mp_start(void) { POSTCODE(MP_START_POST); /* look for MP capable motherboard */ if (mp_capable) mp_enable(boot_address); else panic("MP hardware not found!"); } /* * Print various information about the SMP system hardware and setup. */ void mp_announce(void) { int x; POSTCODE(MP_ANNOUNCE_POST); printf("FreeBSD/SMP: Multiprocessor motherboard\n"); printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0)); printf(", version: 0x%08x", cpu_apic_versions[0]); printf(", at 0x%08x\n", cpu_apic_address); for (x = 1; x <= mp_naps; ++x) { printf(" cpu%d (AP): apic id: %d", x, CPU_TO_ID(x)); printf(", version: 0x%08x", cpu_apic_versions[x]); printf(", at 0x%08x\n", cpu_apic_address); } #if defined(APIC_IO) for (x = 0; x < mp_napics; ++x) { printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x)); printf(", version: 0x%08x", io_apic_versions[x]); printf(", at 0x%08x\n", io_apic_address[x]); } #else printf(" Warning: APIC I/O disabled\n"); #endif /* APIC_IO */ } /* * AP cpu's call this to sync up protected mode. */ void init_secondary(void) { int gsel_tss, slot; r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; r_gdt.rd_base = (int) gdt; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); slot = NGDT + cpuid; gsel_tss = GSEL(slot, SEL_KPL); gdt[slot].sd.sd_type = SDT_SYS386TSS; common_tss.tss_esp0 = 0; /* not used until after switch */ common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); common_tss.tss_ioopt = (sizeof common_tss) << 16; ltr(gsel_tss); load_cr0(0x8005003b); /* XXX! */ PTD[0] = 0; + pmap_set_opt((unsigned *)PTD); + invltlb(); } #if defined(APIC_IO) /* * Final configuration of the BSP's local APIC: * - disable 'pic mode'. * - disable 'virtual wire mode'. * - enable NMI. */ void bsp_apic_configure(void) { u_char byte; u_int32_t temp; /* leave 'pic mode' if necessary */ if (picmode) { outb(0x22, 0x70); /* select IMCR */ byte = inb(0x23); /* current contents */ byte |= 0x01; /* mask external INTR */ outb(0x23, byte); /* disconnect 8259s/NMI */ } /* mask lint0 (the 8259 'virtual wire' connection) */ temp = lapic.lvt_lint0; temp |= APIC_LVT_M; /* set the mask */ lapic.lvt_lint0 = temp; /* setup lint1 to handle NMI */ temp = lapic.lvt_lint1; temp &= ~APIC_LVT_M; /* clear the mask */ lapic.lvt_lint1 = temp; if (bootverbose) apic_dump(); } #endif /* APIC_IO */ /******************************************************************* * local functions and data */ /* * start the SMP system */ static void mp_enable(u_int boot_addr) { int x; #if defined(APIC_IO) int apic; u_int ux; #endif /* APIC_IO */ POSTCODE(MP_ENABLE_POST); /* turn on 4MB of V == P addressing so we can get to MP table */ *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); invltlb(); /* examine the MP table for needed info, uses physical addresses */ x = mptable_pass2(); *(int *)PTD = 0; invltlb(); /* can't process default configs till the CPU APIC is pmapped */ if (x) default_mp_table(x); #if defined(APIC_IO) /* fill the LOGICAL io_apic_versions table */ for (apic = 0; apic < mp_napics; ++apic) { ux = io_apic_read(apic, IOAPIC_VER); io_apic_versions[apic] = ux; } /* program each IO APIC in the system */ for (apic = 0; apic < mp_napics; ++apic) if (io_apic_setup(apic) < 0) panic("IO APIC setup failure"); /* install a 'Spurious INTerrupt' vector */ setidt(XSPURIOUSINT_OFFSET, Xspuriousint, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* install an inter-CPU IPI for TLB invalidation */ setidt(XINVLTLB_OFFSET, Xinvltlb, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #if defined(TEST_CPUSTOP) /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_CPUSTOP */ #if defined(TEST_TEST1) /* install a 'Spurious INTerrupt' vector */ setidt(XTEST1_OFFSET, Xtest1, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif /** TEST_TEST1 */ #endif /* APIC_IO */ /* start each Application Processor */ start_all_aps(boot_addr); /* * The init process might be started on a different CPU now, * and the boot CPU might not call prepare_usermode to get * cr0 correctly configured. Thus we initialize cr0 here. */ load_cr0(rcr0() | CR0_WP | CR0_AM); } /* * look for the MP spec signature */ /* string defined by the Intel MP Spec as identifying the MP table */ #define MP_SIG 0x5f504d5f /* _MP_ */ #define NEXT(X) ((X) += 4) static int search_for_sig(u_int32_t target, int count) { int x; u_int32_t *addr = (u_int32_t *) (KERNBASE + target); for (x = 0; x < count; NEXT(x)) if (addr[x] == MP_SIG) /* make array index a byte index */ return (target + (x * sizeof(u_int32_t))); return -1; } static basetable_entry basetable_entry_types[] = { {0, 20, "Processor"}, {1, 8, "Bus"}, {2, 8, "I/O APIC"}, {3, 8, "I/O INT"}, {4, 8, "Local INT"} }; typedef struct BUSDATA { u_char bus_id; enum busTypes bus_type; } bus_datum; typedef struct INTDATA { u_char int_type; u_short int_flags; u_char src_bus_id; u_char src_bus_irq; u_char dst_apic_id; u_char dst_apic_int; } io_int, local_int; typedef struct BUSTYPENAME { u_char type; char name[7]; } bus_type_name; static bus_type_name bus_type_table[] = { {CBUS, "CBUS"}, {CBUSII, "CBUSII"}, {EISA, "EISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {ISA, "ISA"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {PCI, "PCI"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {UNKNOWN_BUSTYPE, "---"}, {XPRESS, "XPRESS"}, {UNKNOWN_BUSTYPE, "---"} }; /* from MP spec v1.4, table 5-1 */ static int default_data[7][5] = { /* nbus, id0, type0, id1, type1 */ {1, 0, ISA, 255, 255}, {1, 0, EISA, 255, 255}, {1, 0, EISA, 255, 255}, {0, 255, 255, 255, 255},/* MCA not supported */ {2, 0, ISA, 1, PCI}, {2, 0, EISA, 1, PCI}, {0, 255, 255, 255, 255} /* MCA not supported */ }; /* the bus data */ bus_datum bus_data[NBUS]; /* the IO INT data, one entry per possible APIC INTerrupt */ io_int io_apic_ints[NINTR]; static int nintrs; static void fix_mp_table __P((void)); static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); /* * 1st pass on motherboard's Intel MP specification table. * * initializes: * mp_ncpus = 1 * * determines: * cpu_apic_address (common to all CPUs) * io_apic_address[N] * mp_naps * mp_nbusses * mp_napics * nintrs */ static int mptable_pass1(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int mustpanic; POSTCODE(MPTABLE_PASS1_POST); mustpanic = 0; /* clear various tables */ for (x = 0; x < NAPICID; ++x) { io_apic_address[x] = ~0; /* IO APIC address table */ } /* init everything to empty */ mp_naps = 0; mp_nbusses = 0; mp_napics = 0; nintrs = 0; /* check for use of 'default' configuration */ if (mpfps->mpfb1 != 0) { /* use default addresses */ cpu_apic_address = DEFAULT_APIC_BASE; io_apic_address[0] = DEFAULT_IO_APIC_BASE; /* fill in with defaults */ mp_naps = 2; /* includes BSP */ mp_nbusses = default_data[mpfps->mpfb1 - 1][0]; #if defined(APIC_IO) mp_napics = 1; nintrs = 16; #endif /* APIC_IO */ } else { if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); cpu_apic_address = (vm_offset_t) cth->apic_address; /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; while (count--) { switch (type = *(u_char *) position) { case 0: /* processor_entry */ if (((proc_entry_ptr)position)->cpu_flags & PROCENTRY_FLAG_EN) ++mp_naps; break; case 1: /* bus_entry */ ++mp_nbusses; break; case 2: /* io_apic_entry */ if (((io_apic_entry_ptr)position)->apic_flags & IOAPICENTRY_FLAG_EN) io_apic_address[mp_napics++] = (vm_offset_t)((io_apic_entry_ptr) position)->apic_address; break; case 3: /* int_entry */ ++nintrs; break; case 4: /* int_entry */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char*)position += basetable_entry_types[type].length; } } /* qualify the numbers */ if (mp_naps > NCPU) printf("Warning: only using %d of %d available CPUs!\n", NCPU, mp_naps); #if 0 /** XXX we consider this legal now (but should we?) */ mustpanic = 1; #endif if (mp_nbusses > NBUS) { printf("found %d busses, increase NBUS\n", mp_nbusses); mustpanic = 1; } if (mp_napics > NAPIC) { printf("found %d apics, increase NAPIC\n", mp_napics); mustpanic = 1; } if (nintrs > NINTR) { printf("found %d intrs, increase NINTR\n", nintrs); mustpanic = 1; } /* * Count the BSP. * This is also used as a counter while starting the APs. */ mp_ncpus = 1; --mp_naps; /* subtract the BSP */ return mustpanic; } /* * 2nd pass on motherboard's Intel MP specification table. * * sets: * boot_cpu_id * ID_TO_IO(N), phy APIC ID to log CPU/IO table * CPU_TO_ID(N), logical CPU to APIC ID table * IO_TO_ID(N), logical IO to APIC ID table * bus_data[N] * io_apic_ints[N] */ static int mptable_pass2(void) { int x; mpcth_t cth; int totalSize; void* position; int count; int type; int apic, bus, cpu, intr; POSTCODE(MPTABLE_PASS2_POST); /* clear various tables */ for (x = 0; x < NAPICID; ++x) { ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ } /* clear bus data table */ for (x = 0; x < NBUS; ++x) bus_data[x].bus_id = 0xff; /* clear IO APIC INT table */ for (x = 0; x < NINTR; ++x) io_apic_ints[x].int_type = 0xff; /* setup the cpu/apic mapping arrays */ boot_cpu_id = -1; /* record whether PIC or virtual-wire mode */ picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; /* check for use of 'default' configuration */ #if defined(TEST_DEFAULT_CONFIG) return TEST_DEFAULT_CONFIG; #else if (mpfps->mpfb1 != 0) return mpfps->mpfb1; /* return default configuration type */ #endif /* TEST_DEFAULT_CONFIG */ if ((cth = mpfps->pap) == 0) panic("MP Configuration Table Header MISSING!"); /* walk the table, recording info of interest */ totalSize = cth->base_table_length - sizeof(struct MPCTH); position = (u_char *) cth + sizeof(struct MPCTH); count = cth->entry_count; apic = bus = intr = 0; cpu = 1; /* pre-count the BSP */ while (count--) { switch (type = *(u_char *) position) { case 0: if (processor_entry(position, cpu)) ++cpu; break; case 1: if (bus_entry(position, bus)) ++bus; break; case 2: if (io_apic_entry(position, apic)) ++apic; break; case 3: if (int_entry(position, intr)) ++intr; break; case 4: /* int_entry(position); */ break; default: panic("mpfps Base Table HOSED!"); /* NOTREACHED */ } totalSize -= basetable_entry_types[type].length; (u_char *) position += basetable_entry_types[type].length; } if (boot_cpu_id == -1) panic("NO BSP found!"); /* post scan cleanup */ fix_mp_table(); /* report fact that its NOT a default configuration */ return 0; } /* * parse an Intel MP specification table */ static void fix_mp_table(void) { int x; int id; int bus_0; int bus_pci; int num_pci_bus; /* * Fix mis-numbering of the PCI bus and its INT entries if the BIOS * did it wrong. The MP spec says that when more than 1 PCI bus * exists the BIOS must begin with bus entries for the PCI bus and use * actual PCI bus numbering. This implies that when only 1 PCI bus * exists the BIOS can choose to ignore this ordering, and indeed many * MP motherboards do ignore it. This causes a problem when the PCI * sub-system makes requests of the MP sub-system based on PCI bus * numbers. So here we look for the situation and renumber the * busses and associated INTs in an effort to "make it right". */ /* find bus 0, PCI bus, count the number of PCI busses */ for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_id == 0) { bus_0 = x; } if (bus_data[x].bus_type == PCI) { ++num_pci_bus; bus_pci = x; } } /* * bus_0 == slot of bus with ID of 0 * bus_pci == slot of last PCI bus encountered */ /* check the 1 PCI bus case for sanity */ if (num_pci_bus == 1) { /* if it is number 0 all is well */ if (bus_data[bus_pci].bus_id == 0) return; /* mis-numbered, swap with whichever bus uses slot 0 */ /* swap the bus entry types */ bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; bus_data[bus_0].bus_type = PCI; /* swap each relavant INTerrupt entry */ id = bus_data[bus_pci].bus_id; for (x = 0; x < nintrs; ++x) { if (io_apic_ints[x].src_bus_id == id) { io_apic_ints[x].src_bus_id = 0; } else if (io_apic_ints[x].src_bus_id == 0) { io_apic_ints[x].src_bus_id = id; } } } /* sanity check if more than 1 PCI bus */ else if (num_pci_bus > 1) { for (x = 0; x < mp_nbusses; ++x) { if (bus_data[x].bus_type != PCI) continue; if (bus_data[x].bus_id >= num_pci_bus) panic("bad PCI bus numbering"); } } } static int processor_entry(proc_entry_ptr entry, int cpu) { /* check for usability */ if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) return 0; /* check for BSP flag */ if (entry->cpu_flags & PROCENTRY_FLAG_BP) { boot_cpu_id = entry->apic_id; CPU_TO_ID(0) = entry->apic_id; ID_TO_CPU(entry->apic_id) = 0; return 0; /* its already been counted */ } /* add another AP to list, if less than max number of CPUs */ else { CPU_TO_ID(cpu) = entry->apic_id; ID_TO_CPU(entry->apic_id) = cpu; return 1; } } static int bus_entry(bus_entry_ptr entry, int bus) { int x; char c, name[8]; /* encode the name into an index */ for (x = 0; x < 6; ++x) { if ((c = entry->bus_type[x]) == ' ') break; name[x] = c; } name[x] = '\0'; if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) panic("unknown bus type: '%s'", name); bus_data[bus].bus_id = entry->bus_id; bus_data[bus].bus_type = x; return 1; } static int io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; IO_TO_ID(apic) = entry->apic_id; ID_TO_IO(entry->apic_id) = apic; return 1; } static int lookup_bus_type(char *name) { int x; for (x = 0; x < MAX_BUSTYPE; ++x) if (strcmp(bus_type_table[x].name, name) == 0) return bus_type_table[x].type; return UNKNOWN_BUSTYPE; } static int int_entry(int_entry_ptr entry, int intr) { io_apic_ints[intr].int_type = entry->int_type; io_apic_ints[intr].int_flags = entry->int_flags; io_apic_ints[intr].src_bus_id = entry->src_bus_id; io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; } static int apic_int_is_bus_type(int intr, int bus_type) { int bus; for (bus = 0; bus < mp_nbusses; ++bus) if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) && ((int) bus_data[bus].bus_type == bus_type)) return 1; return 0; } /* * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { int isa_irq; int apic_pin; #if defined(SKIP_IRQ15_REDIRECT) if (isa_mask == (1 << 15)) { printf("skipping ISA IRQ15 redirect\n"); return isa_mask; } #endif /* SKIP_IRQ15_REDIRECT */ isa_irq = ffs(isa_mask); /* find its bit position */ if (isa_irq == 0) /* doesn't exist */ return 0; --isa_irq; /* make it zero based */ apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ if (apic_pin == -1) return 0; return (1 << apic_pin); /* convert pin# to a mask */ } /* * Determine which APIC pin an ISA/EISA INT is attached to. */ #define INTTYPE(I) (io_apic_ints[(I)].int_type) #define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) #define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) int isa_apic_pin(int isa_irq) { int intr; #if defined(SMP_TIMER_NC) if (isa_irq == 0) return -1; #endif /* SMP_TIMER_NC */ for (intr = 0; intr < nintrs; ++intr) { /* check each record */ if (INTTYPE(intr) == 0) { /* standard INT */ if (SRCBUSIRQ(intr) == isa_irq) { if (apic_int_is_bus_type(intr, ISA) || apic_int_is_bus_type(intr, EISA)) return INTPIN(intr); /* found */ } } } return -1; /* NOT found */ } #undef SRCBUSIRQ /* * Determine which APIC pin a PCI INT is attached to. */ #define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) #define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) #define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) int pci_apic_pin(int pciBus, int pciDevice, int pciInt) { int intr; --pciInt; /* zero based */ for (intr = 0; intr < nintrs; ++intr) /* check each record */ if ((INTTYPE(intr) == 0) /* standard INT */ && (SRCBUSID(intr) == pciBus) && (SRCBUSDEVICE(intr) == pciDevice) && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ if (apic_int_is_bus_type(intr, PCI)) return INTPIN(intr); /* exact match */ return -1; /* NOT found */ } #undef SRCBUSLINE #undef SRCBUSDEVICE #undef SRCBUSID #undef INTPIN #undef INTTYPE /* * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. * * XXX FIXME: * Exactly what this means is unclear at this point. It is a solution * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard * could route any of the ISA INTs to upper (>15) IRQ values. But most would * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an * option. */ int undirect_isa_irq(int rirq) { #if defined(READY) printf("Freeing redirected ISA irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); return 0; #endif /* READY */ } /* * Reprogram the MB chipset to NOT redirect a PCI INTerrupt */ int undirect_pci_irq(int rirq) { #if defined(READY) if (bootverbose) printf("Freeing redirected PCI irq %d.\n", rirq); /** FIXME: tickle the MB redirector chip */ return ???; #else if (bootverbose) printf("Freeing (NOT implemented) redirected PCI irq %d.\n", rirq); return 0; #endif /* READY */ } /* * given a bus ID, return: * the bus type if found * -1 if NOT found */ int apic_bus_type(int id) { int x; for (x = 0; x < mp_nbusses; ++x) if (bus_data[x].bus_id == id) return bus_data[x].bus_type; return -1; } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus ID if found * -1 if NOT found */ int apic_src_bus_id(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_id); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated src bus IRQ if found * -1 if NOT found */ int apic_src_bus_irq(int apic, int pin) { int x; for (x = 0; x < nintrs; x++) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].src_bus_irq); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated INTerrupt type if found * -1 if NOT found */ int apic_int_type(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_type); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated trigger mode if found * -1 if NOT found */ int apic_trigger(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return ((io_apic_ints[x].int_flags >> 2) & 0x03); return -1; /* NOT found */ } /* * given a LOGICAL APIC# and pin#, return: * the associated 'active' level if found * -1 if NOT found */ int apic_polarity(int apic, int pin) { int x; /* search each of the possible INTerrupt sources */ for (x = 0; x < nintrs; ++x) if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && (pin == io_apic_ints[x].dst_apic_int)) return (io_apic_ints[x].int_flags & 0x03); return -1; /* NOT found */ } /* * set data according to MP defaults * FIXME: probably not complete yet... */ static void default_mp_table(int type) { int ap_cpu_id; #if defined(APIC_IO) u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ #if 0 printf(" MP default config type: %d\n", type); switch (type) { case 1: printf(" bus: ISA, APIC: 82489DX\n"); break; case 2: printf(" bus: EISA, APIC: 82489DX\n"); break; case 3: printf(" bus: EISA, APIC: 82489DX\n"); break; case 4: printf(" bus: MCA, APIC: 82489DX\n"); break; case 5: printf(" bus: ISA+PCI, APIC: Integrated\n"); break; case 6: printf(" bus: EISA+PCI, APIC: Integrated\n"); break; case 7: printf(" bus: MCA+PCI, APIC: Integrated\n"); break; default: printf(" future type\n"); break; /* NOTREACHED */ } #endif /* 0 */ boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; /* BSP */ CPU_TO_ID(0) = boot_cpu_id; ID_TO_CPU(boot_cpu_id) = 0; /* one and only AP */ CPU_TO_ID(1) = ap_cpu_id; ID_TO_CPU(ap_cpu_id) = 1; #if defined(APIC_IO) /* one and only IO APIC */ io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; /* * sanity check, refer to MP spec section 3.6.6, last paragraph * necessary as some hardware isn't properly setting up the IO APIC */ #if defined(REALLY_ANAL_IOAPICID_VALUE) if (io_apic_id != 2) { #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ux &= ~APIC_ID_MASK; /* clear the ID field */ ux |= 0x02000000; /* set it to '2' */ io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ if ((ux & APIC_ID_MASK) != 0x02000000) panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; ID_TO_IO(io_apic_id) = 0; #endif /* APIC_IO */ /* fill out bus entries */ switch (type) { case 1: case 2: case 3: case 5: case 6: bus_data[0].bus_id = default_data[type - 1][1]; bus_data[0].bus_type = default_data[type - 1][2]; bus_data[1].bus_id = default_data[type - 1][3]; bus_data[1].bus_type = default_data[type - 1][4]; break; /* case 4: case 7: MCA NOT supported */ default: /* illegal/reserved */ panic("BAD default MP config: %d", type); /* NOTREACHED */ } #if defined(APIC_IO) /* general cases from MP v1.4, table 5-2 */ for (pin = 0; pin < 16; ++pin) { io_apic_ints[pin].int_type = 0; io_apic_ints[pin].int_flags = 0x05; /* edge-triggered/active-hi */ io_apic_ints[pin].src_bus_id = 0; io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 is caught below */ io_apic_ints[pin].dst_apic_id = io_apic_id; io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 correspondence */ } /* special cases from MP v1.4, table 5-2 */ if (type == 2) { io_apic_ints[2].int_type = 0xff; /* N/C */ io_apic_ints[13].int_type = 0xff; /* N/C */ #if !defined(APIC_MIXED_MODE) /** FIXME: ??? */ panic("sorry, can't support type 2 default yet"); #endif /* APIC_MIXED_MODE */ } else io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ if (type == 7) io_apic_ints[0].int_type = 0xff; /* N/C */ else io_apic_ints[0].int_type = 3; /* vectored 8259 */ #endif /* APIC_IO */ } /* * start each AP in our list */ static int start_all_aps(u_int boot_addr) { int x, i; u_char mpbiosreason; u_long mpbioswarmvec; pd_entry_t newptd; pt_entry_t newpt; int *newpp; POSTCODE(START_ALL_APS_POST); /** * NOTE: this needs further thought: * where does it get released? * should it be set to empy? * * get the initial mp_lock with a count of 1 for the BSP */ mp_lock = 1; /* this uses a LOGICAL cpu ID, ie BSP == 0 */ /* initialize BSP's local APIC */ apic_initialize(); /* install the AP 1st level boot code */ install_ap_tramp(boot_addr); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* record BSP in CPU map */ all_cpus = 1; /* start each AP */ for (x = 1; x <= mp_naps; ++x) { /* HACK HACK HACK !!! */ /* alloc new page table directory */ newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* clone currently active one (ie: IdlePTD) */ bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ /* set up 0 -> 4MB P==V mapping for AP boot */ newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); /* store PTD for this AP */ bootPTD = (pd_entry_t)vtophys(newptd); /* alloc new page table page */ newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE)); /* set the new PTD's private page to point there */ newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt); /* install self referential entry */ newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd); /* get a new private data page */ newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ newpt[0] = PG_V | PG_RW | vtophys(newpp); /* wire the ptp into itself for access */ newpt[1] = PG_V | PG_RW | vtophys(newpt); /* and the local apic */ newpt[2] = SMP_prvpt[2]; /* and the IO apic mapping[s] */ for (i = 16; i < 32; i++) newpt[i] = SMP_prvpt[i]; /* prime data page for it to use */ newpp[0] = x; /* cpuid */ newpp[1] = 0; /* curproc */ newpp[2] = 0; /* curpcb */ newpp[3] = 0; /* npxproc */ newpp[4] = 0; /* runtime.tv_sec */ newpp[5] = 0; /* runtime.tv_usec */ newpp[6] = x << 24; /* cpu_lockid */ /* XXX NOTE: ABANDON bootPTD for now!!!! */ /* END REVOLTING HACKERY */ /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(x, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ /* record its version info */ cpu_apic_versions[x] = cpu_apic_versions[0]; all_cpus |= (1 << x); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ other_cpus = all_cpus & ~(1 << cpuid); /* fill in our (BSP) APIC version */ cpu_apic_versions[0] = lapic.version; /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); + + pmap_set_opt_bsp(); /* number of APs actually started */ return mp_ncpus - 1; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(u_int boot_addr) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_addr + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_addr + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_addr & 0xffff; *dst8 = ((u_int) boot_addr >> 16) & 0xff; } /* * this function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It ain't pretty, * but it seems to work. */ static int start_ap(int logical_cpu, u_int boot_addr) { int physical_cpu; int vector; int cpus; u_long icr_lo, icr_hi; POSTCODE(START_AP_POST); /* get the PHYSICAL APIC ID# */ physical_cpu = CPU_TO_ID(logical_cpu); /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_ncpus; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* setup the address for the target AP */ icr_hi = lapic.icr_hi & ~APIC_ID_MASK; icr_hi |= (physical_cpu << 24); lapic.icr_hi = icr_hi; /* do an INIT IPI: assert RESET */ icr_lo = lapic.icr_lo & 0xfff00000; lapic.icr_lo = icr_lo | 0x0000c500; /* wait for pending status end */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* do an INIT IPI: deassert RESET */ lapic.icr_lo = icr_lo | 0x00008500; /* wait for pending status end */ u_sleep(10000); /* wait ~10mS */ while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic.icr_lo = icr_lo | 0x00000600 | vector; while (lapic.icr_lo & APIC_DELSTAT_MASK) /* spin */ ; u_sleep(200); /* wait ~200uS */ /* wait for it to start */ set_apic_timer(5000000);/* == 5 seconds */ while (read_apic_timer()) if (mp_ncpus > cpus) return 1; /* return SUCCESS */ return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's * * XXX: Needs to handshake and wait for completion before proceding. */ void smp_invltlb(void) { #if defined(APIC_IO) if (smp_active && invltlb_ok) all_but_self_ipi(XINVLTLB_OFFSET); #endif /* APIC_IO */ } void invlpg(u_int addr) { __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); /* send a message to the other CPUs */ smp_invltlb(); } void invltlb(void) { u_long temp; /* * This should be implemented as load_cr3(rcr3()) when load_cr3() is * inlined. */ __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); /* send a message to the other CPUs */ smp_invltlb(); } #if defined(TEST_CPUSTOP) #if defined(DEBUG_CPUSTOP) void db_printf __P((const char *fmt, ...)); #endif /* DEBUG_CPUSTOP */ /* * When called the executing CPU will send an IPI to all other CPUs * requesting that they halt execution. * * Usually (but not necessarily) called with 'other_cpus' as its arg. * * - Signals all CPUs in map to stop. * - Waits for each to stop. * * Returns: * -1: error * 0: NA * 1: ok * * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs * from executing at same time. */ int stop_cpus( u_int map ) { if (!smp_active) return 0; /* send IPI to all CPUs in map */ #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d stopping CPUs: 0x%08x\n", cpuid, map); #endif /* DEBUG_CPUSTOP */ stopped_cpus = 0; /* send the Xcpustop IPI to all CPUs in map */ selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); #if defined(DEBUG_CPUSTOP) db_printf(" spin\n"); #endif /* DEBUG_CPUSTOP */ while (stopped_cpus != map) /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" spun\nstopped\n"); #endif /* DEBUG_CPUSTOP */ return 1; } /* * Called by a CPU to restart stopped CPUs. * * Usually (but not necessarily) called with 'stopped_cpus' as its arg. * * - Signals all CPUs in map to restart. * - Waits for each to restart. * * Returns: * -1: error * 0: NA * 1: ok */ int restart_cpus( u_int map ) { if (!smp_active) return 0; #if defined(DEBUG_CPUSTOP) db_printf("\nCPU%d restarting CPUs: 0x%08x (0x%08x)\n", cpuid, map, stopped_cpus); #endif /* DEBUG_CPUSTOP */ started_cpus = map; /* signal other cpus to restart */ while (started_cpus) /* wait for each to clear its bit */ /* spin */ ; #if defined(DEBUG_CPUSTOP) db_printf(" restarted\n"); #endif /* DEBUG_CPUSTOP */ return 1; } #endif /** TEST_CPUSTOP */