diff --git a/sys/ia64/ia64/clock.c b/sys/ia64/ia64/clock.c index a9c39fce3598..0afbc8788f85 100644 --- a/sys/ia64/ia64/clock.c +++ b/sys/ia64/ia64/clock.c @@ -1,109 +1,191 @@ /*- * Copyright (c) 2005 Marcel Moolenaar * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include +#include +#include +#include #include #include #include -#include #include #include -#include #include #include +#include +#include #include -uint64_t ia64_clock_reload; +SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics"); + +static int adjust_edges = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD, + &adjust_edges, 0, "Number of times ITC got more than 12.5% behind"); + +static int adjust_excess = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD, + &adjust_excess, 0, "Total number of ignored ITC interrupts"); + +static int adjust_lost = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD, + &adjust_lost, 0, "Total number of lost ITC interrupts"); + +static int adjust_ticks = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD, + &adjust_ticks, 0, "Total number of ITC interrupts with adjustment"); + +static u_int ia64_clock_xiv; +static uint64_t ia64_clock_reload; #ifndef SMP static timecounter_get_t ia64_get_timecount; static struct timecounter ia64_timecounter = { ia64_get_timecount, /* get_timecount */ 0, /* no poll_pps */ ~0u, /* counter_mask */ 0, /* frequency */ "ITC" /* name */ }; -static unsigned +static u_int ia64_get_timecount(struct timecounter* tc) { return ia64_get_itc(); } #endif +static u_int +ia64_ih_clock(struct thread *td, u_int xiv, struct trapframe *tf) +{ + uint64_t adj, clk, itc; + int64_t delta; + int count; + + ia64_set_eoi(0); + + PCPU_INC(md.stats.pcs_nclks); + intrcnt[INTRCNT_CLOCK]++; + + ia64_srlz_d(); + + itc = ia64_get_itc(); + + adj = PCPU_GET(md.clockadj); + clk = PCPU_GET(md.clock); + + delta = itc - clk; + count = 0; + while (delta >= ia64_clock_reload) { + /* Only the BSP runs the real clock */ + if (PCPU_GET(cpuid) == 0) + hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); + else + hardclock_cpu(TRAPF_USERMODE(tf)); + if (profprocs != 0) + profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); + statclock(TRAPF_USERMODE(tf)); + delta -= ia64_clock_reload; + clk += ia64_clock_reload; + if (adj != 0) + adjust_ticks++; + count++; + } + ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj); + if (count > 0) { + adjust_lost += count - 1; + if (delta > (ia64_clock_reload >> 3)) { + if (adj == 0) + adjust_edges++; + adj = ia64_clock_reload >> 4; + } else + adj = 0; + } else { + adj = 0; + adjust_excess++; + } + PCPU_SET(md.clock, clk); + PCPU_SET(md.clockadj, adj); + ia64_srlz_d(); + return (0); +} + void pcpu_initclock(void) { PCPU_SET(md.clockadj, 0); PCPU_SET(md.clock, ia64_get_itc()); ia64_set_itm(PCPU_GET(md.clock) + ia64_clock_reload); - ia64_set_itv(CLOCK_VECTOR); /* highest priority class */ + ia64_set_itv(ia64_clock_xiv); ia64_srlz_d(); } /* * Start the real-time and statistics clocks. We use cr.itc and cr.itm * to implement a 1000hz clock. */ void cpu_initclocks() { u_long itc_freq; + ia64_clock_xiv = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IRQ, + ia64_ih_clock); + if (ia64_clock_xiv == 0) + panic("No XIV for clock interrupts"); + itc_freq = (u_long)ia64_itc_freq() * 1000000ul; stathz = hz; ia64_clock_reload = (itc_freq + hz/2) / hz; #ifndef SMP ia64_timecounter.tc_frequency = itc_freq; tc_init(&ia64_timecounter); #endif pcpu_initclock(); } void cpu_startprofclock(void) { /* nothing to do */ } void cpu_stopprofclock(void) { /* nothing to do */ } diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S index 2868c8a3f771..5e186f9be929 100644 --- a/sys/ia64/ia64/exception.S +++ b/sys/ia64/ia64/exception.S @@ -1,1549 +1,1549 @@ /*- * Copyright (c) 2003,2004 Marcel Moolenaar * Copyright (c) 2000 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_xtrace.h" #include #include /* * Nested TLB restart tokens. These are used by the * nested TLB handler for jumping back to the code * where the nested TLB was caused. */ #define NTLBRT_SAVE 0x12c12c #define NTLBRT_RESTORE 0x12c12d /* * ar.k7 = kernel memory stack * ar.k6 = kernel register stack * ar.k5 = EPC gateway page * ar.k4 = PCPU data */ #ifdef EXCEPTION_TRACING .data .global xtrace, xhead xtrace: .space 1024*5*8 xhead: data8 xtrace #define XTRACE(offset) \ { .mmi ; \ mov r24=ar.itc ; \ mov r25=cr.iip ; \ mov r27=offset ; \ } ; \ { .mlx ; \ mov r28=cr.ifa ; \ movl r29=xhead ;; \ } ; \ { .mmi ; \ ld8 r29=[r29] ;; \ st8 [r29]=r24,8 ; \ nop 0 ;; \ } ; \ { .mmi ; \ st8 [r29]=r27,8 ;; \ mov r24=cr.isr ; \ add r27=8,r29 ;; \ } ; \ { .mmi ; \ st8 [r29]=r25,16 ;; \ st8 [r27]=r28,16 ; \ mov r25=pr ;; \ } ; \ { .mlx ; \ st8 [r29]=r24 ; \ movl r28=xhead ;; \ } ; \ { .mii ; \ cmp.eq p15,p0=r27,r28 ; \ addl r29=1024*5*8,r0 ;; \ (p15) sub r27=r28,r29 ;; \ } ; \ { .mmi ; \ st8 [r28]=r27 ; \ nop 0 ; \ mov pr=r25,0x1ffff ;; \ } #else #define XTRACE(offset) #endif .text /* * exception_save: save interrupted state * * Arguments: * r16 address of bundle that contains the branch. The * return address will be the next bundle. * r17 the value to save as ifa in the trapframe. This * normally is cr.ifa, but some interruptions set * set cr.iim and not cr.ifa. * * Returns: * p15 interrupted from user stack * p14 interrupted from kernel stack * p13 interrupted from user backing store * p12 interrupted from kernel backing store * p11 interrupts were enabled * p10 interrupts were disabled */ ENTRY_NOPROFILE(exception_save, 0) { .mii mov r20=ar.unat extr.u r31=sp,61,3 mov r18=pr ;; } { .mmi cmp.le p14,p15=5,r31 ;; (p15) mov r23=ar.k7 // kernel memory stack (p14) mov r23=sp ;; } { .mii mov r21=ar.rsc add r30=-SIZEOF_TRAPFRAME,r23 ;; dep r30=0,r30,0,10 ;; } { .mmi mov ar.rsc=0 sub r19=r23,r30 add r31=8,r30 ;; } { .mmi mov r22=cr.iip nop 0 addl r29=NTLBRT_SAVE,r0 // 22-bit restart token. ;; } /* * We have a 1KB aligned trapframe, pointed to by sp. If we write * to the trapframe, we may trigger a data nested TLB fault. By * aligning the trapframe on a 1KB boundary, we guarantee that if * we get a data nested TLB fault, it will be on the very first * write. Since the data nested TLB fault does not preserve any * state, we have to be careful what we clobber. Consequently, we * have to be careful what we use here. Below a list of registers * that are currently alive: * r16,r17=arguments * r18=pr, r19=length, r20=unat, r21=rsc, r22=iip, r23=TOS * r29=restart point * r30,r31=trapframe pointers * p14,p15=memory stack switch */ exception_save_restart: { .mmi st8 [r30]=r19,16 // length st8 [r31]=r0,16 // flags add r19=16,r19 ;; } { .mmi st8.spill [r30]=sp,16 // sp st8 [r31]=r20,16 // unat sub sp=r23,r19 ;; } { .mmi mov r19=ar.rnat mov r20=ar.bspstore mov r23=rp ;; } // r18=pr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=rp { .mmi st8 [r30]=r23,16 // rp st8 [r31]=r18,16 // pr mov r24=ar.pfs ;; } { .mmb st8 [r30]=r24,16 // pfs st8 [r31]=r20,16 // bspstore cover ;; } { .mmi mov r18=ar.fpsr mov r23=cr.ipsr extr.u r24=r20,61,3 ;; } // r18=fpsr, r19=rnat, r20=bspstore, r21=rsc, r22=iip, r23=ipsr { .mmi st8 [r30]=r19,16 // rnat st8 [r31]=r0,16 // __spare cmp.le p12,p13=5,r24 ;; } { .mmi st8.spill [r30]=r13,16 // tp st8 [r31]=r21,16 // rsc tbit.nz p11,p10=r23,14 // p11=interrupts enabled ;; } { .mmi (p13) mov r21=ar.k6 // kernel register stack ;; st8 [r30]=r18,16 // fpsr (p13) dep r20=r20,r21,0,9 // align dirty registers ;; } // r19=rnat, r20=bspstore, r22=iip, r23=ipsr { .mmi st8 [r31]=r23,16 // psr (p13) mov ar.bspstore=r20 nop 0 ;; } { .mmi (p13) mov ar.rnat=r19 mov r18=ar.bsp nop 0 ;; } { .mmi mov r19=cr.ifs st8.spill [r30]=gp,16 // gp sub r18=r18,r20 ;; } // r19=ifs, r22=iip { .mmi st8 [r31]=r18,16 // ndirty st8 [r30]=r19,16 // cfm nop 0 ;; } { .mmi mov r18=cr.isr st8 [r31]=r22,16 // iip add r29=16,r30 ;; } { .mmi st8 [r30]=r17,24 // ifa st8 [r31]=r18,24 // isr nop 0 ;; } { .mmi .mem.offset 0,0 st8.spill [r30]=r2,16 // r2 .mem.offset 8,0 st8.spill [r31]=r3,16 // r3 add r2=9*8,r29 ;; } { .mmi .mem.offset 0,0 st8.spill [r30]=r8,16 // r8 .mem.offset 8,0 st8.spill [r31]=r9,16 // r9 add r3=8,r2 ;; } { .mmi .mem.offset 0,0 st8.spill [r30]=r10,16 // r10 .mem.offset 8,0 st8.spill [r31]=r11,16 // r11 add r8=16,r16 ;; } { .mmi .mem.offset 0,0 st8.spill [r30]=r14 // r14 .mem.offset 8,0 st8.spill [r31]=r15 // r15 mov r9=r29 } { .mmb mov r10=ar.csd mov r11=ar.ssd bsw.1 ;; } { .mmi .mem.offset 0,0 st8.spill [r2]=r16,16 // r16 .mem.offset 8,0 st8.spill [r3]=r17,16 // r17 mov r14=b6 ;; } { .mmi .mem.offset 0,0 st8.spill [r2]=r18,16 // r18 .mem.offset 8,0 st8.spill [r3]=r19,16 // r19 mov r15=b7 ;; } { .mmi .mem.offset 0,0 st8.spill [r2]=r20,16 // r20 .mem.offset 8,0 st8.spill [r3]=r21,16 // r21 mov b7=r8 ;; } { .mmi .mem.offset 0,0 st8.spill [r2]=r22,16 // r22 .mem.offset 8,0 st8.spill [r3]=r23,16 // r23 ;; } .mem.offset 0,0 st8.spill [r2]=r24,16 // r24 .mem.offset 8,0 st8.spill [r3]=r25,16 // r25 ;; .mem.offset 0,0 st8.spill [r2]=r26,16 // r26 .mem.offset 8,0 st8.spill [r3]=r27,16 // r27 ;; .mem.offset 0,0 st8.spill [r2]=r28,16 // r28 .mem.offset 8,0 st8.spill [r3]=r29,16 // r29 ;; .mem.offset 0,0 st8.spill [r2]=r30,16 // r30 .mem.offset 8,0 st8.spill [r3]=r31,16 // r31 ;; { .mmi st8 [r2]=r14,16 // b6 mov r17=ar.unat nop 0 ;; } { .mmi st8 [r3]=r15,16 // b7 mov r16=ar.ccv nop 0 ;; } { .mmi st8 [r2]=r16,16 // ccv st8 [r3]=r10,16 // csd nop 0 ;; } { .mmi st8 [r2]=r11,24 // ssd st8 [r9]=r17 nop 0 ;; } stf.spill [r3]=f6,32 // f6 stf.spill [r2]=f7,32 // f7 ;; stf.spill [r3]=f8,32 // f8 stf.spill [r2]=f9,32 // f9 ;; stf.spill [r3]=f10,32 // f10 stf.spill [r2]=f11,32 // f11 ;; stf.spill [r3]=f12,32 // f12 stf.spill [r2]=f13,32 // f13 ;; stf.spill [r3]=f14 // f14 stf.spill [r2]=f15 // f15 ;; { .mmi mov ar.rsc=3 mov r13=ar.k4 nop 0 ;; } { .mlx ssm psr.ic|psr.dfh movl gp=__gp ;; } { .mib srlz.d nop 0 br.sptk b7 ;; } END(exception_save) /* * exception_restore: restore interrupted state * * Arguments: * sp+16 trapframe pointer */ ENTRY_NOPROFILE(exception_restore, 0) { .mmi rsm psr.i add r3=SIZEOF_TRAPFRAME-16,sp add r2=SIZEOF_TRAPFRAME,sp ;; } { .mmi srlz.d add r8=SIZEOF_SPECIAL+32,sp nop 0 ;; } // The next load can trap. Let it be... ldf.fill f15=[r2],-32 // f15 ldf.fill f14=[r3],-32 // f14 add sp=16,sp ;; ldf.fill f13=[r2],-32 // f13 ldf.fill f12=[r3],-32 // f12 ;; ldf.fill f11=[r2],-32 // f11 ldf.fill f10=[r3],-32 // f10 ;; ldf.fill f9=[r2],-32 // f9 ldf.fill f8=[r3],-32 // f8 ;; ldf.fill f7=[r2],-24 // f7 ldf.fill f6=[r3],-16 // f6 ;; { .mmi ld8 r8=[r8] // unat (after) ;; mov ar.unat=r8 nop 0 ;; } ld8 r10=[r2],-16 // ssd ld8 r11=[r3],-16 // csd ;; mov ar.ssd=r10 mov ar.csd=r11 ld8 r14=[r2],-16 // ccv ld8 r15=[r3],-16 // b7 ;; { .mmi mov ar.ccv=r14 ld8 r8=[r2],-16 // b6 mov b7=r15 ;; } { .mmi ld8.fill r31=[r3],-16 // r31 ld8.fill r30=[r2],-16 // r30 mov b6=r8 ;; } ld8.fill r29=[r3],-16 // r29 ld8.fill r28=[r2],-16 // r28 ;; ld8.fill r27=[r3],-16 // r27 ld8.fill r26=[r2],-16 // r26 ;; ld8.fill r25=[r3],-16 // r25 ld8.fill r24=[r2],-16 // r24 ;; ld8.fill r23=[r3],-16 // r23 ld8.fill r22=[r2],-16 // r22 ;; ld8.fill r21=[r3],-16 // r21 ld8.fill r20=[r2],-16 // r20 ;; ld8.fill r19=[r3],-16 // r19 ld8.fill r18=[r2],-16 // r18 ;; { .mmb ld8.fill r17=[r3],-16 // r17 ld8.fill r16=[r2],-16 // r16 bsw.0 ;; } { .mmi ld8.fill r15=[r3],-16 // r15 ld8.fill r14=[r2],-16 // r14 add r31=16,sp ;; } { .mmi ld8 r16=[sp] // tf_length ld8.fill r11=[r3],-16 // r11 add r30=24,sp ;; } { .mmi ld8.fill r10=[r2],-16 // r10 ld8.fill r9=[r3],-16 // r9 add r16=r16,sp // ar.k7 ;; } { .mmi ld8.fill r8=[r2],-16 // r8 ld8.fill r3=[r3] // r3 ;; } // We want nested TLB faults from here on... rsm psr.ic|psr.i ld8.fill r2=[r2] // r2 nop 0 ;; srlz.d ld8.fill sp=[r31],16 // sp nop 0 ;; ld8 r17=[r30],16 // unat ld8 r29=[r31],16 // rp ;; ld8 r18=[r30],16 // pr ld8 r28=[r31],16 // pfs mov rp=r29 ;; ld8 r20=[r30],24 // bspstore ld8 r21=[r31],24 // rnat mov ar.pfs=r28 ;; ld8.fill r26=[r30],16 // tp ld8 r22=[r31],16 // rsc ;; { .mmi ld8 r23=[r30],16 // fpsr ld8 r24=[r31],16 // psr extr.u r28=r20,61,3 ;; } { .mmi ld8.fill r1=[r30],16 // gp ld8 r27=[r31],16 // ndirty cmp.le p14,p15=5,r28 ;; } { .mmi ld8 r25=[r30] // cfm ld8 r19=[r31] // ip nop 0 ;; } { .mii // Switch register stack alloc r30=ar.pfs,0,0,0,0 // discard current frame shl r31=r27,16 // value for ar.rsc (p15) mov r13=r26 ;; } // The loadrs can fault if the backing store is not currently // mapped. We assured forward progress by getting everything we // need from the trapframe so that we don't care if the CPU // purges that translation when it needs to insert a new one for // the backing store. { .mmi mov ar.rsc=r31 // setup for loadrs mov ar.k7=r16 addl r29=NTLBRT_RESTORE,r0 // 22-bit restart token ;; } exception_restore_restart: { .mmi mov r30=ar.bspstore ;; loadrs // load user regs nop 0 ;; } { .mmi mov r31=ar.bspstore ;; mov ar.bspstore=r20 dep r31=0,r31,0,13 // 8KB aligned ;; } { .mmi mov ar.k6=r31 mov ar.rnat=r21 nop 0 ;; } { .mmi mov ar.unat=r17 mov cr.iip=r19 nop 0 } { .mmi mov cr.ipsr=r24 mov cr.ifs=r25 mov pr=r18,0x1ffff ;; } { .mmb mov ar.rsc=r22 mov ar.fpsr=r23 rfi ;; } END(exception_restore) /* * Call exception_save_regs to preserve the interrupted state in a * trapframe. Note that we don't use a call instruction because we * must be careful not to lose track of the RSE state. We then call * trap() with the value of _n_ as an argument to handle the * exception. We arrange for trap() to return to exception_restore * which will restore the interrupted state before executing an rfi to * resume it. */ #define CALL(_func_, _n_, _ifa_) \ { .mib ; \ mov r17=_ifa_ ; \ mov r16=ip ; \ br.sptk exception_save ;; \ } ; \ { .mmi ; \ alloc r15=ar.pfs,0,0,2,0 ;; \ (p11) ssm psr.i ; \ mov out0=_n_ ;; \ } ; \ { .mib ; \ (p11) srlz.d ; \ add out1=16,sp ; \ br.call.sptk rp=_func_ ;; \ } ; \ { .mib ; \ nop 0 ; \ nop 0 ; \ br.sptk exception_restore ;; \ } #define IVT_ENTRY(name, offset) \ .org ia64_vector_table + offset; \ .global ivt_##name; \ .proc ivt_##name; \ .prologue; \ .unwabi @svr4, 'I'; \ .save rp, r0; \ .body; \ ivt_##name: \ XTRACE(offset) #define IVT_END(name) \ .endp ivt_##name #ifdef COMPAT_FREEBSD32 #define IA32_TRAP ia32_trap #else #define IA32_TRAP trap #endif /* * The IA64 Interrupt Vector Table (IVT) contains 20 slots with 64 * bundles per vector and 48 slots with 16 bundles per vector. */ .section .text.ivt,"ax" .align 32768 .global ia64_vector_table .size ia64_vector_table, 32768 ia64_vector_table: IVT_ENTRY(VHPT_Translation, 0x0000) CALL(trap, 0, cr.ifa) IVT_END(VHPT_Translation) IVT_ENTRY(Instruction_TLB, 0x0400) mov r16=cr.ifa mov r17=pr ;; thash r18=r16 ttag r19=r16 ;; add r21=16,r18 // tag add r20=24,r18 // collision chain ;; ld8 r21=[r21] // check VHPT tag ld8 r20=[r20] // bucket head ;; cmp.ne p15,p0=r21,r19 (p15) br.dpnt.few 1f ;; ld8 r21=[r18] // read pte ;; itc.i r21 // insert pte mov pr=r17,0x1ffff ;; rfi // done ;; 1: rsm psr.dt // turn off data translations dep r20=0,r20,61,3 // convert vhpt ptr to physical ;; srlz.d // serialize ld8 r20=[r20] // first entry ;; 2: cmp.eq p15,p0=r0,r20 // done? (p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; cmp.ne p15,p0=r21,r19 // compare tags (p15) br.cond.sptk.few 3f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_ACCESSED ;; or r21=r21,r22 ;; st8 [r20]=r21,8 ;; ld8 r22=[r20] // read rest of pte ;; dep r18=0,r18,61,3 // convert vhpt ptr to physical ;; add r20=16,r18 // address of tag ;; ld8.acq r23=[r20] // read old tag ;; dep r23=-1,r23,63,1 // set ti bit ;; st8.rel [r20]=r23 // store old tag + ti ;; mf // make sure everyone sees ;; st8 [r18]=r21,8 // store pte ;; st8 [r18]=r22,8 ;; st8.rel [r18]=r19 // store new tag ;; itc.i r21 // and place in TLB ssm psr.dt ;; srlz.d mov pr=r17,0x1ffff // restore predicates rfi ;; 3: add r20=24,r20 // next in chain ;; ld8 r20=[r20] // read chain br.cond.sptk.few 2b // loop ;; 9: ssm psr.dt mov pr=r17,0x1ffff // restore predicates ;; srlz.d ;; CALL(trap, 20, cr.ifa) // Page Not Present trap IVT_END(Instruction_TLB) IVT_ENTRY(Data_TLB, 0x0800) mov r16=cr.ifa mov r17=pr ;; thash r18=r16 ttag r19=r16 ;; add r21=16,r18 // tag add r20=24,r18 // collision chain ;; ld8 r21=[r21] // check VHPT tag ld8 r20=[r20] // bucket head ;; cmp.ne p15,p0=r21,r19 (p15) br.dpnt.few 1f ;; ld8 r21=[r18] // read pte ;; itc.d r21 // insert pte mov pr=r17,0x1ffff ;; rfi // done ;; 1: rsm psr.dt // turn off data translations dep r20=0,r20,61,3 // convert vhpt ptr to physical ;; srlz.d // serialize ld8 r20=[r20] // first entry ;; 2: cmp.eq p15,p0=r0,r20 // done? (p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; cmp.ne p15,p0=r21,r19 // compare tags (p15) br.cond.sptk.few 3f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_ACCESSED ;; or r21=r21,r22 ;; st8 [r20]=r21,8 ;; ld8 r22=[r20] // read rest of pte ;; dep r18=0,r18,61,3 // convert vhpt ptr to physical ;; add r20=16,r18 // address of tag ;; ld8.acq r23=[r20] // read old tag ;; dep r23=-1,r23,63,1 // set ti bit ;; st8.rel [r20]=r23 // store old tag + ti ;; mf // make sure everyone sees ;; st8 [r18]=r21,8 // store pte ;; st8 [r18]=r22,8 ;; st8.rel [r18]=r19 // store new tag ;; itc.d r21 // and place in TLB ssm psr.dt ;; srlz.d mov pr=r17,0x1ffff // restore predicates rfi ;; 3: add r20=24,r20 // next in chain ;; ld8 r20=[r20] // read chain br.cond.sptk.few 2b // loop ;; 9: ssm psr.dt mov pr=r17,0x1ffff // restore predicates ;; srlz.d ;; CALL(trap, 20, cr.ifa) // Page Not Present trap IVT_END(Data_TLB) IVT_ENTRY(Alternate_Instruction_TLB, 0x0c00) mov r16=cr.ifa // where did it happen mov r18=pr // save predicates ;; extr.u r17=r16,61,3 // get region number ;; cmp.ge p13,p0=5,r17 // RR0-RR5? cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 (p13) br.spnt 9f ;; (p15) movl r17=PTE_PRESENT+PTE_MA_WB+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \ PTE_AR_RX+PTE_ED (p14) movl r17=PTE_PRESENT+PTE_MA_UC+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \ PTE_AR_RX ;; dep r16=0,r16,50,14 // clear bits above PPN ;; dep r16=r17,r16,0,12 // put pte bits in 0..11 ;; itc.i r16 mov pr=r18,0x1ffff // restore predicates ;; rfi ;; 9: mov pr=r18,0x1ffff // restore predicates CALL(trap, 3, cr.ifa) IVT_END(Alternate_Instruction_TLB) IVT_ENTRY(Alternate_Data_TLB, 0x1000) mov r16=cr.ifa // where did it happen mov r18=pr // save predicates ;; extr.u r17=r16,61,3 // get region number ;; cmp.ge p13,p0=5,r17 // RR0-RR5? cmp.eq p15,p14=7,r17 // RR7->p15, RR6->p14 (p13) br.spnt 9f ;; (p15) movl r17=PTE_PRESENT+PTE_MA_WB+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \ PTE_AR_RW+PTE_ED (p14) movl r17=PTE_PRESENT+PTE_MA_UC+PTE_ACCESSED+PTE_DIRTY+PTE_PL_KERN+ \ PTE_AR_RW ;; dep r16=0,r16,50,14 // clear bits above PPN ;; dep r16=r17,r16,0,12 // put pte bits in 0..11 ;; itc.d r16 mov pr=r18,0x1ffff // restore predicates ;; rfi ;; 9: mov pr=r18,0x1ffff // restore predicates CALL(trap, 4, cr.ifa) IVT_END(Alternate_Data_TLB) IVT_ENTRY(Data_Nested_TLB, 0x1400) // See exception_save_restart and exception_restore_restart for the // contexts that may cause a data nested TLB. We can only use the // banked general registers and predicates, but don't use: // p14 & p15 - Set in exception save // r16 & r17 - Arguments to exception save // r30 - Faulting address (modulo page size) // We assume r30 has the virtual addresses that relate to the data // nested TLB fault. The address does not have to be exact, as long // as it's in the same page. We use physical addressing to avoid // double nested faults. Since all virtual addresses we encounter // here are direct mapped region 7 addresses, we have no problem // constructing physical addresses. { .mlx rsm psr.dt movl r27=ia64_kptdir ;; } { .mii srlz.d dep r27=0,r27,61,3 ;; extr.u r28=r30,3*PAGE_SHIFT-8, PAGE_SHIFT-3 // dir L0 index } { .mii ld8 r27=[r27] // dir L0 page extr.u r26=r30,2*PAGE_SHIFT-5, PAGE_SHIFT-3 // dir L1 index ;; dep r27=0,r27,61,3 ;; } { .mmi shladd r27=r28,3,r27 ;; ld8 r27=[r27] // dir L1 page extr.u r28=r30,PAGE_SHIFT,PAGE_SHIFT-5 // pte index ;; } { .mmi shladd r27=r26,3,r27 ;; mov r26=rr[r30] dep r27=0,r27,61,3 ;; } { .mii ld8 r27=[r27] // pte page shl r28=r28,5 dep r26=0,r26,0,2 ;; } { .mmi add r27=r28,r27 ;; mov cr.ifa=r30 dep r27=0,r27,61,3 ;; } { .mmi ld8 r28=[r27] // pte ;; mov cr.itir=r26 or r28=PTE_DIRTY+PTE_ACCESSED,r28 ;; } { .mmi st8 [r27]=r28 ;; addl r26=NTLBRT_SAVE,r0 addl r27=NTLBRT_RESTORE,r0 } { .mmi itc.d r28 ;; ssm psr.dt cmp.eq p12,p0=r29,r26 ;; } { .mib srlz.d cmp.eq p13,p0=r29,r27 (p12) br.sptk exception_save_restart ;; } { .mib nop 0 nop 0 (p13) br.sptk exception_restore_restart ;; } { .mlx mov r26=ar.bsp movl r27=kstack ;; } { .mmi mov r28=sp nop 0 addl r27=KSTACK_PAGES*PAGE_SIZE-16,r0 ;; } { .mmi mov sp=r27 ;; mov r27=ar.bspstore nop 0 ;; } CALL(trap, 5, r30) IVT_END(Data_Nested_TLB) IVT_ENTRY(Instruction_Key_Miss, 0x1800) CALL(trap, 6, cr.ifa) IVT_END(Instruction_Key_Miss) IVT_ENTRY(Data_Key_Miss, 0x1c00) CALL(trap, 7, cr.ifa) IVT_END(Data_Key_Miss) IVT_ENTRY(Dirty_Bit, 0x2000) mov r16=cr.ifa mov r17=pr ;; thash r18=r16 ;; ttag r19=r16 add r20=24,r18 // collision chain ;; ld8 r20=[r20] // bucket head ;; rsm psr.dt // turn off data translations dep r20=0,r20,61,3 // convert vhpt ptr to physical ;; srlz.d // serialize ld8 r20=[r20] // first entry ;; 1: cmp.eq p15,p0=r0,r20 // done? (p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; cmp.ne p15,p0=r21,r19 // compare tags (p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_DIRTY+PTE_ACCESSED ;; or r21=r22,r21 // set dirty & access bit ;; st8 [r20]=r21,8 // store back ;; ld8 r22=[r20] // read rest of pte ;; dep r18=0,r18,61,3 // convert vhpt ptr to physical ;; add r20=16,r18 // address of tag ;; ld8.acq r23=[r20] // read old tag ;; dep r23=-1,r23,63,1 // set ti bit ;; st8.rel [r20]=r23 // store old tag + ti ;; mf // make sure everyone sees ;; st8 [r18]=r21,8 // store pte ;; st8 [r18]=r22,8 ;; st8.rel [r18]=r19 // store new tag ;; itc.d r21 // and place in TLB ssm psr.dt ;; srlz.d mov pr=r17,0x1ffff // restore predicates rfi ;; 2: add r20=24,r20 // next in chain ;; ld8 r20=[r20] // read chain br.cond.sptk.few 1b // loop ;; 9: ssm psr.dt mov pr=r17,0x1ffff // restore predicates ;; srlz.d ;; CALL(trap, 8, cr.ifa) // die horribly IVT_END(Dirty_Bit) IVT_ENTRY(Instruction_Access_Bit, 0x2400) mov r16=cr.ifa mov r17=pr ;; thash r18=r16 ;; ttag r19=r16 add r20=24,r18 // collision chain ;; ld8 r20=[r20] // bucket head ;; rsm psr.dt // turn off data translations dep r20=0,r20,61,3 // convert vhpt ptr to physical ;; srlz.d // serialize ld8 r20=[r20] // first entry ;; 1: cmp.eq p15,p0=r0,r20 // done? (p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; cmp.ne p15,p0=r21,r19 // compare tags (p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_ACCESSED ;; or r21=r22,r21 // set accessed bit ;; st8 [r20]=r21,8 // store back ;; ld8 r22=[r20] // read rest of pte ;; dep r18=0,r18,61,3 // convert vhpt ptr to physical ;; add r20=16,r18 // address of tag ;; ld8.acq r23=[r20] // read old tag ;; dep r23=-1,r23,63,1 // set ti bit ;; st8.rel [r20]=r23 // store old tag + ti ;; mf // make sure everyone sees ;; st8 [r18]=r21,8 // store pte ;; st8 [r18]=r22,8 ;; st8.rel [r18]=r19 // store new tag ;; itc.i r21 // and place in TLB ssm psr.dt ;; srlz.d mov pr=r17,0x1ffff // restore predicates rfi // walker will retry the access ;; 2: add r20=24,r20 // next in chain ;; ld8 r20=[r20] // read chain br.cond.sptk.few 1b // loop ;; 9: ssm psr.dt mov pr=r17,0x1ffff // restore predicates ;; srlz.d ;; CALL(trap, 9, cr.ifa) IVT_END(Instruction_Access_Bit) IVT_ENTRY(Data_Access_Bit, 0x2800) mov r16=cr.ifa mov r17=pr ;; thash r18=r16 ;; ttag r19=r16 add r20=24,r18 // collision chain ;; ld8 r20=[r20] // bucket head ;; rsm psr.dt // turn off data translations dep r20=0,r20,61,3 // convert vhpt ptr to physical ;; srlz.d // serialize ld8 r20=[r20] // first entry ;; 1: cmp.eq p15,p0=r0,r20 // done? (p15) br.cond.spnt.few 9f // bail if done ;; add r21=16,r20 // tag location ;; ld8 r21=[r21] // read tag ;; cmp.ne p15,p0=r21,r19 // compare tags (p15) br.cond.sptk.few 2f // if not, read next in chain ;; ld8 r21=[r20] // read pte mov r22=PTE_ACCESSED ;; or r21=r22,r21 // set accessed bit ;; st8 [r20]=r21,8 // store back ;; ld8 r22=[r20] // read rest of pte ;; dep r18=0,r18,61,3 // convert vhpt ptr to physical ;; add r20=16,r18 // address of tag ;; ld8.acq r23=[r20] // read old tag ;; dep r23=-1,r23,63,1 // set ti bit ;; st8.rel [r20]=r23 // store old tag + ti ;; mf // make sure everyone sees ;; st8 [r18]=r21,8 // store pte ;; st8 [r18]=r22,8 ;; st8.rel [r18]=r19 // store new tag ;; itc.d r21 // and place in TLB ssm psr.dt ;; srlz.d mov pr=r17,0x1ffff // restore predicates rfi // walker will retry the access ;; 2: add r20=24,r20 // next in chain ;; ld8 r20=[r20] // read chain br.cond.sptk.few 1b // loop ;; 9: ssm psr.dt mov pr=r17,0x1ffff // restore predicates ;; srlz.d ;; CALL(trap, 10, cr.ifa) IVT_END(Data_Access_Bit) IVT_ENTRY(Break_Instruction, 0x2c00) { .mib mov r17=cr.iim mov r16=ip br.sptk exception_save ;; } { .mmi alloc r15=ar.pfs,0,0,2,0 ;; (p11) ssm psr.i mov out0=11 ;; } { .mmi flushrs ;; (p11) srlz.d add out1=16,sp } { .mib nop 0 nop 0 br.call.sptk rp=trap ;; } { .mib nop 0 nop 0 br.sptk exception_restore ;; } IVT_END(Break_Instruction) IVT_ENTRY(External_Interrupt, 0x3000) { .mib - mov r17=cr.ivr // Put the vector in the trap frame. + mov r17=cr.ivr // Put the XIV in the trapframe. mov r16=ip br.sptk exception_save ;; } { .mmi alloc r15=ar.pfs,0,0,1,0 nop 0 nop 0 ;; } { .mib add out0=16,sp nop 0 - br.call.sptk rp=interrupt + br.call.sptk rp=ia64_handle_intr ;; } { .mib nop 0 nop 0 br.sptk exception_restore ;; } IVT_END(External_Interrupt) IVT_ENTRY(Reserved_3400, 0x3400) CALL(trap, 13, cr.ifa) IVT_END(Reserved_3400) IVT_ENTRY(Reserved_3800, 0x3800) CALL(trap, 14, cr.ifa) IVT_END(Reserved_3800) IVT_ENTRY(Reserved_3c00, 0x3c00) CALL(trap, 15, cr.ifa) IVT_END(Reserved_3c00) IVT_ENTRY(Reserved_4000, 0x4000) CALL(trap, 16, cr.ifa) IVT_END(Reserved_4000) IVT_ENTRY(Reserved_4400, 0x4400) CALL(trap, 17, cr.ifa) IVT_END(Reserved_4400) IVT_ENTRY(Reserved_4800, 0x4800) CALL(trap, 18, cr.ifa) IVT_END(Reserved_4800) IVT_ENTRY(Reserved_4c00, 0x4c00) CALL(trap, 19, cr.ifa) IVT_END(Reserved_4c00) IVT_ENTRY(Page_Not_Present, 0x5000) CALL(trap, 20, cr.ifa) IVT_END(Page_Not_Present) IVT_ENTRY(Key_Permission, 0x5100) CALL(trap, 21, cr.ifa) IVT_END(Key_Permission) IVT_ENTRY(Instruction_Access_Rights, 0x5200) CALL(trap, 22, cr.ifa) IVT_END(Instruction_Access_Rights) IVT_ENTRY(Data_Access_Rights, 0x5300) CALL(trap, 23, cr.ifa) IVT_END(Data_Access_Rights) IVT_ENTRY(General_Exception, 0x5400) CALL(trap, 24, cr.ifa) IVT_END(General_Exception) IVT_ENTRY(Disabled_FP_Register, 0x5500) CALL(trap, 25, cr.ifa) IVT_END(Disabled_FP_Register) IVT_ENTRY(NaT_Consumption, 0x5600) CALL(trap, 26, cr.ifa) IVT_END(NaT_Consumption) IVT_ENTRY(Speculation, 0x5700) CALL(trap, 27, cr.iim) IVT_END(Speculation) IVT_ENTRY(Reserved_5800, 0x5800) CALL(trap, 28, cr.ifa) IVT_END(Reserved_5800) IVT_ENTRY(Debug, 0x5900) CALL(trap, 29, cr.ifa) IVT_END(Debug) IVT_ENTRY(Unaligned_Reference, 0x5a00) CALL(trap, 30, cr.ifa) IVT_END(Unaligned_Reference) IVT_ENTRY(Unsupported_Data_Reference, 0x5b00) CALL(trap, 31, cr.ifa) IVT_END(Unsupported_Data_Reference) IVT_ENTRY(Floating_Point_Fault, 0x5c00) CALL(trap, 32, cr.ifa) IVT_END(Floating_Point_Fault) IVT_ENTRY(Floating_Point_Trap, 0x5d00) CALL(trap, 33, cr.ifa) IVT_END(Floating_Point_Trap) IVT_ENTRY(Lower_Privilege_Transfer_Trap, 0x5e00) CALL(trap, 34, cr.ifa) IVT_END(Lower_Privilege_Transfer_Trap) IVT_ENTRY(Taken_Branch_Trap, 0x5f00) CALL(trap, 35, cr.ifa) IVT_END(Taken_Branch_Trap) IVT_ENTRY(Single_Step_Trap, 0x6000) CALL(trap, 36, cr.ifa) IVT_END(Single_Step_Trap) IVT_ENTRY(Reserved_6100, 0x6100) CALL(trap, 37, cr.ifa) IVT_END(Reserved_6100) IVT_ENTRY(Reserved_6200, 0x6200) CALL(trap, 38, cr.ifa) IVT_END(Reserved_6200) IVT_ENTRY(Reserved_6300, 0x6300) CALL(trap, 39, cr.ifa) IVT_END(Reserved_6300) IVT_ENTRY(Reserved_6400, 0x6400) CALL(trap, 40, cr.ifa) IVT_END(Reserved_6400) IVT_ENTRY(Reserved_6500, 0x6500) CALL(trap, 41, cr.ifa) IVT_END(Reserved_6500) IVT_ENTRY(Reserved_6600, 0x6600) CALL(trap, 42, cr.ifa) IVT_END(Reserved_6600) IVT_ENTRY(Reserved_6700, 0x6700) CALL(trap, 43, cr.ifa) IVT_END(Reserved_6700) IVT_ENTRY(Reserved_6800, 0x6800) CALL(trap, 44, cr.ifa) IVT_END(Reserved_6800) IVT_ENTRY(IA_32_Exception, 0x6900) CALL(IA32_TRAP, 45, cr.ifa) IVT_END(IA_32_Exception) IVT_ENTRY(IA_32_Intercept, 0x6a00) CALL(IA32_TRAP, 46, cr.iim) IVT_END(IA_32_Intercept) IVT_ENTRY(IA_32_Interrupt, 0x6b00) CALL(IA32_TRAP, 47, cr.ifa) IVT_END(IA_32_Interrupt) IVT_ENTRY(Reserved_6c00, 0x6c00) CALL(trap, 48, cr.ifa) IVT_END(Reserved_6c00) IVT_ENTRY(Reserved_6d00, 0x6d00) CALL(trap, 49, cr.ifa) IVT_END(Reserved_6d00) IVT_ENTRY(Reserved_6e00, 0x6e00) CALL(trap, 50, cr.ifa) IVT_END(Reserved_6e00) IVT_ENTRY(Reserved_6f00, 0x6f00) CALL(trap, 51, cr.ifa) IVT_END(Reserved_6f00) IVT_ENTRY(Reserved_7000, 0x7000) CALL(trap, 52, cr.ifa) IVT_END(Reserved_7000) IVT_ENTRY(Reserved_7100, 0x7100) CALL(trap, 53, cr.ifa) IVT_END(Reserved_7100) IVT_ENTRY(Reserved_7200, 0x7200) CALL(trap, 54, cr.ifa) IVT_END(Reserved_7200) IVT_ENTRY(Reserved_7300, 0x7300) CALL(trap, 55, cr.ifa) IVT_END(Reserved_7300) IVT_ENTRY(Reserved_7400, 0x7400) CALL(trap, 56, cr.ifa) IVT_END(Reserved_7400) IVT_ENTRY(Reserved_7500, 0x7500) CALL(trap, 57, cr.ifa) IVT_END(Reserved_7500) IVT_ENTRY(Reserved_7600, 0x7600) CALL(trap, 58, cr.ifa) IVT_END(Reserved_7600) IVT_ENTRY(Reserved_7700, 0x7700) CALL(trap, 59, cr.ifa) IVT_END(Reserved_7700) IVT_ENTRY(Reserved_7800, 0x7800) CALL(trap, 60, cr.ifa) IVT_END(Reserved_7800) IVT_ENTRY(Reserved_7900, 0x7900) CALL(trap, 61, cr.ifa) IVT_END(Reserved_7900) IVT_ENTRY(Reserved_7a00, 0x7a00) CALL(trap, 62, cr.ifa) IVT_END(Reserved_7a00) IVT_ENTRY(Reserved_7b00, 0x7b00) CALL(trap, 63, cr.ifa) IVT_END(Reserved_7b00) IVT_ENTRY(Reserved_7c00, 0x7c00) CALL(trap, 64, cr.ifa) IVT_END(Reserved_7c00) IVT_ENTRY(Reserved_7d00, 0x7d00) CALL(trap, 65, cr.ifa) IVT_END(Reserved_7d00) IVT_ENTRY(Reserved_7e00, 0x7e00) CALL(trap, 66, cr.ifa) IVT_END(Reserved_7e00) IVT_ENTRY(Reserved_7f00, 0x7f00) CALL(trap, 67, cr.ifa) IVT_END(Reserved_7f00) diff --git a/sys/ia64/ia64/highfp.c b/sys/ia64/ia64/highfp.c index 145ee488ba66..3aa3af0e4a0a 100644 --- a/sys/ia64/ia64/highfp.c +++ b/sys/ia64/ia64/highfp.c @@ -1,181 +1,181 @@ /*- * Copyright (c) 2009 Marcel Moolenaar * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include static struct mtx ia64_highfp_mtx; static void ia64_highfp_init(void *_) { mtx_init(&ia64_highfp_mtx, "High FP lock", NULL, MTX_SPIN); } SYSINIT(ia64_highfp_init, SI_SUB_LOCK, SI_ORDER_ANY, ia64_highfp_init, NULL); #ifdef SMP static int ia64_highfp_ipi(struct pcpu *cpu) { int error; - ipi_send(cpu, IPI_HIGH_FP); + ipi_send(cpu, ia64_ipi_highfp); error = msleep_spin(&cpu->pc_fpcurthread, &ia64_highfp_mtx, "High FP", 0); return (error); } #endif int ia64_highfp_drop(struct thread *td) { struct pcb *pcb; struct pcpu *cpu; pcb = td->td_pcb; mtx_lock_spin(&ia64_highfp_mtx); cpu = pcb->pcb_fpcpu; if (cpu != NULL) { KASSERT(cpu->pc_fpcurthread == td, ("cpu->pc_fpcurthread != td")); td->td_frame->tf_special.psr |= IA64_PSR_DFH; pcb->pcb_fpcpu = NULL; cpu->pc_fpcurthread = NULL; } mtx_unlock_spin(&ia64_highfp_mtx); return ((cpu != NULL) ? 1 : 0); } int ia64_highfp_enable(struct thread *td, struct trapframe *tf) { struct pcb *pcb; struct pcpu *cpu; struct thread *td1; pcb = td->td_pcb; mtx_lock_spin(&ia64_highfp_mtx); KASSERT((tf->tf_special.psr & IA64_PSR_DFH) != 0, ("(tf->tf_special.psr & IA64_PSR_DFH) == 0")); cpu = pcb->pcb_fpcpu; #ifdef SMP if (cpu != NULL && cpu != pcpup) { KASSERT(cpu->pc_fpcurthread == td, ("cpu->pc_fpcurthread != td")); ia64_highfp_ipi(cpu); } #endif td1 = PCPU_GET(fpcurthread); if (td1 != NULL && td1 != td) { KASSERT(td1->td_pcb->pcb_fpcpu == pcpup, ("td1->td_pcb->pcb_fpcpu != pcpup")); save_high_fp(&td1->td_pcb->pcb_high_fp); td1->td_frame->tf_special.psr |= IA64_PSR_DFH; td1->td_pcb->pcb_fpcpu = NULL; PCPU_SET(fpcurthread, NULL); td1 = NULL; } if (td1 == NULL) { KASSERT(pcb->pcb_fpcpu == NULL, ("pcb->pcb_fpcpu != NULL")); KASSERT(PCPU_GET(fpcurthread) == NULL, ("PCPU_GET(fpcurthread) != NULL")); restore_high_fp(&pcb->pcb_high_fp); PCPU_SET(fpcurthread, td); pcb->pcb_fpcpu = pcpup; tf->tf_special.psr &= ~IA64_PSR_MFH; } tf->tf_special.psr &= ~IA64_PSR_DFH; mtx_unlock_spin(&ia64_highfp_mtx); return ((td1 != NULL) ? 1 : 0); } int ia64_highfp_save(struct thread *td) { struct pcb *pcb; struct pcpu *cpu; pcb = td->td_pcb; mtx_lock_spin(&ia64_highfp_mtx); cpu = pcb->pcb_fpcpu; #ifdef SMP if (cpu != NULL && cpu != pcpup) { KASSERT(cpu->pc_fpcurthread == td, ("cpu->pc_fpcurthread != td")); ia64_highfp_ipi(cpu); } else #endif if (cpu != NULL) { KASSERT(cpu->pc_fpcurthread == td, ("cpu->pc_fpcurthread != td")); save_high_fp(&pcb->pcb_high_fp); td->td_frame->tf_special.psr |= IA64_PSR_DFH; pcb->pcb_fpcpu = NULL; cpu->pc_fpcurthread = NULL; } mtx_unlock_spin(&ia64_highfp_mtx); return ((cpu != NULL) ? 1 : 0); } #ifdef SMP int ia64_highfp_save_ipi(void) { struct thread *td; mtx_lock_spin(&ia64_highfp_mtx); td = PCPU_GET(fpcurthread); if (td != NULL) { KASSERT(td->td_pcb->pcb_fpcpu == pcpup, ("td->td_pcb->pcb_fpcpu != pcpup")); save_high_fp(&td->td_pcb->pcb_high_fp); td->td_frame->tf_special.psr |= IA64_PSR_DFH; td->td_pcb->pcb_fpcpu = NULL; PCPU_SET(fpcurthread, NULL); } mtx_unlock_spin(&ia64_highfp_mtx); wakeup(&PCPU_GET(fpcurthread)); return ((td != NULL) ? 1 : 0); } #endif diff --git a/sys/ia64/ia64/interrupt.c b/sys/ia64/ia64/interrupt.c index a9bee2798df0..ad8a4e3ceda5 100644 --- a/sys/ia64/ia64/interrupt.c +++ b/sys/ia64/ia64/interrupt.c @@ -1,433 +1,394 @@ /* $FreeBSD$ */ /* $NetBSD: interrupt.c,v 1.23 1998/02/24 07:38:01 thorpej Exp $ */ /*- * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University. * All rights reserved. * * Authors: Keith Bostic, Chris G. Demetriou * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ /*- * Additional Copyright (c) 1997 by Matthew Jacob for NASA/Ames Research Center. * Redistribute and modify at will, leaving only this additional copyright * notice. */ #include "opt_ddb.h" #include /* RCS ID & Copyright macro defns */ #include #include #include #include #include #include +#include #include #include #include #include #include #include #include #include -#include #include #include #include #include +#include #include #include #include #include -#ifdef EVCNT_COUNTERS -struct evcnt clock_intr_evcnt; /* event counter for clock intrs. */ -#else -#include -#include -#endif - #ifdef DDB #include #endif -SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics"); - -static int adjust_edges = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD, - &adjust_edges, 0, "Number of times ITC got more than 12.5% behind"); - -static int adjust_excess = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD, - &adjust_excess, 0, "Total number of ignored ITC interrupts"); - -static int adjust_lost = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD, - &adjust_lost, 0, "Total number of lost ITC interrupts"); - -static int adjust_ticks = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD, - &adjust_ticks, 0, "Total number of ITC interrupts with adjustment"); - - struct ia64_intr { struct intr_event *event; /* interrupt event */ volatile long *cntp; /* interrupt counter */ struct sapic *sapic; u_int irq; }; -static struct ia64_intr *ia64_intrs[256]; +ia64_ihtype *ia64_handler[IA64_NXIVS]; +static enum ia64_xiv_use ia64_xiv[IA64_NXIVS]; +static struct ia64_intr *ia64_intrs[IA64_NXIVS]; -static void ia64_dispatch_intr(void *, u_int); +static ia64_ihtype ia64_ih_invalid; +static ia64_ihtype ia64_ih_irq; void -interrupt(struct trapframe *tf) +ia64_xiv_init(void) { - struct thread *td; - uint64_t adj, clk, itc; - int64_t delta; - u_int vector; - int count; - uint8_t inta; + u_int xiv; - ia64_set_fpsr(IA64_FPSR_DEFAULT); + for (xiv = 0; xiv < IA64_NXIVS; xiv++) { + ia64_handler[xiv] = ia64_ih_invalid; + ia64_xiv[xiv] = IA64_XIV_FREE; + ia64_intrs[xiv] = NULL; + } + (void)ia64_xiv_reserve(15, IA64_XIV_ARCH, NULL); +} - td = curthread; +int +ia64_xiv_free(u_int xiv, enum ia64_xiv_use what) +{ - PCPU_INC(cnt.v_intr); + if (xiv >= IA64_NXIVS) + return (EINVAL); + if (what == IA64_XIV_FREE || what == IA64_XIV_ARCH) + return (EINVAL); + if (ia64_xiv[xiv] != what) + return (ENXIO); + ia64_xiv[xiv] = IA64_XIV_FREE; + ia64_handler[xiv] = ia64_ih_invalid; + return (0); +} - vector = tf->tf_special.ifa; +int +ia64_xiv_reserve(u_int xiv, enum ia64_xiv_use what, ia64_ihtype ih) +{ - next: - /* - * Handle ExtINT interrupts by generating an INTA cycle to - * read the vector. - */ - if (vector == 0) { - PCPU_INC(md.stats.pcs_nextints); - inta = ia64_ld1(&ia64_pib->ib_inta); - if (inta == 15) { - PCPU_INC(md.stats.pcs_nstrays); - __asm __volatile("mov cr.eoi = r0;; srlz.d"); - goto stray; - } - vector = (int)inta; - } else if (vector == 15) { - PCPU_INC(md.stats.pcs_nstrays); - goto stray; - } + if (xiv >= IA64_NXIVS) + return (EINVAL); + if (what == IA64_XIV_FREE) + return (EINVAL); + if (ia64_xiv[xiv] != IA64_XIV_FREE) + return (EBUSY); + ia64_xiv[xiv] = what; + ia64_handler[xiv] = (ih == NULL) ? ia64_ih_invalid: ih; + if (1 || bootverbose) + printf("XIV %u: use=%u, IH=%p\n", xiv, what, ih); + return (0); +} - if (vector == CLOCK_VECTOR) {/* clock interrupt */ - /* CTR0(KTR_INTR, "clock interrupt"); */ +u_int +ia64_xiv_alloc(u_int prio, enum ia64_xiv_use what, ia64_ihtype ih) +{ + u_int hwprio; + u_int xiv0, xiv; - itc = ia64_get_itc(); + hwprio = prio >> 2; + if (hwprio > IA64_MAX_HWPRIO) + hwprio = IA64_MAX_HWPRIO; - PCPU_INC(md.stats.pcs_nclks); -#ifdef EVCNT_COUNTERS - clock_intr_evcnt.ev_count++; -#else - intrcnt[INTRCNT_CLOCK]++; -#endif + xiv0 = IA64_NXIVS - (hwprio + 1) * 16; - critical_enter(); + KASSERT(xiv0 > IA64_MIN_XIV, ("%s: min XIV", __func__)); + KASSERT(xiv0 < IA64_NXIVS, ("%s: max XIV", __func__)); - adj = PCPU_GET(md.clockadj); - clk = PCPU_GET(md.clock); - delta = itc - clk; - count = 0; - while (delta >= ia64_clock_reload) { - /* Only the BSP runs the real clock */ - if (PCPU_GET(cpuid) == 0) - hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); - else - hardclock_cpu(TRAPF_USERMODE(tf)); - if (profprocs != 0) - profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); - statclock(TRAPF_USERMODE(tf)); - delta -= ia64_clock_reload; - clk += ia64_clock_reload; - if (adj != 0) - adjust_ticks++; - count++; - } - ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj); - if (count > 0) { - adjust_lost += count - 1; - if (delta > (ia64_clock_reload >> 3)) { - if (adj == 0) - adjust_edges++; - adj = ia64_clock_reload >> 4; - } else - adj = 0; - } else { - adj = 0; - adjust_excess++; - } - PCPU_SET(md.clock, clk); - PCPU_SET(md.clockadj, adj); - critical_exit(); - ia64_srlz_d(); + xiv = xiv0; + while (xiv < IA64_NXIVS && ia64_xiv_reserve(xiv, what, ih)) + xiv++; -#ifdef SMP - } else if (vector == ipi_vector[IPI_AST]) { - PCPU_INC(md.stats.pcs_nasts); - CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid)); - } else if (vector == ipi_vector[IPI_HIGH_FP]) { - PCPU_INC(md.stats.pcs_nhighfps); - ia64_highfp_save_ipi(); - } else if (vector == ipi_vector[IPI_RENDEZVOUS]) { - PCPU_INC(md.stats.pcs_nrdvs); - CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid)); - enable_intr(); - smp_rendezvous_action(); - disable_intr(); - } else if (vector == ipi_vector[IPI_STOP]) { - PCPU_INC(md.stats.pcs_nstops); - cpumask_t mybit = PCPU_GET(cpumask); - - /* Make sure IPI_STOP_HARD is mapped to IPI_STOP. */ - KASSERT(IPI_STOP == IPI_STOP_HARD, - ("%s: IPI_STOP_HARD not handled.", __func__)); - - savectx(PCPU_PTR(md.pcb)); - atomic_set_int(&stopped_cpus, mybit); - while ((started_cpus & mybit) == 0) - cpu_spinwait(); - atomic_clear_int(&started_cpus, mybit); - atomic_clear_int(&stopped_cpus, mybit); - } else if (vector == ipi_vector[IPI_PREEMPT]) { - PCPU_INC(md.stats.pcs_npreempts); - CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid)); - __asm __volatile("mov cr.eoi = r0;; srlz.d"); - enable_intr(); - sched_preempt(curthread); - disable_intr(); - goto stray; -#endif - } else { - PCPU_INC(md.stats.pcs_nhwints); - atomic_add_int(&td->td_intr_nesting_level, 1); - ia64_dispatch_intr(tf, vector); - atomic_subtract_int(&td->td_intr_nesting_level, 1); - } + if (xiv < IA64_NXIVS) + return (xiv); - __asm __volatile("mov cr.eoi = r0;; srlz.d"); - vector = ia64_get_ivr(); - if (vector != 15) - goto next; + xiv = xiv0; + while (xiv >= IA64_MIN_XIV && ia64_xiv_reserve(xiv, what, ih)) + xiv--; -stray: - if (TRAPF_USERMODE(tf)) { - enable_intr(); - userret(td, tf); - mtx_assert(&Giant, MA_NOTOWNED); - do_ast(tf); - } + return ((xiv >= IA64_MIN_XIV) ? xiv : 0); } - static void ia64_intr_eoi(void *arg) { - u_int vector = (uintptr_t)arg; + u_int xiv = (uintptr_t)arg; struct ia64_intr *i; - i = ia64_intrs[vector]; - if (i != NULL) - sapic_eoi(i->sapic, vector); + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s", __func__)); + sapic_eoi(i->sapic, xiv); } static void ia64_intr_mask(void *arg) { - u_int vector = (uintptr_t)arg; + u_int xiv = (uintptr_t)arg; struct ia64_intr *i; - i = ia64_intrs[vector]; - if (i != NULL) { - sapic_mask(i->sapic, i->irq); - sapic_eoi(i->sapic, vector); - } + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s", __func__)); + sapic_mask(i->sapic, i->irq); + sapic_eoi(i->sapic, xiv); } static void ia64_intr_unmask(void *arg) { - u_int vector = (uintptr_t)arg; + u_int xiv = (uintptr_t)arg; struct ia64_intr *i; - i = ia64_intrs[vector]; - if (i != NULL) - sapic_unmask(i->sapic, i->irq); + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s", __func__)); + sapic_unmask(i->sapic, i->irq); } int ia64_setup_intr(const char *name, int irq, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) { struct ia64_intr *i; struct sapic *sa; char *intrname; - u_int prio, vector; + u_int prio, xiv; int error; prio = intr_priority(flags); if (prio > PRI_MAX_ITHD) return (EINVAL); /* XXX lock */ - /* Get the I/O SAPIC and vector that corresponds to the IRQ. */ - sa = sapic_lookup(irq, &vector); + /* Get the I/O SAPIC and XIV that corresponds to the IRQ. */ + sa = sapic_lookup(irq, &xiv); if (sa == NULL) { /* XXX unlock */ return (EINVAL); } - if (vector == 0) { + if (xiv == 0) { /* XXX unlock */ i = malloc(sizeof(struct ia64_intr), M_DEVBUF, M_ZERO | M_WAITOK); /* XXX lock */ - sa = sapic_lookup(irq, &vector); + sa = sapic_lookup(irq, &xiv); KASSERT(sa != NULL, ("sapic_lookup")); - if (vector != 0) + if (xiv != 0) free(i, M_DEVBUF); } /* - * If the IRQ has no vector assigned to it yet, assign one based + * If the IRQ has no XIV assigned to it yet, assign one based * on the priority. */ - if (vector == 0) { - vector = (256 - 64) - (prio << 1); - while (vector < 256 && ia64_intrs[vector] != NULL) - vector++; + if (xiv == 0) { + xiv = ia64_xiv_alloc(prio, IA64_XIV_IRQ, ia64_ih_irq); + if (xiv == 0) { + /* XXX unlock */ + free(i, M_DEVBUF); + return (ENOSPC); + } - error = intr_event_create(&i->event, (void *)(uintptr_t)vector, + error = intr_event_create(&i->event, (void *)(uintptr_t)xiv, 0, irq, ia64_intr_mask, ia64_intr_unmask, ia64_intr_eoi, NULL, "irq%u:", irq); if (error) { + ia64_xiv_free(xiv, IA64_XIV_IRQ); /* XXX unlock */ free(i, M_DEVBUF); return (error); } i->sapic = sa; i->irq = irq; - i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ; - ia64_intrs[vector] = i; - sapic_enable(sa, irq, vector); + i->cntp = intrcnt + xiv; + ia64_intrs[xiv] = i; /* XXX unlock */ + sapic_enable(sa, irq, xiv); + if (name != NULL && *name != '\0') { /* XXX needs abstraction. Too error prone. */ - intrname = intrnames + - (irq + INTRCNT_ISA_IRQ) * INTRNAME_LEN; + intrname = intrnames + xiv * INTRNAME_LEN; memset(intrname, ' ', INTRNAME_LEN - 1); bcopy(name, intrname, strlen(name)); } } else { - i = ia64_intrs[vector]; + i = ia64_intrs[xiv]; /* XXX unlock */ } - KASSERT(i != NULL, ("vector mapping bug")); + KASSERT(i != NULL, ("XIV mapping bug")); error = intr_event_add_handler(i->event, name, filter, handler, arg, prio, flags, cookiep); return (error); } int ia64_teardown_intr(void *cookie) { return (intr_event_remove_handler(cookie)); } -static void -ia64_dispatch_intr(void *frame, u_int vector) +/* + * Interrupt handlers. + */ + +void +ia64_handle_intr(struct trapframe *tf) +{ + struct thread *td; + u_int rfi, xiv; + + td = curthread; + ia64_set_fpsr(IA64_FPSR_DEFAULT); + PCPU_INC(cnt.v_intr); + + xiv = tf->tf_special.ifa; + if (xiv == 15) { + PCPU_INC(md.stats.pcs_nstrays); + goto out; + } + + while (xiv != 15) { + CTR1(KTR_INTR, "INTR: XIV=%u", xiv); + critical_enter(); + rfi = (ia64_handler[xiv])(td, xiv, tf); + if (rfi) { + critical_exit(); + return; + } + xiv = ia64_get_ivr(); + critical_exit(); + ia64_srlz_d(); + } + + out: + if (TRAPF_USERMODE(tf)) { + while (td->td_flags & (TDF_ASTPENDING|TDF_NEEDRESCHED)) { + enable_intr(); + ast(tf); + disable_intr(); + } + } +} + +static u_int +ia64_ih_invalid(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + ia64_srlz_d(); + panic("invalid XIV: %u", xiv); + return (1); +} + +static u_int +ia64_ih_irq(struct thread *td, u_int xiv, struct trapframe *tf) { struct ia64_intr *i; struct intr_event *ie; /* our interrupt event */ - /* - * Find the interrupt thread for this vector. - */ - i = ia64_intrs[vector]; - KASSERT(i != NULL, ("%s: unassigned vector", __func__)); + PCPU_INC(md.stats.pcs_nhwints); + + /* Find the interrupt thread for this XIV. */ + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s: unassigned XIV", __func__)); (*i->cntp)++; ie = i->event; KASSERT(ie != NULL, ("%s: interrupt without event", __func__)); - if (intr_event_handle(ie, frame) != 0) { - /* - * XXX: The pre-INTR_FILTER code didn't mask stray - * interrupts. - */ - ia64_intr_mask((void *)(uintptr_t)vector); + if (intr_event_handle(ie, tf) != 0) { + ia64_intr_mask((void *)(uintptr_t)xiv); log(LOG_ERR, "stray irq%u\n", i->irq); } + ia64_set_eoi(0); + ia64_srlz_d(); + return (0); } #ifdef DDB static void -db_print_vector(u_int vector, int always) +db_print_xiv(u_int xiv, int always) { struct ia64_intr *i; - i = ia64_intrs[vector]; + i = ia64_intrs[xiv]; if (i != NULL) { - db_printf("vector %u (%p): ", vector, i); + db_printf("XIV %u (%p): ", xiv, i); sapic_print(i->sapic, i->irq); } else if (always) - db_printf("vector %u: unassigned\n", vector); + db_printf("XIV %u: unassigned\n", xiv); } -DB_SHOW_COMMAND(vector, db_show_vector) +DB_SHOW_COMMAND(xiv, db_show_xiv) { - u_int vector; + u_int xiv; if (have_addr) { - vector = ((addr >> 4) % 16) * 10 + (addr % 16); - if (vector >= 256) - db_printf("error: vector %u not in range [0..255]\n", - vector); + xiv = ((addr >> 4) % 16) * 10 + (addr % 16); + if (xiv >= IA64_NXIVS) + db_printf("error: XIV %u not in range [0..%u]\n", + xiv, IA64_NXIVS - 1); else - db_print_vector(vector, 1); + db_print_xiv(xiv, 1); } else { - for (vector = 0; vector < 256; vector++) - db_print_vector(vector, 0); + for (xiv = 0; xiv < IA64_NXIVS; xiv++) + db_print_xiv(xiv, 0); } } #endif diff --git a/sys/ia64/ia64/locore.S b/sys/ia64/ia64/locore.S index afba834823b0..ce66dcacec6f 100644 --- a/sys/ia64/ia64/locore.S +++ b/sys/ia64/ia64/locore.S @@ -1,472 +1,469 @@ /*- * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ +#include #include #include +#include #include -#include -#include - -#ifndef EVCNT_COUNTERS -#define _LOCORE #include -#endif +#include .section .data.proc0,"aw" .global kstack .align PAGE_SIZE kstack: .space KSTACK_PAGES * PAGE_SIZE .text /* * Not really a leaf but we can't return. * The EFI loader passes the physical address of the bootinfo block in * register r8. */ ENTRY_NOPROFILE(__start, 1) .prologue .save rp,r0 .body { .mlx mov ar.rsc=0 movl r16=ia64_vector_table // set up IVT early ;; } { .mlx mov cr.iva=r16 movl r16=kstack ;; } { .mmi srlz.i ;; ssm IA64_PSR_DFH mov r17=KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16 ;; } { .mlx add sp=r16,r17 // proc0's stack movl gp=__gp // find kernel globals ;; } { .mlx mov ar.bspstore=r16 // switch backing store movl r16=pa_bootinfo ;; } { .mmi st8 [r16]=r8 // save the PA of the bootinfo block loadrs // invalidate regs mov r17=IA64_DCR_DEFAULT ;; } { .mmi mov cr.dcr=r17 mov ar.rsc=3 // turn rse back on nop 0 ;; } { .mmi srlz.d alloc r16=ar.pfs,0,0,1,0 mov out0=r0 // we are linked at the right address ;; // we just need to process fptrs } { .mib nop 0 nop 0 br.call.sptk.many rp=_reloc ;; } { .mib nop 0 nop 0 br.call.sptk.many rp=ia64_init ;; } // We have the new bspstore in r8 and the new sp in r9. // Switch onto the new stack and call mi_startup(). { .mmi mov ar.rsc = 0 ;; mov ar.bspstore = r8 mov sp = r9 ;; } { .mmi loadrs ;; mov ar.rsc = 3 nop 0 ;; } { .mib nop 0 nop 0 br.call.sptk.many rp=mi_startup ;; } /* NOTREACHED */ 1: br.cond.sptk.few 1b END(__start) /* * fork_trampoline() * * Arrange for a function to be invoked neatly, after a cpu_switch(). * * Invokes fork_exit() passing in three arguments: a callout function, an * argument to the callout, and a trapframe pointer. For child processes * returning from fork(2), the argument is a pointer to the child process. * * The callout function and its argument is in the trapframe in scratch * registers r2 and r3. */ ENTRY(fork_trampoline, 0) .prologue .save rp,r0 .body { .mmi alloc r14=ar.pfs,0,0,3,0 add r15=32+SIZEOF_SPECIAL+8,sp add r16=32+SIZEOF_SPECIAL+16,sp ;; } { .mmi ld8 out0=[r15] ld8 out1=[r16] nop 0 } { .mib add out2=16,sp nop 0 br.call.sptk rp=fork_exit ;; } // If we get back here, it means we're a user space process that's // the immediate result of fork(2). .global enter_userland .type enter_userland, @function enter_userland: { .mib nop 0 nop 0 br.sptk epc_syscall_return ;; } END(fork_trampoline) #ifdef SMP /* * AP wake-up entry point. The handoff state is similar as for the BSP, * as described on page 3-9 of the IPF SAL Specification. The difference * lies in the contents of register b0. For APs this register holds the * return address into the SAL rendezvous routine. * * Note that we're responsible for clearing the IRR bit by reading cr.ivr * and issuing the EOI to the local SAPIC. */ .align 32 ENTRY_NOPROFILE(os_boot_rendez,0) mov r16=cr.ivr // clear IRR bit ;; srlz.d mov cr.eoi=r0 // ACK the wake-up ;; srlz.d rsm IA64_PSR_IC|IA64_PSR_I ;; mov r16 = (5<<8)|(PAGE_SHIFT<<2)|1 movl r17 = 5<<61 ;; mov rr[r17] = r16 ;; srlz.d mov r16 = (6<<8)|(IA64_ID_PAGE_SHIFT<<2) movl r17 = 6<<61 ;; mov rr[r17] = r16 ;; srlz.d mov r16 = (7<<8)|(IA64_ID_PAGE_SHIFT<<2) movl r17 = 7<<61 ;; mov rr[r17] = r16 ;; srlz.d mov r18 = 28<<2 movl r16 = PTE_PRESENT+PTE_MA_WB+PTE_ACCESSED+PTE_DIRTY+ \ PTE_PL_KERN+PTE_AR_RWX+PTE_ED ;; mov cr.ifa = r17 mov cr.itir = r18 ptr.d r17, r18 ptr.i r17, r18 ;; srlz.i ;; itr.d dtr[r0] = r16 mov r18 = IA64_DCR_DEFAULT ;; itr.i itr[r0] = r16 mov cr.dcr = r18 ;; srlz.i ;; 1: mov r16 = ip add r17 = 2f-1b, r17 movl r18 = (IA64_PSR_AC|IA64_PSR_BN|IA64_PSR_DFH|IA64_PSR_DT|IA64_PSR_IC|IA64_PSR_IT|IA64_PSR_RT) ;; add r17 = r17, r16 mov cr.ipsr = r18 mov cr.ifs = r0 ;; mov cr.iip = r17 ;; rfi .align 32 2: { .mlx mov ar.rsc = 0 movl r16 = ia64_vector_table // set up IVT early ;; } { .mlx mov cr.iva = r16 movl r16 = ap_stack ;; } { .mmi srlz.i ;; ld8 r16 = [r16] mov r18 = KSTACK_PAGES*PAGE_SIZE-SIZEOF_PCB-SIZEOF_TRAPFRAME-16 ;; } { .mlx mov ar.bspstore = r16 movl gp = __gp ;; } { .mmi loadrs ;; alloc r17 = ar.pfs, 0, 0, 0, 0 add sp = r18, r16 ;; } { .mib mov ar.rsc = 3 nop 0 br.call.sptk.few rp = ia64_ap_startup ;; } /* NOT REACHED */ 9: { .mib nop 0 nop 0 br.sptk 9b ;; } END(os_boot_rendez) #endif /* !SMP */ /* * Create a default interrupt name table. The first entry (vector 0) is * hardwaired to the clock interrupt. */ .data .align 8 EXPORT(intrnames) .ascii "clock" .fill INTRNAME_LEN - 5 - 1, 1, ' ' .byte 0 -intr_n = 0 +intr_n = 1 .rept INTRCNT_COUNT - 1 .ascii "#" .byte intr_n / 100 + '0' .byte (intr_n % 100) / 10 + '0' .byte intr_n % 10 + '0' .fill INTRNAME_LEN - 1 - 3 - 1, 1, ' ' .byte 0 intr_n = intr_n + 1 .endr EXPORT(eintrnames) .align 8 EXPORT(intrcnt) .fill INTRCNT_COUNT, 8, 0 EXPORT(eintrcnt) .text // in0: image base STATIC_ENTRY(_reloc, 1) alloc loc0=ar.pfs,1,2,0,0 mov loc1=rp ;; movl r15=@gprel(_DYNAMIC) // find _DYNAMIC etc. movl r2=@gprel(fptr_storage) movl r3=@gprel(fptr_storage_end) ;; add r15=r15,gp // relocate _DYNAMIC etc. add r2=r2,gp add r3=r3,gp ;; 1: ld8 r16=[r15],8 // read r15->d_tag ;; ld8 r17=[r15],8 // and r15->d_val ;; cmp.eq p6,p0=DT_NULL,r16 // done? (p6) br.cond.dpnt.few 2f ;; cmp.eq p6,p0=DT_RELA,r16 ;; (p6) add r18=r17,in0 // found rela section ;; cmp.eq p6,p0=DT_RELASZ,r16 ;; (p6) mov r19=r17 // found rela size ;; cmp.eq p6,p0=DT_SYMTAB,r16 ;; (p6) add r20=r17,in0 // found symbol table ;; (p6) setf.sig f8=r20 ;; cmp.eq p6,p0=DT_SYMENT,r16 ;; (p6) setf.sig f9=r17 // found symbol entry size ;; cmp.eq p6,p0=DT_RELAENT,r16 ;; (p6) mov r22=r17 // found rela entry size ;; br.sptk.few 1b 2: ld8 r15=[r18],8 // read r_offset ;; ld8 r16=[r18],8 // read r_info add r15=r15,in0 // relocate r_offset ;; ld8 r17=[r18],8 // read r_addend sub r19=r19,r22 // update relasz extr.u r23=r16,0,32 // ELF64_R_TYPE(r16) ;; cmp.eq p6,p0=R_IA_64_NONE,r23 (p6) br.cond.dpnt.few 3f ;; cmp.eq p6,p0=R_IA_64_REL64LSB,r23 (p6) br.cond.dptk.few 4f ;; extr.u r16=r16,32,32 // ELF64_R_SYM(r16) ;; setf.sig f10=r16 // so we can multiply ;; xma.lu f10=f10,f9,f8 // f10=symtab + r_sym*syment ;; getf.sig r16=f10 ;; add r16=8,r16 // address of st_value ;; ld8 r16=[r16] // read symbol value ;; add r16=r16,in0 // relocate symbol value ;; cmp.eq p6,p0=R_IA_64_DIR64LSB,r23 (p6) br.cond.dptk.few 5f ;; cmp.eq p6,p0=R_IA_64_FPTR64LSB,r23 (p6) br.cond.dptk.few 6f ;; 3: cmp.ltu p6,p0=0,r19 // more? (p6) br.cond.dptk.few 2b // loop mov r8=0 // success return value br.cond.sptk.few 9f // done 4: add r16=in0,r17 // BD + A ;; st8 [r15]=r16 // word64 (LSB) br.cond.sptk.few 3b 5: add r16=r16,r17 // S + A ;; st8 [r15]=r16 // word64 (LSB) br.cond.sptk.few 3b 6: movl r17=@gprel(fptr_storage) ;; add r17=r17,gp // start of fptrs ;; 7: cmp.geu p6,p0=r17,r2 // end of fptrs? (p6) br.cond.dpnt.few 8f // can't find existing fptr ld8 r20=[r17] // read function from fptr ;; cmp.eq p6,p0=r16,r20 // same function? ;; (p6) st8 [r15]=r17 // reuse fptr (p6) br.cond.sptk.few 3b // done add r17=16,r17 // next fptr br.cond.sptk.few 7b 8: // allocate new fptr mov r8=1 // failure return value cmp.geu p6,p0=r2,r3 // space left? (p6) br.cond.dpnt.few 9f // bail out st8 [r15]=r2 // install fptr st8 [r2]=r16,8 // write fptr address ;; st8 [r2]=gp,8 // write fptr gp br.cond.sptk.few 3b 9: mov ar.pfs=loc0 mov rp=loc1 ;; br.ret.sptk.few rp END(_reloc) .data .align 16 .global fptr_storage fptr_storage: .space 4096*16 // XXX fptr_storage_end: diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index ed7a288e0c24..fd1bc2aea878 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -1,1558 +1,1559 @@ /*- * Copyright (c) 2003,2004 Marcel Moolenaar * Copyright (c) 2000,2001 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include "opt_msgbuf.h" #include "opt_sched.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #endif #include #include SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, ""); SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RD, 0, ""); static u_int bus_freq; SYSCTL_UINT(_hw_freq, OID_AUTO, bus, CTLFLAG_RD, &bus_freq, 0, "Bus clock frequency"); static u_int cpu_freq; SYSCTL_UINT(_hw_freq, OID_AUTO, cpu, CTLFLAG_RD, &cpu_freq, 0, "CPU clock frequency"); static u_int itc_freq; SYSCTL_UINT(_hw_freq, OID_AUTO, itc, CTLFLAG_RD, &itc_freq, 0, "ITC frequency"); int cold = 1; u_int64_t pa_bootinfo; struct bootinfo bootinfo; struct pcpu pcpu0; extern u_int64_t kernel_text[], _end[]; extern u_int64_t ia64_gateway_page[]; extern u_int64_t break_sigtramp[]; extern u_int64_t epc_sigtramp[]; struct fpswa_iface *fpswa_iface; u_int64_t ia64_pal_base; u_int64_t ia64_port_base; u_int64_t ia64_lapic_addr = PAL_PIB_DEFAULT_ADDR; struct ia64_pib *ia64_pib; static int ia64_sync_icache_needed; char machine[] = MACHINE; SYSCTL_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD, machine, 0, ""); static char cpu_model[64]; SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD, cpu_model, 0, "The CPU model name"); static char cpu_family[64]; SYSCTL_STRING(_hw, OID_AUTO, family, CTLFLAG_RD, cpu_family, 0, "The CPU family name"); #ifdef DDB extern vm_offset_t ksym_start, ksym_end; #endif struct msgbuf *msgbufp = NULL; /* Other subsystems (e.g., ACPI) can hook this later. */ void (*cpu_idle_hook)(void) = NULL; long Maxmem = 0; long realmem = 0; #define PHYSMAP_SIZE (2 * VM_PHYSSEG_MAX) vm_paddr_t phys_avail[PHYSMAP_SIZE + 2]; /* must be 2 less so 0 0 can signal end of chunks */ #define PHYS_AVAIL_ARRAY_END ((sizeof(phys_avail) / sizeof(vm_offset_t)) - 2) struct kva_md_info kmi; #define Mhz 1000000L #define Ghz (1000L*Mhz) static void identifycpu(void) { char vendor[17]; char *family_name, *model_name; u_int64_t features, tmp; int number, revision, model, family, archrev; /* * Assumes little-endian. */ *(u_int64_t *) &vendor[0] = ia64_get_cpuid(0); *(u_int64_t *) &vendor[8] = ia64_get_cpuid(1); vendor[16] = '\0'; tmp = ia64_get_cpuid(3); number = (tmp >> 0) & 0xff; revision = (tmp >> 8) & 0xff; model = (tmp >> 16) & 0xff; family = (tmp >> 24) & 0xff; archrev = (tmp >> 32) & 0xff; family_name = model_name = "unknown"; switch (family) { case 0x07: family_name = "Itanium"; model_name = "Merced"; break; case 0x1f: family_name = "Itanium 2"; switch (model) { case 0x00: model_name = "McKinley"; break; case 0x01: /* * Deerfield is a low-voltage variant based on the * Madison core. We need circumstantial evidence * (i.e. the clock frequency) to identify those. * Allow for roughly 1% error margin. */ if (cpu_freq > 990 && cpu_freq < 1010) model_name = "Deerfield"; else model_name = "Madison"; break; case 0x02: model_name = "Madison II"; break; } break; case 0x20: ia64_sync_icache_needed = 1; family_name = "Itanium 2"; switch (model) { case 0x00: model_name = "Montecito"; break; } break; } snprintf(cpu_family, sizeof(cpu_family), "%s", family_name); snprintf(cpu_model, sizeof(cpu_model), "%s", model_name); features = ia64_get_cpuid(4); printf("CPU: %s (", model_name); if (cpu_freq) printf("%u Mhz ", cpu_freq); printf("%s)\n", family_name); printf(" Origin = \"%s\" Revision = %d\n", vendor, revision); printf(" Features = 0x%b\n", (u_int32_t) features, "\020" "\001LB" /* long branch (brl) instruction. */ "\002SD" /* Spontaneous deferral. */ "\003AO" /* 16-byte atomic operations (ld, st, cmpxchg). */ ); } static void cpu_startup(void *dummy) { char nodename[16]; struct pcpu *pc; struct pcpu_stats *pcs; /* * Good {morning,afternoon,evening,night}. */ identifycpu(); #ifdef PERFMON perfmon_init(); #endif printf("real memory = %ld (%ld MB)\n", ia64_ptob(Maxmem), ia64_ptob(Maxmem) / 1048576); realmem = Maxmem; /* * Display any holes after the first chunk of extended memory. */ if (bootverbose) { int indx; printf("Physical memory chunk(s):\n"); for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) { long size1 = phys_avail[indx + 1] - phys_avail[indx]; printf("0x%08lx - 0x%08lx, %ld bytes (%ld pages)\n", phys_avail[indx], phys_avail[indx + 1] - 1, size1, size1 >> PAGE_SHIFT); } } vm_ksubmap_init(&kmi); printf("avail memory = %ld (%ld MB)\n", ptoa(cnt.v_free_count), ptoa(cnt.v_free_count) / 1048576); if (fpswa_iface == NULL) printf("Warning: no FPSWA package supplied\n"); else printf("FPSWA Revision = 0x%lx, Entry = %p\n", (long)fpswa_iface->if_rev, (void *)fpswa_iface->if_fpswa); /* * Set up buffers, so they can be used to read disk labels. */ bufinit(); vm_pager_bufferinit(); /* * Traverse the MADT to discover IOSAPIC and Local SAPIC * information. */ ia64_probe_sapics(); ia64_pib = pmap_mapdev(ia64_lapic_addr, sizeof(*ia64_pib)); ia64_mca_init(); /* * Create sysctl tree for per-CPU information. */ SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { snprintf(nodename, sizeof(nodename), "%u", pc->pc_cpuid); sysctl_ctx_init(&pc->pc_md.sysctl_ctx); pc->pc_md.sysctl_tree = SYSCTL_ADD_NODE(&pc->pc_md.sysctl_ctx, SYSCTL_STATIC_CHILDREN(_machdep_cpu), OID_AUTO, nodename, CTLFLAG_RD, NULL, ""); if (pc->pc_md.sysctl_tree == NULL) continue; pcs = &pc->pc_md.stats; SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nasts", CTLFLAG_RD, &pcs->pcs_nasts, "Number of IPI_AST interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nclks", CTLFLAG_RD, &pcs->pcs_nclks, "Number of clock interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nextints", CTLFLAG_RD, &pcs->pcs_nextints, "Number of ExtINT interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nhighfps", CTLFLAG_RD, &pcs->pcs_nhighfps, "Number of IPI_HIGH_FP interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nhwints", CTLFLAG_RD, &pcs->pcs_nhwints, "Number of hardware (device) interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "npreempts", CTLFLAG_RD, &pcs->pcs_npreempts, "Number of IPI_PREEMPT interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nrdvs", CTLFLAG_RD, &pcs->pcs_nrdvs, "Number of IPI_RENDEZVOUS interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nstops", CTLFLAG_RD, &pcs->pcs_nstops, "Number of IPI_STOP interrupts"); SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nstrays", CTLFLAG_RD, &pcs->pcs_nstrays, - "Number of stray vectors"); + "Number of stray interrupts"); } } SYSINIT(cpu_startup, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); void cpu_flush_dcache(void *ptr, size_t len) { vm_offset_t lim, va; va = (uintptr_t)ptr & ~31; lim = (uintptr_t)ptr + len; while (va < lim) { ia64_fc(va); va += 32; } ia64_srlz_d(); } /* Get current clock frequency for the given cpu id. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { if (pcpu_find(cpu_id) == NULL || rate == NULL) return (EINVAL); *rate = (u_long)cpu_freq * 1000000ul; return (0); } void cpu_halt() { efi_reset_system(); } void cpu_idle(int busy) { struct ia64_pal_result res; if (cpu_idle_hook != NULL) (*cpu_idle_hook)(); else res = ia64_call_pal_static(PAL_HALT_LIGHT, 0, 0, 0); } int cpu_idle_wakeup(int cpu) { return (0); } void cpu_reset() { efi_reset_system(); } void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) { struct pcb *oldpcb, *newpcb; oldpcb = old->td_pcb; #ifdef COMPAT_FREEBSD32 ia32_savectx(oldpcb); #endif if (PCPU_GET(fpcurthread) == old) old->td_frame->tf_special.psr |= IA64_PSR_DFH; if (!savectx(oldpcb)) { atomic_store_rel_ptr(&old->td_lock, mtx); newpcb = new->td_pcb; oldpcb->pcb_current_pmap = pmap_switch(newpcb->pcb_current_pmap); #if defined(SCHED_ULE) && defined(SMP) while (atomic_load_acq_ptr(&new->td_lock) == &blocked_lock) cpu_spinwait(); #endif PCPU_SET(curthread, new); #ifdef COMPAT_FREEBSD32 ia32_restorectx(newpcb); #endif if (PCPU_GET(fpcurthread) == new) new->td_frame->tf_special.psr &= ~IA64_PSR_DFH; restorectx(newpcb); /* We should not get here. */ panic("cpu_switch: restorectx() returned"); /* NOTREACHED */ } } void cpu_throw(struct thread *old __unused, struct thread *new) { struct pcb *newpcb; newpcb = new->td_pcb; (void)pmap_switch(newpcb->pcb_current_pmap); #if defined(SCHED_ULE) && defined(SMP) while (atomic_load_acq_ptr(&new->td_lock) == &blocked_lock) cpu_spinwait(); #endif PCPU_SET(curthread, new); #ifdef COMPAT_FREEBSD32 ia32_restorectx(newpcb); #endif restorectx(newpcb); /* We should not get here. */ panic("cpu_throw: restorectx() returned"); /* NOTREACHED */ } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { /* * Set pc_acpi_id to "uninitialized". * See sys/dev/acpica/acpi_cpu.c */ pcpu->pc_acpi_id = 0xffffffff; } void spinlock_enter(void) { struct thread *td; td = curthread; if (td->td_md.md_spinlock_count == 0) td->td_md.md_saved_intr = intr_disable(); td->td_md.md_spinlock_count++; critical_enter(); } void spinlock_exit(void) { struct thread *td; td = curthread; critical_exit(); td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) intr_restore(td->td_md.md_saved_intr); } void map_vhpt(uintptr_t vhpt) { pt_entry_t pte; uint64_t psr; pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RW; pte |= vhpt & PTE_PPN_MASK; __asm __volatile("ptr.d %0,%1" :: "r"(vhpt), "r"(IA64_ID_PAGE_SHIFT<<2)); __asm __volatile("mov %0=psr" : "=r"(psr)); __asm __volatile("rsm psr.ic|psr.i"); ia64_srlz_i(); ia64_set_ifa(vhpt); ia64_set_itir(IA64_ID_PAGE_SHIFT << 2); ia64_srlz_d(); __asm __volatile("itr.d dtr[%0]=%1" :: "r"(2), "r"(pte)); __asm __volatile("mov psr.l=%0" :: "r" (psr)); ia64_srlz_i(); } void map_pal_code(void) { pt_entry_t pte; uint64_t psr; if (ia64_pal_base == 0) return; pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RWX; pte |= ia64_pal_base & PTE_PPN_MASK; __asm __volatile("ptr.d %0,%1; ptr.i %0,%1" :: "r"(IA64_PHYS_TO_RR7(ia64_pal_base)), "r"(IA64_ID_PAGE_SHIFT<<2)); __asm __volatile("mov %0=psr" : "=r"(psr)); __asm __volatile("rsm psr.ic|psr.i"); ia64_srlz_i(); ia64_set_ifa(IA64_PHYS_TO_RR7(ia64_pal_base)); ia64_set_itir(IA64_ID_PAGE_SHIFT << 2); ia64_srlz_d(); __asm __volatile("itr.d dtr[%0]=%1" :: "r"(1), "r"(pte)); ia64_srlz_d(); __asm __volatile("itr.i itr[%0]=%1" :: "r"(1), "r"(pte)); __asm __volatile("mov psr.l=%0" :: "r" (psr)); ia64_srlz_i(); } void map_gateway_page(void) { pt_entry_t pte; uint64_t psr; pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_X_RX; pte |= (uint64_t)ia64_gateway_page & PTE_PPN_MASK; __asm __volatile("ptr.d %0,%1; ptr.i %0,%1" :: "r"(VM_MAX_ADDRESS), "r"(PAGE_SHIFT << 2)); __asm __volatile("mov %0=psr" : "=r"(psr)); __asm __volatile("rsm psr.ic|psr.i"); ia64_srlz_i(); ia64_set_ifa(VM_MAX_ADDRESS); ia64_set_itir(PAGE_SHIFT << 2); ia64_srlz_d(); __asm __volatile("itr.d dtr[%0]=%1" :: "r"(3), "r"(pte)); ia64_srlz_d(); __asm __volatile("itr.i itr[%0]=%1" :: "r"(3), "r"(pte)); __asm __volatile("mov psr.l=%0" :: "r" (psr)); ia64_srlz_i(); /* Expose the mapping to userland in ar.k5 */ ia64_set_k5(VM_MAX_ADDRESS); } static u_int freq_ratio(u_long base, u_long ratio) { u_long f; f = (base * (ratio >> 32)) / (ratio & 0xfffffffful); return ((f + 500000) / 1000000); } static void calculate_frequencies(void) { struct ia64_sal_result sal; struct ia64_pal_result pal; sal = ia64_sal_entry(SAL_FREQ_BASE, 0, 0, 0, 0, 0, 0, 0); pal = ia64_call_pal_static(PAL_FREQ_RATIOS, 0, 0, 0); if (sal.sal_status == 0 && pal.pal_status == 0) { if (bootverbose) { printf("Platform clock frequency %ld Hz\n", sal.sal_result[0]); printf("Processor ratio %ld/%ld, Bus ratio %ld/%ld, " "ITC ratio %ld/%ld\n", pal.pal_result[0] >> 32, pal.pal_result[0] & ((1L << 32) - 1), pal.pal_result[1] >> 32, pal.pal_result[1] & ((1L << 32) - 1), pal.pal_result[2] >> 32, pal.pal_result[2] & ((1L << 32) - 1)); } cpu_freq = freq_ratio(sal.sal_result[0], pal.pal_result[0]); bus_freq = freq_ratio(sal.sal_result[0], pal.pal_result[1]); itc_freq = freq_ratio(sal.sal_result[0], pal.pal_result[2]); } } struct ia64_init_return ia64_init(void) { struct ia64_init_return ret; int phys_avail_cnt; vm_offset_t kernstart, kernend; vm_offset_t kernstartpfn, kernendpfn, pfn0, pfn1; char *p; struct efi_md *md; int metadata_missing; /* NO OUTPUT ALLOWED UNTIL FURTHER NOTICE */ /* * TODO: Disable interrupts, floating point etc. * Maybe flush cache and tlb */ ia64_set_fpsr(IA64_FPSR_DEFAULT); /* * TODO: Get critical system information (if possible, from the * information provided by the boot program). */ /* * pa_bootinfo is the physical address of the bootinfo block as * passed to us by the loader and set in locore.s. */ bootinfo = *(struct bootinfo *)(IA64_PHYS_TO_RR7(pa_bootinfo)); if (bootinfo.bi_magic != BOOTINFO_MAGIC || bootinfo.bi_version != 1) { bzero(&bootinfo, sizeof(bootinfo)); bootinfo.bi_kernend = (vm_offset_t) round_page(_end); } /* * Look for the I/O ports first - we need them for console * probing. */ for (md = efi_md_first(); md != NULL; md = efi_md_next(md)) { switch (md->md_type) { case EFI_MD_TYPE_IOPORT: ia64_port_base = (uintptr_t)pmap_mapdev(md->md_phys, md->md_pages * EFI_PAGE_SIZE); break; case EFI_MD_TYPE_PALCODE: ia64_pal_base = md->md_phys; break; } } metadata_missing = 0; if (bootinfo.bi_modulep) preload_metadata = (caddr_t)bootinfo.bi_modulep; else metadata_missing = 1; if (envmode == 0 && bootinfo.bi_envp) kern_envp = (caddr_t)bootinfo.bi_envp; else kern_envp = static_env; /* * Look at arguments passed to us and compute boothowto. */ boothowto = bootinfo.bi_boothowto; /* * Catch case of boot_verbose set in environment. */ if ((p = getenv("boot_verbose")) != NULL) { if (strcmp(p, "yes") == 0 || strcmp(p, "YES") == 0) { boothowto |= RB_VERBOSE; } freeenv(p); } if (boothowto & RB_VERBOSE) bootverbose = 1; /* * Find the beginning and end of the kernel. */ kernstart = trunc_page(kernel_text); #ifdef DDB ksym_start = bootinfo.bi_symtab; ksym_end = bootinfo.bi_esymtab; kernend = (vm_offset_t)round_page(ksym_end); #else kernend = (vm_offset_t)round_page(_end); #endif /* But if the bootstrap tells us otherwise, believe it! */ if (bootinfo.bi_kernend) kernend = round_page(bootinfo.bi_kernend); /* * Setup the PCPU data for the bootstrap processor. It is needed * by printf(). Also, since printf() has critical sections, we * need to initialize at least pc_curthread. */ pcpup = &pcpu0; ia64_set_k4((u_int64_t)pcpup); pcpu_init(pcpup, 0, sizeof(pcpu0)); dpcpu_init((void *)kernend, 0); kernend += DPCPU_SIZE; PCPU_SET(curthread, &thread0); /* * Initialize the console before we print anything out. */ cninit(); /* OUTPUT NOW ALLOWED */ if (ia64_pal_base != 0) { ia64_pal_base &= ~IA64_ID_PAGE_MASK; /* * We use a TR to map the first 256M of memory - this might * cover the palcode too. */ if (ia64_pal_base == 0) printf("PAL code mapped by the kernel's TR\n"); } else printf("PAL code not found\n"); /* * Wire things up so we can call the firmware. */ map_pal_code(); efi_boot_minimal(bootinfo.bi_systab); + ia64_xiv_init(); ia64_sal_init(); calculate_frequencies(); if (metadata_missing) printf("WARNING: loader(8) metadata is missing!\n"); /* Get FPSWA interface */ fpswa_iface = (bootinfo.bi_fpswa == 0) ? NULL : (struct fpswa_iface *)IA64_PHYS_TO_RR7(bootinfo.bi_fpswa); /* Init basic tunables, including hz */ init_param1(); p = getenv("kernelname"); if (p) { strncpy(kernelname, p, sizeof(kernelname) - 1); freeenv(p); } kernstartpfn = atop(IA64_RR_MASK(kernstart)); kernendpfn = atop(IA64_RR_MASK(kernend)); /* * Size the memory regions and load phys_avail[] with the results. */ /* * Find out how much memory is available, by looking at * the memory descriptors. */ #ifdef DEBUG_MD printf("Memory descriptor count: %d\n", mdcount); #endif phys_avail_cnt = 0; for (md = efi_md_first(); md != NULL; md = efi_md_next(md)) { #ifdef DEBUG_MD printf("MD %p: type %d pa 0x%lx cnt 0x%lx\n", md, md->md_type, md->md_phys, md->md_pages); #endif pfn0 = ia64_btop(round_page(md->md_phys)); pfn1 = ia64_btop(trunc_page(md->md_phys + md->md_pages * 4096)); if (pfn1 <= pfn0) continue; if (md->md_type != EFI_MD_TYPE_FREE) continue; /* * We have a memory descriptor that describes conventional * memory that is for general use. We must determine if the * loader has put the kernel in this region. */ physmem += (pfn1 - pfn0); if (pfn0 <= kernendpfn && kernstartpfn <= pfn1) { /* * Must compute the location of the kernel * within the segment. */ #ifdef DEBUG_MD printf("Descriptor %p contains kernel\n", mp); #endif if (pfn0 < kernstartpfn) { /* * There is a chunk before the kernel. */ #ifdef DEBUG_MD printf("Loading chunk before kernel: " "0x%lx / 0x%lx\n", pfn0, kernstartpfn); #endif phys_avail[phys_avail_cnt] = ia64_ptob(pfn0); phys_avail[phys_avail_cnt+1] = ia64_ptob(kernstartpfn); phys_avail_cnt += 2; } if (kernendpfn < pfn1) { /* * There is a chunk after the kernel. */ #ifdef DEBUG_MD printf("Loading chunk after kernel: " "0x%lx / 0x%lx\n", kernendpfn, pfn1); #endif phys_avail[phys_avail_cnt] = ia64_ptob(kernendpfn); phys_avail[phys_avail_cnt+1] = ia64_ptob(pfn1); phys_avail_cnt += 2; } } else { /* * Just load this cluster as one chunk. */ #ifdef DEBUG_MD printf("Loading descriptor %d: 0x%lx / 0x%lx\n", i, pfn0, pfn1); #endif phys_avail[phys_avail_cnt] = ia64_ptob(pfn0); phys_avail[phys_avail_cnt+1] = ia64_ptob(pfn1); phys_avail_cnt += 2; } } phys_avail[phys_avail_cnt] = 0; Maxmem = physmem; init_param2(physmem); /* * Initialize error message buffer (at end of core). */ msgbufp = (struct msgbuf *)pmap_steal_memory(MSGBUF_SIZE); msgbufinit(msgbufp, MSGBUF_SIZE); proc_linkup0(&proc0, &thread0); /* * Init mapping for kernel stack for proc 0 */ thread0.td_kstack = pmap_steal_memory(KSTACK_PAGES * PAGE_SIZE); thread0.td_kstack_pages = KSTACK_PAGES; mutex_init(); /* * Initialize the rest of proc 0's PCB. * * Set the kernel sp, reserving space for an (empty) trapframe, * and make proc0's trapframe pointer point to it for sanity. * Initialise proc0's backing store to start after u area. */ cpu_thread_alloc(&thread0); thread0.td_frame->tf_flags = FRAME_SYSCALL; thread0.td_pcb->pcb_special.sp = (u_int64_t)thread0.td_frame - 16; thread0.td_pcb->pcb_special.bspstore = thread0.td_kstack; /* * Initialize the virtual memory system. */ pmap_bootstrap(); /* * Initialize debuggers, and break into them if appropriate. */ kdb_init(); #ifdef KDB if (boothowto & RB_KDB) kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger\n"); #endif ia64_set_tpr(0); ia64_srlz_d(); ret.bspstore = thread0.td_pcb->pcb_special.bspstore; ret.sp = thread0.td_pcb->pcb_special.sp; return (ret); } uint64_t ia64_get_hcdp(void) { return (bootinfo.bi_hcdp); } void bzero(void *buf, size_t len) { caddr_t p = buf; while (((vm_offset_t) p & (sizeof(u_long) - 1)) && len) { *p++ = 0; len--; } while (len >= sizeof(u_long) * 8) { *(u_long*) p = 0; *((u_long*) p + 1) = 0; *((u_long*) p + 2) = 0; *((u_long*) p + 3) = 0; len -= sizeof(u_long) * 8; *((u_long*) p + 4) = 0; *((u_long*) p + 5) = 0; *((u_long*) p + 6) = 0; *((u_long*) p + 7) = 0; p += sizeof(u_long) * 8; } while (len >= sizeof(u_long)) { *(u_long*) p = 0; len -= sizeof(u_long); p += sizeof(u_long); } while (len) { *p++ = 0; len--; } } u_int ia64_itc_freq(void) { return (itc_freq); } void DELAY(int n) { u_int64_t start, end, now; sched_pin(); start = ia64_get_itc(); end = start + itc_freq * n; /* printf("DELAY from 0x%lx to 0x%lx\n", start, end); */ do { now = ia64_get_itc(); } while (now < end || (now > start && end < start)); sched_unpin(); } /* * Send an interrupt (signal) to a process. */ void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { struct proc *p; struct thread *td; struct trapframe *tf; struct sigacts *psp; struct sigframe sf, *sfp; u_int64_t sbs, sp; int oonstack; int sig; u_long code; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; code = ksi->ksi_code; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; sp = tf->tf_special.sp; oonstack = sigonstack(sp); sbs = 0; /* save user context */ bzero(&sf, sizeof(struct sigframe)); sf.sf_uc.uc_sigmask = *mask; sf.sf_uc.uc_stack = td->td_sigstk; sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE; /* * Allocate and validate space for the signal handler * context. Note that if the stack is in P0 space, the * call to grow() is a nop, and the useracc() check * will fail if the process has not already allocated * the space with a `brk'. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { sbs = (u_int64_t)td->td_sigstk.ss_sp; sbs = (sbs + 15) & ~15; sfp = (struct sigframe *)(sbs + td->td_sigstk.ss_size); #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else sfp = (struct sigframe *)sp; sfp = (struct sigframe *)((u_int64_t)(sfp - 1) & ~15); /* Fill in the siginfo structure for POSIX handlers. */ if (SIGISMEMBER(psp->ps_siginfo, sig)) { sf.sf_si = ksi->ksi_info; sf.sf_si.si_signo = sig; /* * XXX this shouldn't be here after code in trap.c * is fixed */ sf.sf_si.si_addr = (void*)tf->tf_special.ifa; code = (u_int64_t)&sfp->sf_si; } mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(p); get_mcontext(td, &sf.sf_uc.uc_mcontext, 0); /* Copy the frame out to userland. */ if (copyout(&sf, sfp, sizeof(sf)) != 0) { /* * Process has trashed its stack; give it an illegal * instruction to halt it in its tracks. */ PROC_LOCK(p); sigexit(td, SIGILL); return; } if ((tf->tf_flags & FRAME_SYSCALL) == 0) { tf->tf_special.psr &= ~IA64_PSR_RI; tf->tf_special.iip = ia64_get_k5() + ((uint64_t)break_sigtramp - (uint64_t)ia64_gateway_page); } else tf->tf_special.iip = ia64_get_k5() + ((uint64_t)epc_sigtramp - (uint64_t)ia64_gateway_page); /* * Setup the trapframe to return to the signal trampoline. We pass * information to the trampoline in the following registers: * * gp new backing store or NULL * r8 signal number * r9 signal code or siginfo pointer * r10 signal handler (function descriptor) */ tf->tf_special.sp = (u_int64_t)sfp - 16; tf->tf_special.gp = sbs; tf->tf_special.bspstore = sf.sf_uc.uc_mcontext.mc_special.bspstore; tf->tf_special.ndirty = 0; tf->tf_special.rnat = sf.sf_uc.uc_mcontext.mc_special.rnat; tf->tf_scratch.gr8 = sig; tf->tf_scratch.gr9 = code; tf->tf_scratch.gr10 = (u_int64_t)catcher; PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } /* * System call to cleanup state after a signal * has been taken. Reset signal mask and * stack state from context left by sendsig (above). * Return to previous pc and psl as specified by * context left by sendsig. Check carefully to * make sure that the user has not modified the * state to gain improper privileges. * * MPSAFE */ int sigreturn(struct thread *td, struct sigreturn_args /* { ucontext_t *sigcntxp; } */ *uap) { ucontext_t uc; struct trapframe *tf; struct pcb *pcb; tf = td->td_frame; pcb = td->td_pcb; /* * Fetch the entire context structure at once for speed. * We don't use a normal argument to simplify RSE handling. */ if (copyin(uap->sigcntxp, (caddr_t)&uc, sizeof(uc))) return (EFAULT); set_mcontext(td, &uc.uc_mcontext); #if defined(COMPAT_43) if (sigonstack(tf->tf_special.sp)) td->td_sigstk.ss_flags |= SS_ONSTACK; else td->td_sigstk.ss_flags &= ~SS_ONSTACK; #endif kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } #ifdef COMPAT_FREEBSD4 int freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap) { return sigreturn(td, (struct sigreturn_args *)uap); } #endif /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { pcb->pcb_special = tf->tf_special; pcb->pcb_special.__spare = ~0UL; /* XXX see unwind.c */ save_callee_saved(&pcb->pcb_preserved); save_callee_saved_fp(&pcb->pcb_preserved_fp); } int ia64_flush_dirty(struct thread *td, struct _special *r) { struct iovec iov; struct uio uio; uint64_t bspst, kstk, rnat; int error, locked; if (r->ndirty == 0) return (0); kstk = td->td_kstack + (r->bspstore & 0x1ffUL); if (td == curthread) { __asm __volatile("mov ar.rsc=0;;"); __asm __volatile("mov %0=ar.bspstore" : "=r"(bspst)); /* Make sure we have all the user registers written out. */ if (bspst - kstk < r->ndirty) { __asm __volatile("flushrs;;"); __asm __volatile("mov %0=ar.bspstore" : "=r"(bspst)); } __asm __volatile("mov %0=ar.rnat;;" : "=r"(rnat)); __asm __volatile("mov ar.rsc=3"); error = copyout((void*)kstk, (void*)r->bspstore, r->ndirty); kstk += r->ndirty; r->rnat = (bspst > kstk && (bspst & 0x1ffL) < (kstk & 0x1ffL)) ? *(uint64_t*)(kstk | 0x1f8L) : rnat; } else { locked = PROC_LOCKED(td->td_proc); if (!locked) PHOLD(td->td_proc); iov.iov_base = (void*)(uintptr_t)kstk; iov.iov_len = r->ndirty; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = r->bspstore; uio.uio_resid = r->ndirty; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; uio.uio_td = td; error = proc_rwmem(td->td_proc, &uio); /* * XXX proc_rwmem() doesn't currently return ENOSPC, * so I think it can bogusly return 0. Neither do * we allow short writes. */ if (uio.uio_resid != 0 && error == 0) error = ENOSPC; if (!locked) PRELE(td->td_proc); } r->bspstore += r->ndirty; r->ndirty = 0; return (error); } int get_mcontext(struct thread *td, mcontext_t *mc, int flags) { struct trapframe *tf; int error; tf = td->td_frame; bzero(mc, sizeof(*mc)); mc->mc_special = tf->tf_special; error = ia64_flush_dirty(td, &mc->mc_special); if (tf->tf_flags & FRAME_SYSCALL) { mc->mc_flags |= _MC_FLAGS_SYSCALL_CONTEXT; mc->mc_scratch = tf->tf_scratch; if (flags & GET_MC_CLEAR_RET) { mc->mc_scratch.gr8 = 0; mc->mc_scratch.gr9 = 0; mc->mc_scratch.gr10 = 0; mc->mc_scratch.gr11 = 0; } } else { mc->mc_flags |= _MC_FLAGS_ASYNC_CONTEXT; mc->mc_scratch = tf->tf_scratch; mc->mc_scratch_fp = tf->tf_scratch_fp; /* * XXX If the thread never used the high FP registers, we * probably shouldn't waste time saving them. */ ia64_highfp_save(td); mc->mc_flags |= _MC_FLAGS_HIGHFP_VALID; mc->mc_high_fp = td->td_pcb->pcb_high_fp; } save_callee_saved(&mc->mc_preserved); save_callee_saved_fp(&mc->mc_preserved_fp); return (error); } int set_mcontext(struct thread *td, const mcontext_t *mc) { struct _special s; struct trapframe *tf; uint64_t psrmask; tf = td->td_frame; KASSERT((tf->tf_special.ndirty & ~PAGE_MASK) == 0, ("Whoa there! We have more than 8KB of dirty registers!")); s = mc->mc_special; /* * Only copy the user mask and the restart instruction bit from * the new context. */ psrmask = IA64_PSR_BE | IA64_PSR_UP | IA64_PSR_AC | IA64_PSR_MFL | IA64_PSR_MFH | IA64_PSR_RI; s.psr = (tf->tf_special.psr & ~psrmask) | (s.psr & psrmask); /* We don't have any dirty registers of the new context. */ s.ndirty = 0; if (mc->mc_flags & _MC_FLAGS_ASYNC_CONTEXT) { /* * We can get an async context passed to us while we * entered the kernel through a syscall: sigreturn(2) * takes contexts that could previously be the result of * a trap or interrupt. * Hence, we cannot assert that the trapframe is not * a syscall frame, but we can assert that it's at * least an expected syscall. */ if (tf->tf_flags & FRAME_SYSCALL) { KASSERT(tf->tf_scratch.gr15 == SYS_sigreturn, ("foo")); tf->tf_flags &= ~FRAME_SYSCALL; } tf->tf_scratch = mc->mc_scratch; tf->tf_scratch_fp = mc->mc_scratch_fp; if (mc->mc_flags & _MC_FLAGS_HIGHFP_VALID) td->td_pcb->pcb_high_fp = mc->mc_high_fp; } else { KASSERT((tf->tf_flags & FRAME_SYSCALL) != 0, ("foo")); if ((mc->mc_flags & _MC_FLAGS_SYSCALL_CONTEXT) == 0) { s.cfm = tf->tf_special.cfm; s.iip = tf->tf_special.iip; tf->tf_scratch.gr15 = 0; /* Clear syscall nr. */ } else tf->tf_scratch = mc->mc_scratch; } tf->tf_special = s; restore_callee_saved(&mc->mc_preserved); restore_callee_saved_fp(&mc->mc_preserved_fp); return (0); } /* * Clear registers on exec. */ void exec_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) { struct trapframe *tf; uint64_t *ksttop, *kst; tf = td->td_frame; ksttop = (uint64_t*)(td->td_kstack + tf->tf_special.ndirty + (tf->tf_special.bspstore & 0x1ffUL)); /* * We can ignore up to 8KB of dirty registers by masking off the * lower 13 bits in exception_restore() or epc_syscall(). This * should be enough for a couple of years, but if there are more * than 8KB of dirty registers, we lose track of the bottom of * the kernel stack. The solution is to copy the active part of * the kernel stack down 1 page (or 2, but not more than that) * so that we always have less than 8KB of dirty registers. */ KASSERT((tf->tf_special.ndirty & ~PAGE_MASK) == 0, ("Whoa there! We have more than 8KB of dirty registers!")); bzero(&tf->tf_special, sizeof(tf->tf_special)); if ((tf->tf_flags & FRAME_SYSCALL) == 0) { /* break syscalls. */ bzero(&tf->tf_scratch, sizeof(tf->tf_scratch)); bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp)); tf->tf_special.cfm = (1UL<<63) | (3UL<<7) | 3UL; tf->tf_special.bspstore = IA64_BACKINGSTORE; /* * Copy the arguments onto the kernel register stack so that * they get loaded by the loadrs instruction. Skip over the * NaT collection points. */ kst = ksttop - 1; if (((uintptr_t)kst & 0x1ff) == 0x1f8) *kst-- = 0; *kst-- = 0; if (((uintptr_t)kst & 0x1ff) == 0x1f8) *kst-- = 0; *kst-- = ps_strings; if (((uintptr_t)kst & 0x1ff) == 0x1f8) *kst-- = 0; *kst = stack; tf->tf_special.ndirty = (ksttop - kst) << 3; } else { /* epc syscalls (default). */ tf->tf_special.cfm = (3UL<<62) | (3UL<<7) | 3UL; tf->tf_special.bspstore = IA64_BACKINGSTORE + 24; /* * Write values for out0, out1 and out2 to the user's backing * store and arrange for them to be restored into the user's * initial register frame. * Assumes that (bspstore & 0x1f8) < 0x1e0. */ suword((caddr_t)tf->tf_special.bspstore - 24, stack); suword((caddr_t)tf->tf_special.bspstore - 16, ps_strings); suword((caddr_t)tf->tf_special.bspstore - 8, 0); } tf->tf_special.iip = entry; tf->tf_special.sp = (stack & ~15) - 16; tf->tf_special.rsc = 0xf; tf->tf_special.fpsr = IA64_FPSR_DEFAULT; tf->tf_special.psr = IA64_PSR_IC | IA64_PSR_I | IA64_PSR_IT | IA64_PSR_DT | IA64_PSR_RT | IA64_PSR_DFH | IA64_PSR_BN | IA64_PSR_CPL_USER; } int ptrace_set_pc(struct thread *td, unsigned long addr) { uint64_t slot; switch (addr & 0xFUL) { case 0: slot = IA64_PSR_RI_0; break; case 1: /* XXX we need to deal with MLX bundles here */ slot = IA64_PSR_RI_1; break; case 2: slot = IA64_PSR_RI_2; break; default: return (EINVAL); } td->td_frame->tf_special.iip = addr & ~0x0FULL; td->td_frame->tf_special.psr = (td->td_frame->tf_special.psr & ~IA64_PSR_RI) | slot; return (0); } int ptrace_single_step(struct thread *td) { struct trapframe *tf; /* * There's no way to set single stepping when we're leaving the * kernel through the EPC syscall path. The way we solve this is * by enabling the lower-privilege trap so that we re-enter the * kernel as soon as the privilege level changes. See trap.c for * how we proceed from there. */ tf = td->td_frame; if (tf->tf_flags & FRAME_SYSCALL) tf->tf_special.psr |= IA64_PSR_LP; else tf->tf_special.psr |= IA64_PSR_SS; return (0); } int ptrace_clear_single_step(struct thread *td) { struct trapframe *tf; /* * Clear any and all status bits we may use to implement single * stepping. */ tf = td->td_frame; tf->tf_special.psr &= ~IA64_PSR_SS; tf->tf_special.psr &= ~IA64_PSR_LP; tf->tf_special.psr &= ~IA64_PSR_TB; return (0); } int fill_regs(struct thread *td, struct reg *regs) { struct trapframe *tf; tf = td->td_frame; regs->r_special = tf->tf_special; regs->r_scratch = tf->tf_scratch; save_callee_saved(®s->r_preserved); return (0); } int set_regs(struct thread *td, struct reg *regs) { struct trapframe *tf; int error; tf = td->td_frame; error = ia64_flush_dirty(td, &tf->tf_special); if (!error) { tf->tf_special = regs->r_special; tf->tf_special.bspstore += tf->tf_special.ndirty; tf->tf_special.ndirty = 0; tf->tf_scratch = regs->r_scratch; restore_callee_saved(®s->r_preserved); } return (error); } int fill_dbregs(struct thread *td, struct dbreg *dbregs) { return (ENOSYS); } int set_dbregs(struct thread *td, struct dbreg *dbregs) { return (ENOSYS); } int fill_fpregs(struct thread *td, struct fpreg *fpregs) { struct trapframe *frame = td->td_frame; struct pcb *pcb = td->td_pcb; /* Save the high FP registers. */ ia64_highfp_save(td); fpregs->fpr_scratch = frame->tf_scratch_fp; save_callee_saved_fp(&fpregs->fpr_preserved); fpregs->fpr_high = pcb->pcb_high_fp; return (0); } int set_fpregs(struct thread *td, struct fpreg *fpregs) { struct trapframe *frame = td->td_frame; struct pcb *pcb = td->td_pcb; /* Throw away the high FP registers (should be redundant). */ ia64_highfp_drop(td); frame->tf_scratch_fp = fpregs->fpr_scratch; restore_callee_saved_fp(&fpregs->fpr_preserved); pcb->pcb_high_fp = fpregs->fpr_high; return (0); } void ia64_sync_icache(vm_offset_t va, vm_offset_t sz) { vm_offset_t lim; if (!ia64_sync_icache_needed) return; lim = va + sz; while (va < lim) { ia64_fc_i(va); va += 32; /* XXX */ } ia64_sync_i(); ia64_srlz_i(); } diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c index c5ed48f501d6..b1d32f706c1a 100644 --- a/sys/ia64/ia64/mp_machdep.c +++ b/sys/ia64/ia64/mp_machdep.c @@ -1,380 +1,446 @@ /*- * Copyright (c) 2001-2005 Marcel Moolenaar * Copyright (c) 2000 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include -#include -#include -#include -#include - #include #include #include #include #include #include #include #include -#include #include #include -#include + +#include +#include +#include +#include MALLOC_DEFINE(M_SMP, "SMP", "SMP related allocations"); void ia64_ap_startup(void); #define LID_SAPIC(x) ((u_int)((x) >> 16)) #define LID_SAPIC_ID(x) ((u_int)((x) >> 24) & 0xff) #define LID_SAPIC_EID(x) ((u_int)((x) >> 16) & 0xff) #define LID_SAPIC_SET(id,eid) (((id & 0xff) << 8 | (eid & 0xff)) << 16); #define LID_SAPIC_MASK 0xffff0000UL /* Variables used by os_boot_rendez and ia64_ap_startup */ struct pcpu *ap_pcpu; void *ap_stack; volatile int ap_delay; volatile int ap_awake; volatile int ap_spin; -static void cpu_mp_unleash(void *); +int ia64_ipi_ast; +int ia64_ipi_highfp; +int ia64_ipi_nmi; +int ia64_ipi_preempt; +int ia64_ipi_rndzvs; +int ia64_ipi_stop; + +static u_int +ia64_ih_ast(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nasts); + CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid)); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_highfp(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nhighfps); + ia64_highfp_save_ipi(); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_preempt(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_npreempts); + CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid)); + sched_preempt(curthread); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_rndzvs(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nrdvs); + CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid)); + smp_rendezvous_action(); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_stop(struct thread *td, u_int xiv, struct trapframe *tf) +{ + cpumask_t mybit; + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nstops); + mybit = PCPU_GET(cpumask); + ia64_srlz_d(); + + savectx(PCPU_PTR(md.pcb)); + + atomic_set_int(&stopped_cpus, mybit); + while ((started_cpus & mybit) == 0) + cpu_spinwait(); + atomic_clear_int(&started_cpus, mybit); + atomic_clear_int(&stopped_cpus, mybit); + return (0); +} struct cpu_group * cpu_topo(void) { return smp_topo_none(); } static void ia64_store_mca_state(void* arg) { unsigned int ncpu = (unsigned int)(uintptr_t)arg; struct thread* td; /* ia64_mca_save_state() is CPU-sensitive, so bind ourself to our target CPU */ td = curthread; thread_lock(td); sched_bind(td, ncpu); thread_unlock(td); /* * Get and save the CPU specific MCA records. Should we get the * MCA state for each processor, or just the CMC state? */ ia64_mca_save_state(SAL_INFO_MCA); ia64_mca_save_state(SAL_INFO_CMC); kproc_exit(0); } void ia64_ap_startup(void) { uint64_t vhpt; - int vector; pcpup = ap_pcpu; ia64_set_k4((intptr_t)pcpup); vhpt = PCPU_GET(md.vhpt); map_vhpt(vhpt); ia64_set_pta(vhpt + (1 << 8) + (pmap_vhpt_log2size << 2) + 1); ia64_srlz_i(); ap_awake = 1; ap_delay = 0; map_pal_code(); map_gateway_page(); ia64_set_fpsr(IA64_FPSR_DEFAULT); /* Wait until it's time for us to be unleashed */ while (ap_spin) cpu_spinwait(); /* Initialize curthread. */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); PCPU_SET(curthread, PCPU_GET(idlethread)); atomic_add_int(&ap_awake, 1); while (!smp_started) cpu_spinwait(); CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid)); - /* Acknowledge and EOI all interrupts. */ - vector = ia64_get_ivr(); - while (vector != 15) { - ia64_srlz_d(); - if (vector == 0) - vector = (int)ia64_ld1(&ia64_pib->ib_inta); - ia64_set_eoi(0); - ia64_srlz_d(); - vector = ia64_get_ivr(); - } - ia64_srlz_d(); - /* kick off the clock on this AP */ pcpu_initclock(); ia64_set_tpr(0); ia64_srlz_d(); enable_intr(); sched_throw(NULL); /* NOTREACHED */ } void cpu_mp_setmaxid(void) { /* * Count the number of processors in the system by walking the ACPI * tables. Note that we record the actual number of processors, even * if this is larger than MAXCPU. We only activate MAXCPU processors. */ mp_ncpus = ia64_count_cpus(); /* * Set the largest cpuid we're going to use. This is necessary for * VM initialization. */ mp_maxid = min(mp_ncpus, MAXCPU) - 1; } int cpu_mp_probe(void) { /* * If there's only 1 processor, or we don't have a wake-up vector, * we're not going to enable SMP. Note that no wake-up vector can * also mean that the wake-up mechanism is not supported. In this * case we can have multiple processors, but we simply can't wake * them up... */ - return (mp_ncpus > 1 && ipi_vector[IPI_AP_WAKEUP] != 0); + return (mp_ncpus > 1 && ia64_ipi_wakeup != 0); } void cpu_mp_add(u_int acpiid, u_int apicid, u_int apiceid) { struct pcpu *pc; u_int64_t lid; void *dpcpu; u_int cpuid; lid = LID_SAPIC_SET(apicid, apiceid); cpuid = ((ia64_get_lid() & LID_SAPIC_MASK) == lid) ? 0 : smp_cpus++; KASSERT((all_cpus & (1UL << cpuid)) == 0, ("%s: cpu%d already in CPU map", __func__, acpiid)); if (cpuid != 0) { pc = (struct pcpu *)malloc(sizeof(*pc), M_SMP, M_WAITOK); pcpu_init(pc, cpuid, sizeof(*pc)); dpcpu = (void *)kmem_alloc(kernel_map, DPCPU_SIZE); dpcpu_init(dpcpu, cpuid); } else pc = pcpup; pc->pc_acpi_id = acpiid; pc->pc_md.lid = lid; all_cpus |= (1UL << cpuid); } void cpu_mp_announce() { struct pcpu *pc; int i; for (i = 0; i <= mp_maxid; i++) { pc = pcpu_find(i); if (pc != NULL) { printf("cpu%d: ACPI Id=%x, SAPIC Id=%x, SAPIC Eid=%x", i, pc->pc_acpi_id, LID_SAPIC_ID(pc->pc_md.lid), LID_SAPIC_EID(pc->pc_md.lid)); if (i == 0) printf(" (BSP)\n"); else printf("\n"); } } } void cpu_mp_start() { struct pcpu *pc; ap_spin = 1; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { pc->pc_md.current_pmap = kernel_pmap; pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask; if (pc->pc_cpuid > 0) { ap_pcpu = pc; pc->pc_md.vhpt = pmap_alloc_vhpt(); if (pc->pc_md.vhpt == 0) { printf("SMP: WARNING: unable to allocate VHPT" " for cpu%d", pc->pc_cpuid); continue; } ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_SMP, M_WAITOK); ap_delay = 2000; ap_awake = 0; if (bootverbose) printf("SMP: waking up cpu%d\n", pc->pc_cpuid); - ipi_send(pc, IPI_AP_WAKEUP); + ipi_send(pc, ia64_ipi_wakeup); do { DELAY(1000); } while (--ap_delay > 0); pc->pc_md.awake = ap_awake; if (!ap_awake) printf("SMP: WARNING: cpu%d did not wake up\n", pc->pc_cpuid); } else pc->pc_md.awake = 1; } } static void cpu_mp_unleash(void *dummy) { struct pcpu *pc; int cpus; if (mp_ncpus <= 1) return; + /* Allocate XIVs for IPIs */ + ia64_ipi_ast = ia64_xiv_alloc(PI_DULL, IA64_XIV_IPI, ia64_ih_ast); + ia64_ipi_highfp = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_highfp); + ia64_ipi_preempt = ia64_xiv_alloc(PI_SOFT, IA64_XIV_IPI, + ia64_ih_preempt); + ia64_ipi_rndzvs = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_rndzvs); + ia64_ipi_stop = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI, ia64_ih_stop); + + /* Reserve the NMI vector for IPI_STOP_HARD if possible */ + ia64_ipi_nmi = (ia64_xiv_reserve(2, IA64_XIV_IPI, ia64_ih_stop) != 0) + ? ia64_ipi_stop : 0x400; /* DM=NMI, Vector=n/a */ + cpus = 0; smp_cpus = 0; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { cpus++; if (pc->pc_md.awake) { kproc_create(ia64_store_mca_state, (void*)((uintptr_t)pc->pc_cpuid), NULL, 0, 0, "mca %u", pc->pc_cpuid); smp_cpus++; } } ap_awake = 1; ap_spin = 0; while (ap_awake != smp_cpus) cpu_spinwait(); if (smp_cpus != cpus || cpus != mp_ncpus) { printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n", mp_ncpus, cpus, smp_cpus); } smp_active = 1; smp_started = 1; } /* * send an IPI to a set of cpus. */ void ipi_selected(cpumask_t cpus, int ipi) { struct pcpu *pc; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { if (cpus & pc->pc_cpumask) ipi_send(pc, ipi); } } /* * send an IPI to all CPUs EXCEPT myself. */ void ipi_all_but_self(int ipi) { struct pcpu *pc; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { if (pc != pcpup) ipi_send(pc, ipi); } } /* * Send an IPI to the specified processor. The lid parameter holds the * cr.lid (CR64) contents of the target processor. Only the id and eid * fields are used here. */ void -ipi_send(struct pcpu *cpu, int ipi) +ipi_send(struct pcpu *cpu, int xiv) { u_int lid; - uint8_t vector; + + KASSERT(xiv != 0, ("ipi_send")); lid = LID_SAPIC(cpu->pc_md.lid); - vector = ipi_vector[ipi]; - KASSERT(vector != 0, ("IPI %d is not assigned a vector", ipi)); ia64_mf(); - ia64_st8(&(ia64_pib->ib_ipi[lid][0]), vector); + ia64_st8(&(ia64_pib->ib_ipi[lid][0]), xiv); ia64_mf_a(); - CTR4(KTR_SMP, "ipi_send(%p, %ld): cpuid=%d, vector=%u", cpu, ipi, - PCPU_GET(cpuid), vector); + CTR3(KTR_SMP, "ipi_send(%p, %d): cpuid=%d", cpu, xiv, PCPU_GET(cpuid)); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); diff --git a/sys/ia64/ia64/nexus.c b/sys/ia64/ia64/nexus.c index 2f7c3744a2be..69f62a9b2fbd 100644 --- a/sys/ia64/ia64/nexus.c +++ b/sys/ia64/ia64/nexus.c @@ -1,574 +1,548 @@ /*- * Copyright 1998 Massachusetts Institute of Technology * * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby * granted, provided that both the above copyright notice and this * permission notice appear in all copies, that both the above * copyright notice and this permission notice appear in all * supporting documentation, and that the name of M.I.T. not be used * in advertising or publicity pertaining to distribution of the * software without specific, written prior permission. M.I.T. makes * no representations about the suitability of this software for any * purpose. It is provided "as is" without express or implied * warranty. * * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * This code implements a `root nexus' for Intel Architecture * machines. The function of the root nexus is to serve as an * attachment point for both processors and buses, and to manage * resources which are common to all of them. In particular, * this code implements the core resource managers for interrupt * requests, DMA requests (which rightfully should be a part of the * ISA code but it's easier to do it here for now), I/O port addresses, * and I/O memory address space. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "clock_if.h" static MALLOC_DEFINE(M_NEXUSDEV, "nexusdev", "Nexus device"); struct nexus_device { struct resource_list nx_resources; int nx_pcibus; }; #define DEVTONX(dev) ((struct nexus_device *)device_get_ivars(dev)) static struct rman irq_rman, port_rman, mem_rman; static int nexus_probe(device_t); static int nexus_attach(device_t); static int nexus_print_child(device_t, device_t); static device_t nexus_add_child(device_t bus, int order, const char *name, int unit); static struct resource *nexus_alloc_resource(device_t, device_t, int, int *, u_long, u_long, u_long, u_int); static int nexus_read_ivar(device_t, device_t, int, uintptr_t *); static int nexus_write_ivar(device_t, device_t, int, uintptr_t); static int nexus_activate_resource(device_t, device_t, int, int, struct resource *); static int nexus_deactivate_resource(device_t, device_t, int, int, struct resource *); static int nexus_release_resource(device_t, device_t, int, int, struct resource *); static int nexus_setup_intr(device_t, device_t, struct resource *, int flags, driver_filter_t filter, void (*)(void *), void *, void **); static int nexus_teardown_intr(device_t, device_t, struct resource *, void *); static struct resource_list *nexus_get_reslist(device_t dev, device_t child); static int nexus_set_resource(device_t, device_t, int, int, u_long, u_long); static int nexus_get_resource(device_t, device_t, int, int, u_long *, u_long *); static void nexus_delete_resource(device_t, device_t, int, int); static int nexus_config_intr(device_t, int, enum intr_trigger, enum intr_polarity); static int nexus_gettime(device_t, struct timespec *); static int nexus_settime(device_t, struct timespec *); static device_method_t nexus_methods[] = { /* Device interface */ DEVMETHOD(device_probe, nexus_probe), DEVMETHOD(device_attach, nexus_attach), DEVMETHOD(device_detach, bus_generic_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* Bus interface */ DEVMETHOD(bus_print_child, nexus_print_child), DEVMETHOD(bus_add_child, nexus_add_child), DEVMETHOD(bus_read_ivar, nexus_read_ivar), DEVMETHOD(bus_write_ivar, nexus_write_ivar), DEVMETHOD(bus_alloc_resource, nexus_alloc_resource), DEVMETHOD(bus_release_resource, nexus_release_resource), DEVMETHOD(bus_activate_resource, nexus_activate_resource), DEVMETHOD(bus_deactivate_resource, nexus_deactivate_resource), DEVMETHOD(bus_setup_intr, nexus_setup_intr), DEVMETHOD(bus_teardown_intr, nexus_teardown_intr), DEVMETHOD(bus_get_resource_list, nexus_get_reslist), DEVMETHOD(bus_set_resource, nexus_set_resource), DEVMETHOD(bus_get_resource, nexus_get_resource), DEVMETHOD(bus_delete_resource, nexus_delete_resource), DEVMETHOD(bus_config_intr, nexus_config_intr), /* Clock interface */ DEVMETHOD(clock_gettime, nexus_gettime), DEVMETHOD(clock_settime, nexus_settime), { 0, 0 } }; static driver_t nexus_driver = { "nexus", nexus_methods, 1, /* no softc */ }; static devclass_t nexus_devclass; DRIVER_MODULE(nexus, root, nexus_driver, nexus_devclass, 0, 0); static int nexus_probe(device_t dev) { device_quiet(dev); /* suppress attach message for neatness */ - /* - * XXX working notes: - * - * - IRQ resource creation should be moved to the PIC/APIC driver. - * - DRQ resource creation should be moved to the DMAC driver. - * - The above should be sorted to probe earlier than any child busses. - * - * - Leave I/O and memory creation here, as child probes may need them. - * (especially eg. ACPI) - */ - - /* - * IRQ's are on the mainboard on old systems, but on the ISA part - * of PCI->ISA bridges. There would be multiple sets of IRQs on - * multi-ISA-bus systems. PCI interrupts are routed to the ISA - * component, so in a way, PCI can be a partial child of an ISA bus(!). - * APIC interrupts are global though. - * - * XXX We depend on the AT PIC driver correctly claiming IRQ 2 - * to prevent its reuse elsewhere in the !APIC_IO case. - */ - irq_rman.rm_start = 0; irq_rman.rm_type = RMAN_ARRAY; irq_rman.rm_descr = "Interrupt request lines"; - irq_rman.rm_end = 255; + irq_rman.rm_start = 0; + irq_rman.rm_end = IA64_NXIVS - 1; if (rman_init(&irq_rman) || rman_manage_region(&irq_rman, irq_rman.rm_start, irq_rman.rm_end)) panic("nexus_probe irq_rman"); - /* - * However, IO ports and Memory truely are global at this level, - * as are APIC interrupts (however many IO APICS there turn out - * to be on large systems..) - */ port_rman.rm_start = 0; port_rman.rm_end = 0xffff; port_rman.rm_type = RMAN_ARRAY; port_rman.rm_descr = "I/O ports"; if (rman_init(&port_rman) || rman_manage_region(&port_rman, 0, 0xffff)) panic("nexus_probe port_rman"); mem_rman.rm_start = 0; mem_rman.rm_end = ~0u; mem_rman.rm_type = RMAN_ARRAY; mem_rman.rm_descr = "I/O memory addresses"; if (rman_init(&mem_rman) || rman_manage_region(&mem_rman, 0, ~0)) panic("nexus_probe mem_rman"); return bus_generic_probe(dev); } static int nexus_attach(device_t dev) { /* * Mask the legacy PICs - we will use the I/O SAPIC for interrupt. */ outb(IO_ICU1+1, 0xff); outb(IO_ICU2+1, 0xff); if (acpi_identify() == 0) BUS_ADD_CHILD(dev, 10, "acpi", 0); clock_register(dev, 1000); bus_generic_attach(dev); return 0; } static int nexus_print_child(device_t bus, device_t child) { struct nexus_device *ndev = DEVTONX(child); struct resource_list *rl = &ndev->nx_resources; int retval = 0; retval += bus_print_child_header(bus, child); retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#lx"); retval += resource_list_print_type(rl, "iomem", SYS_RES_MEMORY, "%#lx"); retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%ld"); if (ndev->nx_pcibus != -1) retval += printf(" pcibus %d", ndev->nx_pcibus); if (device_get_flags(child)) retval += printf(" flags %#x", device_get_flags(child)); retval += printf(" on motherboard\n"); /* XXX "motherboard", ick */ return (retval); } static device_t nexus_add_child(device_t bus, int order, const char *name, int unit) { device_t child; struct nexus_device *ndev; ndev = malloc(sizeof(struct nexus_device), M_NEXUSDEV, M_NOWAIT|M_ZERO); if (!ndev) return(0); resource_list_init(&ndev->nx_resources); ndev->nx_pcibus = -1; child = device_add_child_ordered(bus, order, name, unit); /* should we free this in nexus_child_detached? */ device_set_ivars(child, ndev); return(child); } static int nexus_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct nexus_device *ndev = DEVTONX(child); switch (which) { case NEXUS_IVAR_PCIBUS: *result = ndev->nx_pcibus; break; default: return ENOENT; } return 0; } static int nexus_write_ivar(device_t dev, device_t child, int which, uintptr_t value) { struct nexus_device *ndev = DEVTONX(child); switch (which) { case NEXUS_IVAR_PCIBUS: ndev->nx_pcibus = value; break; default: return ENOENT; } return 0; } /* * Allocate a resource on behalf of child. NB: child is usually going to be a * child of one of our descendants, not a direct child of nexus0. * (Exceptions include npx.) */ static struct resource * nexus_alloc_resource(device_t bus, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { struct nexus_device *ndev = DEVTONX(child); struct resource *rv; struct resource_list_entry *rle; struct rman *rm; int needactivate = flags & RF_ACTIVE; /* * If this is an allocation of the "default" range for a given RID, and * we know what the resources for this device are (ie. they aren't maintained * by a child bus), then work out the start/end values. */ if ((start == 0UL) && (end == ~0UL) && (count == 1)) { if (ndev == NULL) return(NULL); rle = resource_list_find(&ndev->nx_resources, type, *rid); if (rle == NULL) return(NULL); start = rle->start; end = rle->end; count = rle->count; } flags &= ~RF_ACTIVE; switch (type) { case SYS_RES_IRQ: rm = &irq_rman; break; case SYS_RES_IOPORT: rm = &port_rman; break; case SYS_RES_MEMORY: rm = &mem_rman; break; default: return 0; } rv = rman_reserve_resource(rm, start, end, count, flags, child); if (rv == 0) return 0; rman_set_rid(rv, *rid); if (needactivate) { if (bus_activate_resource(child, type, *rid, rv)) { rman_release_resource(rv); return 0; } } return rv; } static int nexus_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { vm_paddr_t paddr; void *vaddr; paddr = rman_get_start(r); switch (type) { case SYS_RES_IOPORT: rman_set_bustag(r, IA64_BUS_SPACE_IO); rman_set_bushandle(r, paddr); break; case SYS_RES_MEMORY: vaddr = pmap_mapdev(paddr, rman_get_size(r)); rman_set_bustag(r, IA64_BUS_SPACE_MEM); rman_set_bushandle(r, (bus_space_handle_t) vaddr); rman_set_virtual(r, vaddr); break; } return (rman_activate_resource(r)); } static int nexus_deactivate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { return (rman_deactivate_resource(r)); } static int nexus_release_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { if (rman_get_flags(r) & RF_ACTIVE) { int error = bus_deactivate_resource(child, type, rid, r); if (error) return error; } return (rman_release_resource(r)); } /* * Currently this uses the really grody interface from kern/kern_intr.c * (which really doesn't belong in kern/anything.c). Eventually, all of * the code in kern_intr.c and machdep_intr.c should get moved here, since * this is going to be the official interface. */ static int nexus_setup_intr(device_t bus, device_t child, struct resource *irq, int flags, driver_filter_t filter, void (*ihand)(void *), void *arg, void **cookiep) { driver_t *driver; int error; /* somebody tried to setup an irq that failed to allocate! */ if (irq == NULL) panic("nexus_setup_intr: NULL irq resource!"); *cookiep = 0; if ((rman_get_flags(irq) & RF_SHAREABLE) == 0) flags |= INTR_EXCL; driver = device_get_driver(child); /* * We depend here on rman_activate_resource() being idempotent. */ error = rman_activate_resource(irq); if (error) return (error); error = ia64_setup_intr(device_get_nameunit(child), rman_get_start(irq), filter, ihand, arg, flags, cookiep); return (error); } static int nexus_teardown_intr(device_t dev, device_t child, struct resource *ires, void *cookie) { return (ia64_teardown_intr(cookie)); } static struct resource_list * nexus_get_reslist(device_t dev, device_t child) { struct nexus_device *ndev = DEVTONX(child); return (&ndev->nx_resources); } static int nexus_set_resource(device_t dev, device_t child, int type, int rid, u_long start, u_long count) { struct nexus_device *ndev = DEVTONX(child); struct resource_list *rl = &ndev->nx_resources; if (type == SYS_RES_IOPORT && start > (0x10000 - count)) { /* * Work around a firmware bug in the HP rx2660, where in ACPI * an I/O port is really a memory mapped I/O address. The bug * is in the GAS that describes the address and in particular * the SpaceId field. The field should not say the address is * an I/O port when it is in fact an I/O memory address. */ if (bootverbose) printf("%s: invalid port range (%#lx-%#lx); " "assuming I/O memory range.\n", __func__, start, start + count - 1); type = SYS_RES_MEMORY; } /* XXX this should return a success/failure indicator */ resource_list_add(rl, type, rid, start, start + count - 1, count); return(0); } static int nexus_get_resource(device_t dev, device_t child, int type, int rid, u_long *startp, u_long *countp) { struct nexus_device *ndev = DEVTONX(child); struct resource_list *rl = &ndev->nx_resources; struct resource_list_entry *rle; rle = resource_list_find(rl, type, rid); device_printf(child, "type %d rid %d startp %p countp %p - got %p\n", type, rid, startp, countp, rle); if (!rle) return(ENOENT); if (startp) *startp = rle->start; if (countp) *countp = rle->count; return(0); } static void nexus_delete_resource(device_t dev, device_t child, int type, int rid) { struct nexus_device *ndev = DEVTONX(child); struct resource_list *rl = &ndev->nx_resources; resource_list_delete(rl, type, rid); } static int nexus_config_intr(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { return (sapic_config_intr(irq, trig, pol)); } static int nexus_gettime(device_t dev, struct timespec *ts) { struct clocktime ct; struct efi_tm tm; efi_get_time(&tm); /* * This code was written in 2005, so logically EFI cannot return * a year smaller than that. Assume the EFI clock is out of whack * in that case and reset the EFI clock. */ if (tm.tm_year < 2005) return (EINVAL); ct.nsec = tm.tm_nsec; ct.sec = tm.tm_sec; ct.min = tm.tm_min; ct.hour = tm.tm_hour; ct.day = tm.tm_mday; ct.mon = tm.tm_mon; ct.year = tm.tm_year; ct.dow = -1; return (clock_ct_to_ts(&ct, ts)); } static int nexus_settime(device_t dev, struct timespec *ts) { struct clocktime ct; struct efi_tm tm; efi_get_time(&tm); clock_ts_to_ct(ts, &ct); tm.tm_nsec = ts->tv_nsec; tm.tm_sec = ct.sec; tm.tm_min = ct.min; tm.tm_hour = ct.hour; tm.tm_year = ct.year; tm.tm_mon = ct.mon; tm.tm_mday = ct.day; return (efi_set_time(&tm)); } diff --git a/sys/ia64/ia64/sal.c b/sys/ia64/ia64/sal.c index 4f46d652b02a..6f081be8601a 100644 --- a/sys/ia64/ia64/sal.c +++ b/sys/ia64/ia64/sal.c @@ -1,181 +1,142 @@ /*- * Copyright (c) 2001 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include #include #include +#include #include #include #include -/* - * IPIs are used more genericly than only - * for inter-processor interrupts. Don't - * make it a SMP specific thing... - */ -int ipi_vector[IPI_COUNT]; +int ia64_ipi_wakeup; static struct ia64_fdesc sal_fdesc; static sal_entry_t fake_sal; extern u_int64_t ia64_pal_entry; sal_entry_t *ia64_sal_entry = fake_sal; static struct uuid sal_table = EFI_TABLE_SAL; static struct sal_system_table *sal_systbl; static struct ia64_sal_result fake_sal(u_int64_t a1, u_int64_t a2, u_int64_t a3, u_int64_t a4, u_int64_t a5, u_int64_t a6, u_int64_t a7, u_int64_t a8) { struct ia64_sal_result res; res.sal_status = -3; res.sal_result[0] = 0; res.sal_result[1] = 0; res.sal_result[2] = 0; return res; } -static void -setup_ipi_vectors(int ceil) -{ - int ipi; - - ipi_vector[IPI_MCA_RENDEZ] = ceil - 0x10; - - ipi = IPI_AST; /* First generic IPI. */ - ceil -= 0x20; /* First vector in group. */ - while (ipi < IPI_COUNT) - ipi_vector[ipi++] = ceil++; - - ipi_vector[IPI_HIGH_FP] = ceil - 0x30; - ipi_vector[IPI_MCA_CMCV] = ceil - 0x30 + 1; -} - void ia64_sal_init(void) { static int sizes[6] = { 48, 32, 16, 32, 16, 16 }; u_int8_t *p; - int i; + int error, i; sal_systbl = efi_get_table(&sal_table); if (sal_systbl == NULL) return; if (bcmp(sal_systbl->sal_signature, SAL_SIGNATURE, 4)) { printf("Bad signature for SAL System Table\n"); return; } p = (u_int8_t *) (sal_systbl + 1); for (i = 0; i < sal_systbl->sal_entry_count; i++) { switch (*p) { case 0: { struct sal_entrypoint_descriptor *dp; dp = (struct sal_entrypoint_descriptor*)p; ia64_pal_entry = IA64_PHYS_TO_RR7(dp->sale_pal_proc); if (bootverbose) printf("PAL Proc at 0x%lx\n", ia64_pal_entry); sal_fdesc.func = IA64_PHYS_TO_RR7(dp->sale_sal_proc); sal_fdesc.gp = IA64_PHYS_TO_RR7(dp->sale_sal_gp); if (bootverbose) printf("SAL Proc at 0x%lx, GP at 0x%lx\n", sal_fdesc.func, sal_fdesc.gp); ia64_sal_entry = (sal_entry_t *) &sal_fdesc; break; } case 5: { struct sal_ap_wakeup_descriptor *dp; #ifdef SMP struct ia64_sal_result result; struct ia64_fdesc *fd; #endif dp = (struct sal_ap_wakeup_descriptor*)p; if (dp->sale_mechanism != 0) { printf("SAL: unsupported AP wake-up mechanism " "(%d)\n", dp->sale_mechanism); break; } - if (dp->sale_vector < 0x10 || dp->sale_vector > 0xff) { - printf("SAL: invalid AP wake-up vector " - "(0x%lx)\n", dp->sale_vector); - break; - } - - /* - * SAL documents that the wake-up vector should be - * high (close to 255). The MCA rendezvous vector - * should be less than the wake-up vector, but still - * "high". We use the following priority assignment: - * Wake-up: priority of the sale_vector - * Rendezvous: priority-1 - * Generic IPIs: priority-2 - * Special IPIs: priority-3 - * Consequently, the wake-up priority should be at - * least 4 (ie vector >= 0x40). - */ - if (dp->sale_vector < 0x40) { - printf("SAL: AP wake-up vector too low " - "(0x%lx)\n", dp->sale_vector); + /* Reserve the XIV so that we won't use it. */ + error = ia64_xiv_reserve(dp->sale_vector, + IA64_XIV_PLAT, NULL); + if (error) { + printf("SAL: invalid AP wake-up XIV (%#lx)\n", + dp->sale_vector); break; } + ia64_ipi_wakeup = dp->sale_vector; if (bootverbose) - printf("SAL: AP wake-up vector: 0x%lx\n", - dp->sale_vector); - - ipi_vector[IPI_AP_WAKEUP] = dp->sale_vector; - setup_ipi_vectors(dp->sale_vector & 0xf0); + printf("SAL: AP wake-up XIV: %#x\n", + ia64_ipi_wakeup); #ifdef SMP fd = (struct ia64_fdesc *) os_boot_rendez; result = ia64_sal_entry(SAL_SET_VECTORS, SAL_OS_BOOT_RENDEZ, ia64_tpa(fd->func), ia64_tpa(fd->gp), 0, 0, 0, 0); #endif break; } } p += sizes[*p]; } - - if (ipi_vector[IPI_AP_WAKEUP] == 0) - setup_ipi_vectors(0xf0); } diff --git a/sys/ia64/include/clock.h b/sys/ia64/include/clock.h index 772fc2a29ed3..6b87a89f9ede 100644 --- a/sys/ia64/include/clock.h +++ b/sys/ia64/include/clock.h @@ -1,20 +1,12 @@ /*- * Kernel interface to machine-dependent clock driver. * Garrett Wollman, September 1994. * This file is in the public domain. * * $FreeBSD$ */ #ifndef _MACHINE_CLOCK_H_ #define _MACHINE_CLOCK_H_ -#ifdef _KERNEL - -#define CLOCK_VECTOR 254 - -extern uint64_t ia64_clock_reload; - -#endif - #endif /* !_MACHINE_CLOCK_H_ */ diff --git a/sys/ia64/include/intr.h b/sys/ia64/include/intr.h index 0635e3409f53..b26190c52fdd 100644 --- a/sys/ia64/include/intr.h +++ b/sys/ia64/include/intr.h @@ -1,67 +1,90 @@ /*- * Copyright (c) 2007-2010 Marcel Moolenaar * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_INTR_H_ #define _MACHINE_INTR_H_ +#define IA64_NXIVS 256 /* External Interrupt Vectors */ +#define IA64_MIN_XIV 16 + +#define IA64_MAX_HWPRIO 14 + struct sapic; +struct thread; +struct trapframe; /* * Layout of the Processor Interrupt Block. */ struct ia64_pib { uint64_t ib_ipi[65536][2]; /* 64K-way IPIs (1MB area). */ uint8_t _rsvd1[0xe0000]; uint8_t ib_inta; /* Generate INTA cycle. */ uint8_t _rsvd2[7]; uint8_t ib_xtp; /* External Task Priority. */ uint8_t _rsvd3[7]; uint8_t _rsvd4[0x1fff0]; }; +enum ia64_xiv_use { + IA64_XIV_FREE, + IA64_XIV_ARCH, /* Architecturally defined. */ + IA64_XIV_PLAT, /* Platform defined. */ + IA64_XIV_IPI, /* Used for IPIs. */ + IA64_XIV_IRQ /* Used for external interrupts. */ +}; + +typedef u_int (ia64_ihtype)(struct thread *, u_int, struct trapframe *); + extern struct ia64_pib *ia64_pib; +void ia64_handle_intr(struct trapframe *); int ia64_setup_intr(const char *, int, driver_filter_t, driver_intr_t, void *, enum intr_type, void **); int ia64_teardown_intr(void *); +void ia64_xiv_init(void); +u_int ia64_xiv_alloc(u_int, enum ia64_xiv_use, ia64_ihtype); +int ia64_xiv_free(u_int, enum ia64_xiv_use); +int ia64_xiv_reserve(u_int, enum ia64_xiv_use, ia64_ihtype); + int sapic_config_intr(u_int, enum intr_trigger, enum intr_polarity); struct sapic *sapic_create(u_int, u_int, uint64_t); int sapic_enable(struct sapic *, u_int, u_int); void sapic_eoi(struct sapic *, u_int); struct sapic *sapic_lookup(u_int, u_int *); void sapic_mask(struct sapic *, u_int); void sapic_unmask(struct sapic *, u_int); #ifdef DDB void sapic_print(struct sapic *, u_int); #endif #endif /* !_MACHINE_INTR_H_ */ diff --git a/sys/ia64/include/intrcnt.h b/sys/ia64/include/intrcnt.h index c4f73c014306..5e165ea75efe 100644 --- a/sys/ia64/include/intrcnt.h +++ b/sys/ia64/include/intrcnt.h @@ -1,44 +1,40 @@ /* $FreeBSD$ */ /* $NetBSD: intrcnt.h,v 1.17 1998/11/19 01:48:04 ross Exp $ */ /*- * Copyright (c) 1995, 1996 Carnegie-Mellon University. * All rights reserved. * * Author: Chris G. Demetriou * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ #define INTRCNT_CLOCK 0 -#define INTRCNT_ISA_IRQ (INTRCNT_CLOCK + 1) -#define INTRCNT_ISA_IRQ_LEN 16 -#define INTRCNT_OTHER_BASE (INTRCNT_ISA_IRQ + INTRCNT_ISA_IRQ_LEN) -#define INTRCNT_OTHER_LEN 240 -#define INTRCNT_COUNT (INTRCNT_OTHER_BASE + INTRCNT_OTHER_LEN) +#define INTRCNT_COUNT 256 /* * Maximum name length in intrnames table (including terminating '\0'. * Since vmstat(8) assumes a maximum length of 13 (including '\0'), we're * pretty much limited to that (unless we don't care about the alignment * of the columns :-) */ #define INTRNAME_LEN 13 diff --git a/sys/ia64/include/smp.h b/sys/ia64/include/smp.h index 4eddf7434fea..9f976dec5970 100644 --- a/sys/ia64/include/smp.h +++ b/sys/ia64/include/smp.h @@ -1,41 +1,33 @@ /* * $FreeBSD$ */ #ifndef _MACHINE_SMP_H_ #define _MACHINE_SMP_H_ #ifdef _KERNEL -/* - * Interprocessor interrupts for SMP. The following values are indices - * into the IPI vector table. The SAL gives us the vector used for AP - * wake-up. We base the other vectors on that. Keep IPI_AP_WAKEUP at - * index 0. See sal.c for details. - */ -/* Architecture specific IPIs. */ -#define IPI_AP_WAKEUP 0 -#define IPI_HIGH_FP 1 -#define IPI_MCA_CMCV 2 -#define IPI_MCA_RENDEZ 3 -/* Machine independent IPIs. */ -#define IPI_AST 4 -#define IPI_RENDEZVOUS 5 -#define IPI_STOP 6 -#define IPI_STOP_HARD 6 -#define IPI_PREEMPT 7 - -#define IPI_COUNT 8 +#define IPI_AST ia64_ipi_ast +#define IPI_PREEMPT ia64_ipi_preempt +#define IPI_RENDEZVOUS ia64_ipi_rndzvs +#define IPI_STOP ia64_ipi_stop +#define IPI_STOP_HARD ia64_ipi_nmi #ifndef LOCORE struct pcpu; -extern int ipi_vector[]; +extern int ia64_ipi_ast; +extern int ia64_ipi_highfp; +extern int ia64_ipi_nmi; +extern int ia64_ipi_preempt; +extern int ia64_ipi_rndzvs; +extern int ia64_ipi_stop; +extern int ia64_ipi_wakeup; void ipi_all_but_self(int ipi); void ipi_selected(cpumask_t cpus, int ipi); void ipi_send(struct pcpu *, int ipi); #endif /* !LOCORE */ #endif /* _KERNEL */ #endif /* !_MACHINE_SMP_H */