Index: head/sys/amd64/include/pcpu.h =================================================================== --- head/sys/amd64/include/pcpu.h +++ head/sys/amd64/include/pcpu.h @@ -78,6 +78,7 @@ extern struct pcpu *pcpup; +#define get_pcpu() (pcpup) #define PCPU_GET(member) (pcpup->pc_ ## member) #define PCPU_ADD(member, val) (pcpup->pc_ ## member += (val)) #define PCPU_INC(member) PCPU_ADD(member, 1) @@ -203,6 +204,15 @@ } \ } +#define get_pcpu() __extension__ ({ \ + struct pcpu *__pc; \ + \ + __asm __volatile("movq %%gs:%1,%0" \ + : "=r" (__pc) \ + : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace)))); \ + __pc; \ +}) + #define PCPU_GET(member) __PCPU_GET(pc_ ## member) #define PCPU_ADD(member, val) __PCPU_ADD(pc_ ## member, val) #define PCPU_INC(member) __PCPU_INC(pc_ ## member) Index: head/sys/kern/kern_rmlock.c =================================================================== --- head/sys/kern/kern_rmlock.c +++ head/sys/kern/kern_rmlock.c @@ -156,7 +156,7 @@ */ critical_enter(); td = curthread; - pc = pcpu_find(curcpu); + pc = get_pcpu(); for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; queue = queue->rmq_next) { tracker = (struct rm_priotracker *)queue; @@ -258,7 +258,7 @@ struct rmlock *rm = arg; struct rm_priotracker *tracker; struct rm_queue *queue; - pc = pcpu_find(curcpu); + pc = get_pcpu(); for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; queue = queue->rmq_next) { @@ -355,7 +355,7 @@ struct pcpu *pc; critical_enter(); - pc = pcpu_find(curcpu); + pc = get_pcpu(); /* Check if we just need to do a proper critical_exit. */ if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) { @@ -416,7 +416,7 @@ } critical_enter(); - pc = pcpu_find(curcpu); + pc = get_pcpu(); CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus); rm_tracker_add(pc, tracker); sched_pin(); @@ -641,7 +641,7 @@ #ifdef INVARIANTS if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) { critical_enter(); - KASSERT(rm_trackers_present(pcpu_find(curcpu), rm, + KASSERT(rm_trackers_present(get_pcpu(), rm, curthread) == 0, ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n", rm->lock_object.lo_name, file, line)); @@ -771,7 +771,7 @@ } critical_enter(); - count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); + count = rm_trackers_present(get_pcpu(), rm, curthread); critical_exit(); if (count == 0) @@ -797,7 +797,7 @@ rm->lock_object.lo_name, file, line); critical_enter(); - count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); + count = rm_trackers_present(get_pcpu(), rm, curthread); critical_exit(); if (count != 0) Index: head/sys/mips/include/pcpu.h =================================================================== --- head/sys/mips/include/pcpu.h +++ head/sys/mips/include/pcpu.h @@ -39,16 +39,17 @@ struct pmap *pc_curpmap; /* pmap of curthread */ \ u_int32_t pc_next_asid; /* next ASID to alloc */ \ u_int32_t pc_asid_generation; /* current ASID generation */ \ - u_int pc_pending_ipis; /* IPIs pending to this CPU */ + u_int pc_pending_ipis; /* IPIs pending to this CPU */ \ + struct pcpu *pc_self; /* globally-uniqe self pointer */ #ifdef __mips_n64 #define PCPU_MD_MIPS64_FIELDS \ PCPU_MD_COMMON_FIELDS \ - char __pad[61] + char __pad[53] #else #define PCPU_MD_MIPS32_FIELDS \ PCPU_MD_COMMON_FIELDS \ - char __pad[193] + char __pad[189] #endif #ifdef __mips_n64 @@ -65,6 +66,13 @@ extern struct pcpu *pcpup; #define PCPUP pcpup +/* + * Since we use a wired TLB entry to map the same VA to a different + * physical page for each CPU, get_pcpu() must use the pc_self + * field to obtain a globally-unique pointer. + */ +#define get_pcpu() (PCPUP->pc_self) + #define PCPU_ADD(member, value) (PCPUP->pc_ ## member += (value)) #define PCPU_GET(member) (PCPUP->pc_ ## member) #define PCPU_INC(member) PCPU_ADD(member, 1) Index: head/sys/mips/mips/machdep.c =================================================================== --- head/sys/mips/mips/machdep.c +++ head/sys/mips/mips/machdep.c @@ -475,6 +475,7 @@ pcpu->pc_next_asid = 1; pcpu->pc_asid_generation = 1; + pcpu->pc_self = pcpu; #ifdef SMP if ((vm_offset_t)pcpup >= VM_MIN_KERNEL_ADDRESS && (vm_offset_t)pcpup <= VM_MAX_KERNEL_ADDRESS) { Index: head/sys/net/netisr.c =================================================================== --- head/sys/net/netisr.c +++ head/sys/net/netisr.c @@ -1268,9 +1268,7 @@ static void netisr_init(void *arg) { -#ifdef EARLY_AP_STARTUP struct pcpu *pc; -#endif NETISR_LOCK_INIT(); if (netisr_maxthreads == 0 || netisr_maxthreads < -1 ) @@ -1308,7 +1306,8 @@ netisr_start_swi(pc->pc_cpuid, pc); } #else - netisr_start_swi(curcpu, pcpu_find(curcpu)); + pc = get_pcpu(); + netisr_start_swi(pc->pc_cpuid, pc); #endif } SYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL); Index: head/sys/powerpc/include/cpufunc.h =================================================================== --- head/sys/powerpc/include/cpufunc.h +++ head/sys/powerpc/include/cpufunc.h @@ -201,7 +201,7 @@ } static __inline struct pcpu * -powerpc_get_pcpup(void) +get_pcpu(void) { struct pcpu *ret; Index: head/sys/powerpc/include/pcpu.h =================================================================== --- head/sys/powerpc/include/pcpu.h +++ head/sys/powerpc/include/pcpu.h @@ -142,7 +142,7 @@ #ifdef _KERNEL -#define pcpup ((struct pcpu *) powerpc_get_pcpup()) +#define pcpup (get_pcpu()) static __inline __pure2 struct thread * __curthread(void) Index: head/sys/sparc64/include/pcpu.h =================================================================== --- head/sys/sparc64/include/pcpu.h +++ head/sys/sparc64/include/pcpu.h @@ -74,6 +74,7 @@ register struct pcb *curpcb __asm__(__XSTRING(PCB_REG)); register struct pcpu *pcpup __asm__(__XSTRING(PCPU_REG)); +#define get_pcpu() (pcpup) #define PCPU_GET(member) (pcpup->pc_ ## member) static __inline __pure2 struct thread *