Index: sys/amd64/include/counter.h =================================================================== --- sys/amd64/include/counter.h +++ sys/amd64/include/counter.h @@ -33,7 +33,7 @@ #include -#define EARLY_COUNTER &temp_bsp_pcpu.pc_early_dummy_counter +#define EARLY_COUNTER (void *)__offsetof(struct pcpu, pc_early_dummy_counter) #define counter_enter() do {} while (0) #define counter_exit() do {} while (0) @@ -43,6 +43,7 @@ counter_u64_read_one(counter_u64_t c, int cpu) { + MPASS(c != EARLY_COUNTER); return (*zpcpu_get_cpu(c, cpu)); } @@ -65,6 +66,7 @@ counter_u64_t c; c = arg; + MPASS(c != EARLY_COUNTER); *(zpcpu_get(c)) = 0; } @@ -86,7 +88,7 @@ KASSERT(IS_BSP() || c != EARLY_COUNTER, ("EARLY_COUNTER used on AP")); __asm __volatile("addq\t%1,%%gs:(%0)" : - : "r" ((char *)c - (char *)&__pcpu[0]), "ri" (inc) + : "r" (c), "ri" (inc) : "memory", "cc"); } Index: sys/amd64/include/pcpu.h =================================================================== --- sys/amd64/include/pcpu.h +++ sys/amd64/include/pcpu.h @@ -240,6 +240,10 @@ #define IS_BSP() (PCPU_GET(cpuid) == 0) +#define zpcpu_offset_cpu(cpu) ((uintptr_t)&__pcpu[0] + UMA_PCPU_ALLOC_SIZE * cpu) +#define zpcpu_base_to_offset(base) (void *)((uintptr_t)(base) - (uintptr_t)&__pcpu[0]) +#define zpcpu_offset_to_base(base) (void *)((uintptr_t)(base) + (uintptr_t)&__pcpu[0]) + #else /* !__GNUCLIKE_ASM || !__GNUCLIKE___TYPEOF */ #error "this file needs to be ported to your compiler" Index: sys/kern/subr_pcpu.c =================================================================== --- sys/kern/subr_pcpu.c +++ sys/kern/subr_pcpu.c @@ -95,6 +95,7 @@ cpu_pcpu_init(pcpu, cpuid, size); pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; + pcpu->pc_zpcpu_offset = zpcpu_offset_cpu(cpuid); } void Index: sys/sys/pcpu.h =================================================================== --- sys/sys/pcpu.h +++ sys/sys/pcpu.h @@ -194,6 +194,7 @@ struct rm_queue pc_rm_queue; /* rmlock list of trackers */ uintptr_t pc_dynamic; /* Dynamic per-cpu data area */ uint64_t pc_early_dummy_counter; /* Startup time counter(9) */ + uintptr_t pc_zpcpu_offset; /* Offset into zpcpu allocs */ /* * Keep MD fields last, so that CPU-specific variations on a @@ -227,14 +228,28 @@ #endif #define curproc (curthread->td_proc) +#ifndef zpcpu_offset_cpu +#define zpcpu_offset_cpu(cpu) (UMA_PCPU_ALLOC_SIZE * cpu) +#endif +#ifndef zpcpu_offset +#define zpcpu_offset() (PCPU_GET(zpcpu_offset)) +#endif + +#ifndef zpcpu_base_to_offset +#define zpcpu_base_to_offset(base) base +#endif +#ifndef zpcpu_offset_to_base +#define zpcpu_offset_to_base(base) base +#endif + /* Accessor to elements allocated via UMA_ZONE_PCPU zone. */ #define zpcpu_get(base) ({ \ - __typeof(base) _ptr = (void *)((char *)(base) + UMA_PCPU_ALLOC_SIZE * curcpu); \ + __typeof(base) _ptr = (void *)((char *)(base) + zpcpu_offset()); \ _ptr; \ }) #define zpcpu_get_cpu(base, cpu) ({ \ - __typeof(base) _ptr = (void *)((char *)(base) + UMA_PCPU_ALLOC_SIZE * cpu); \ + __typeof(base) _ptr = (void *)((char *)(base) + zpcpu_offset_cpu(cpu)); \ _ptr; \ }) Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -2944,34 +2944,39 @@ void * uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) { - void *item; + void *item, *pcpu_item; #ifdef SMP int i; MPASS(zone->uz_flags & UMA_ZONE_PCPU); #endif item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); - if (item != NULL && (flags & M_ZERO)) { + if (item == NULL) + return (NULL); + pcpu_item = zpcpu_base_to_offset(item); + if (flags & M_ZERO) { #ifdef SMP for (i = 0; i <= mp_maxid; i++) - bzero(zpcpu_get_cpu(item, i), zone->uz_size); + bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size); #else bzero(item, zone->uz_size); #endif } - return (item); + return (pcpu_item); } /* * A stub while both regular and pcpu cases are identical. */ void -uma_zfree_pcpu_arg(uma_zone_t zone, void *item, void *udata) +uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata) { + void *item; #ifdef SMP MPASS(zone->uz_flags & UMA_ZONE_PCPU); #endif + item = zpcpu_offset_to_base(pcpu_item); uma_zfree_arg(zone, item, udata); }