Index: sys/amd64/amd64/mp_machdep.c =================================================================== --- sys/amd64/amd64/mp_machdep.c +++ sys/amd64/amd64/mp_machdep.c @@ -94,8 +94,6 @@ #define AP_BOOTPT_SZ (PAGE_SIZE * 3) -extern struct pcpu *__pcpu; - /* Temporary variables for init_secondary() */ char *doublefault_stack; char *mce_stack; Index: sys/amd64/amd64/pmap.c =================================================================== --- sys/amd64/amd64/pmap.c +++ sys/amd64/amd64/pmap.c @@ -443,10 +443,6 @@ static vm_pindex_t pti_pg_idx; static bool pti_finalized; -extern struct pcpu *__pcpu; -extern struct pcpu temp_bsp_pcpu; -extern pt_entry_t *pcpu_pte; - struct pmap_pkru_range { struct rs_el pkru_rs_el; u_int pkru_keyidx; Index: sys/amd64/amd64/vm_machdep.c =================================================================== --- sys/amd64/amd64/vm_machdep.c +++ sys/amd64/amd64/vm_machdep.c @@ -84,10 +84,6 @@ #include #include -_Static_assert(OFFSETOF_CURTHREAD == offsetof(struct pcpu, pc_curthread), - "OFFSETOF_CURTHREAD does not correspond with offset of pc_curthread."); -_Static_assert(OFFSETOF_CURPCB == offsetof(struct pcpu, pc_curpcb), - "OFFSETOF_CURPCB does not correspond with offset of pc_curpcb."); _Static_assert(OFFSETOF_MONITORBUF == offsetof(struct pcpu, pc_monitorbuf), "OFFSETOF_MONITORBUF does not correspond with offset of pc_monitorbuf."); Index: sys/amd64/include/counter.h =================================================================== --- sys/amd64/include/counter.h +++ sys/amd64/include/counter.h @@ -33,9 +33,6 @@ #include -extern struct pcpu *__pcpu; -extern struct pcpu temp_bsp_pcpu; - #define EARLY_COUNTER &temp_bsp_pcpu.pc_early_dummy_counter #define counter_enter() do {} while (0) Index: sys/amd64/include/pcpu.h =================================================================== --- sys/amd64/include/pcpu.h +++ sys/amd64/include/pcpu.h @@ -233,35 +233,6 @@ #define PCPU_PTR(member) __PCPU_PTR(pc_ ## member) #define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val) -#define OFFSETOF_CURTHREAD 0 -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wnull-dereference" -#endif -static __inline __pure2 struct thread * -__curthread(void) -{ - struct thread *td; - - __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (OFFSETOF_CURTHREAD)); - return (td); -} -#ifdef __clang__ -#pragma clang diagnostic pop -#endif -#define curthread (__curthread()) - -#define OFFSETOF_CURPCB 32 -static __inline __pure2 struct pcb * -__curpcb(void) -{ - struct pcb *pcb; - - __asm("movq %%gs:%P1,%0" : "=r" (pcb) : "n" (OFFSETOF_CURPCB)); - return (pcb); -} -#define curpcb (__curpcb()) - #define IS_BSP() (PCPU_GET(cpuid) == 0) #else /* !__GNUCLIKE_ASM || !__GNUCLIKE___TYPEOF */ Index: sys/amd64/include/pcpu_aux.h =================================================================== --- /dev/null +++ sys/amd64/include/pcpu_aux.h @@ -0,0 +1,72 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_PCPU_AUX_H_ +#define _MACHINE_PCPU_AUX_H_ + +#ifndef _KERNEL +#error "Not for userspace" +#endif + +#ifndef _SYS_PCPU_H_ +#error "Do not include machine/pcpu_aux.h directly" +#endif + +/* Required for counters(9) to work on x86. */ +_Static_assert(sizeof(struct pcpu) == UMA_PCPU_ALLOC_SIZE, "fix pcpu size"); + +extern struct pcpu *__pcpu; +extern struct pcpu temp_bsp_pcpu; + +static __inline __pure2 struct thread * +__curthread(void) +{ + struct thread *td; + + __asm("movq %%gs:%P1,%0" : "=r" (td) : "n" (offsetof(struct pcpu, + pc_curthread))); + return (td); +} +#define curthread (__curthread()) + +static __inline __pure2 struct pcb * +__curpcb(void) +{ + struct pcb *pcb; + + __asm("movq %%gs:%P1,%0" : "=r" (pcb) : "n" (offsetof(struct pcpu, + pc_curpcb))); + return (pcb); +} +#define curpcb (__curpcb()) + +#endif /* _MACHINE_PCPU_AUX_H_ */ Index: sys/amd64/vmm/amd/svm.c =================================================================== --- sys/amd64/vmm/amd/svm.c +++ sys/amd64/vmm/amd/svm.c @@ -102,9 +102,6 @@ static MALLOC_DEFINE(M_SVM, "svm", "svm"); static MALLOC_DEFINE(M_SVM_VLAPIC, "svm-vlapic", "svm-vlapic"); -/* Per-CPU context area. */ -extern struct pcpu __pcpu[]; - static uint32_t svm_feature = ~0U; /* AMD SVM features. */ SYSCTL_UINT(_hw_vmm_svm, OID_AUTO, features, CTLFLAG_RDTUN, &svm_feature, 0, "SVM features advertised by CPUID.8000000AH:EDX"); Index: sys/amd64/vmm/vmm_host.h =================================================================== --- sys/amd64/vmm/vmm_host.h +++ sys/amd64/vmm/vmm_host.h @@ -72,9 +72,6 @@ return ((uint64_t)&gdt[NGDT * curcpu]); } -struct pcpu; -extern struct pcpu __pcpu[]; - static __inline uint64_t vmm_get_host_gsbase(void) { Index: sys/i386/i386/mp_machdep.c =================================================================== --- sys/i386/i386/mp_machdep.c +++ sys/i386/i386/mp_machdep.c @@ -130,8 +130,6 @@ #endif /* CHECK_POINTS */ -extern struct pcpu __pcpu[]; - /* * Local data and functions. */ Index: sys/i386/i386/vm_machdep.c =================================================================== --- sys/i386/i386/vm_machdep.c +++ sys/i386/i386/vm_machdep.c @@ -90,10 +90,6 @@ #define NSFBUFS (512 + maxusers * 16) #endif -_Static_assert(OFFSETOF_CURTHREAD == offsetof(struct pcpu, pc_curthread), - "OFFSETOF_CURTHREAD does not correspond with offset of pc_curthread."); -_Static_assert(OFFSETOF_CURPCB == offsetof(struct pcpu, pc_curpcb), - "OFFSETOF_CURPCB does not correspond with offset of pc_curpcb."); _Static_assert(__OFFSETOF_MONITORBUF == offsetof(struct pcpu, pc_monitorbuf), "__OFFSETOF_MONITORBUF does not correspond with offset of pc_monitorbuf."); Index: sys/i386/include/counter.h =================================================================== --- sys/i386/include/counter.h +++ sys/i386/include/counter.h @@ -38,8 +38,6 @@ #include #include -extern struct pcpu __pcpu[]; - #define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter #define counter_enter() do { \ Index: sys/i386/include/pcpu.h =================================================================== --- sys/i386/include/pcpu.h +++ sys/i386/include/pcpu.h @@ -228,36 +228,6 @@ #define PCPU_PTR(member) __PCPU_PTR(pc_ ## member) #define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val) -#define OFFSETOF_CURTHREAD 0 -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wnull-dereference" -#endif -static __inline __pure2 struct thread * -__curthread(void) -{ - struct thread *td; - - __asm("movl %%fs:%1,%0" : "=r" (td) - : "m" (*(char *)OFFSETOF_CURTHREAD)); - return (td); -} -#ifdef __clang__ -#pragma clang diagnostic pop -#endif -#define curthread (__curthread()) - -#define OFFSETOF_CURPCB 16 -static __inline __pure2 struct pcb * -__curpcb(void) -{ - struct pcb *pcb; - - __asm("movl %%fs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB)); - return (pcb); -} -#define curpcb (__curpcb()) - #define IS_BSP() (PCPU_GET(cpuid) == 0) #else /* defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */ Index: sys/i386/include/pcpu_aux.h =================================================================== --- /dev/null +++ sys/i386/include/pcpu_aux.h @@ -0,0 +1,71 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_PCPU_AUX_H_ +#define _MACHINE_PCPU_AUX_H_ + +#ifndef _KERNEL +#error "Not for userspace" +#endif + +#ifndef _SYS_PCPU_H_ +#error "Do not include machine/pcpu_aux.h directly" +#endif + +/* Required for counters(9) to work on x86. */ +_Static_assert(sizeof(struct pcpu) == UMA_PCPU_ALLOC_SIZE, "fix pcpu size"); + +extern struct pcpu __pcpu[]; + +static __inline __pure2 struct thread * +__curthread(void) +{ + struct thread *td; + + __asm("movl %%fs:%1,%0" : "=r" (td) + : "m" (*(char *)offsetof(struct pcpu, pc_curthread))); + return (td); +} +#define curthread (__curthread()) + +static __inline __pure2 struct pcb * +__curpcb(void) +{ + struct pcb *pcb; + + __asm("movl %%fs:%1,%0" : "=r" (pcb) + : "m" (*(char *)offsetof(struct pcpu, pc_curpcb))); + return (pcb); +} +#define curpcb (__curpcb()) + +#endif /* _MACHINE_PCPU_AUX_H_ */ Index: sys/riscv/include/counter.h =================================================================== --- sys/riscv/include/counter.h +++ sys/riscv/include/counter.h @@ -34,8 +34,6 @@ #include #endif -extern struct pcpu __pcpu[]; - #define EARLY_COUNTER &__pcpu[0].pc_early_dummy_counter #define counter_enter() critical_enter() Index: sys/riscv/include/pcpu_aux.h =================================================================== --- /dev/null +++ sys/riscv/include/pcpu_aux.h @@ -0,0 +1,52 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_PCPU_AUX_H_ +#define _MACHINE_PCPU_AUX_H_ + +#ifndef _KERNEL +#error "Not for userspace" +#endif + +#ifndef _SYS_PCPU_H_ +#error "Do not include machine/pcpu_aux.h directly" +#endif + +/* + * To minimize memory waste in per-cpu UMA zones, the page size should + * be a multiple of the size of struct pcpu. + */ +_Static_assert((PAGE_SIZE % sizeof(struct pcpu) == 0, "fix pcpu size"); + +extern struct pcpu __pcpu[]; + +#endif /* _MACHINE_PCPU_AUX_H_ */ Index: sys/riscv/riscv/mp_machdep.c =================================================================== --- sys/riscv/riscv/mp_machdep.c +++ sys/riscv/riscv/mp_machdep.c @@ -72,8 +72,6 @@ boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *); -extern struct pcpu __pcpu[]; - uint32_t __riscv_boot_ap[MAXCPU]; static enum { Index: sys/sys/pcpu.h =================================================================== --- sys/sys/pcpu.h +++ sys/sys/pcpu.h @@ -216,26 +216,16 @@ extern struct pcpu *cpuid_to_pcpu[]; #define curcpu PCPU_GET(cpuid) -#define curproc (curthread->td_proc) -#ifndef curthread -#define curthread PCPU_GET(curthread) -#endif #define curvidata PCPU_GET(vidata) #define UMA_PCPU_ALLOC_SIZE PAGE_SIZE -#ifdef CTASSERT -#if defined(__i386__) || defined(__amd64__) -/* Required for counters(9) to work on x86. */ -CTASSERT(sizeof(struct pcpu) == UMA_PCPU_ALLOC_SIZE); -#else -/* - * To minimize memory waste in per-cpu UMA zones, size of struct pcpu - * should be denominator of PAGE_SIZE. - */ -CTASSERT((PAGE_SIZE / sizeof(struct pcpu)) * sizeof(struct pcpu) == PAGE_SIZE); -#endif /* UMA_PCPU_ALLOC_SIZE && x86 */ -#endif /* CTASSERT */ +#include + +#ifndef curthread +#define curthread PCPU_GET(curthread) +#endif +#define curproc (curthread->td_proc) /* Accessor to elements allocated via UMA_ZONE_PCPU zone. */ static inline void *