diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index 6326fbdae0be..dc99d4249bd2 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -1,317 +1,288 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #ifndef _SYS_CDEFS_H_ #error "sys/cdefs.h is a prerequisite for this file" #endif #include #include #define PC_PTI_STACK_SZ 16 struct monitorbuf { int idle_state; /* Used by cpu_idle_mwait. */ int stop_state; /* Used by cpustop_handler. */ char padding[128 - (2 * sizeof(int))]; }; _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line"); /* * The SMP parts are setup in pmap.c and locore.s for the BSP, and * mp_machdep.c sets up the data for the AP's to "see" when they awake. * The reason for doing it via a struct is so that an array of pointers * to each CPU's data can be set up for things like "check curproc on all * other processors" */ #define PCPU_MD_FIELDS \ struct monitorbuf pc_monitorbuf __aligned(128); /* cache line */\ struct pcpu *pc_prvspace; /* Self-reference */ \ struct pmap *pc_curpmap; \ struct amd64tss *pc_tssp; /* TSS segment active on CPU */ \ void *pc_pad0; \ uint64_t pc_kcr3; \ uint64_t pc_ucr3; \ uint64_t pc_saved_ucr3; \ register_t pc_rsp0; \ register_t pc_scratch_rsp; /* User %rsp in syscall */ \ register_t pc_scratch_rax; \ u_int pc_apic_id; \ u_int pc_acpi_id; /* ACPI CPU id */ \ /* Pointer to the CPU %fs descriptor */ \ struct user_segment_descriptor *pc_fs32p; \ /* Pointer to the CPU %gs descriptor */ \ struct user_segment_descriptor *pc_gs32p; \ /* Pointer to the CPU LDT descriptor */ \ struct system_segment_descriptor *pc_ldt; \ /* Pointer to the CPU TSS descriptor */ \ struct system_segment_descriptor *pc_tss; \ u_int pc_cmci_mask; /* MCx banks for CMCI */ \ uint64_t pc_dbreg[16]; /* ddb debugging regs */ \ uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \ register_t pc_pti_rsp0; \ int pc_dbreg_cmd; /* ddb debugging reg cmd */ \ u_int pc_vcpu_id; /* Xen vCPU ID */ \ uint32_t pc_pcid_next; \ uint32_t pc_pcid_gen; \ uint32_t pc_unused; \ uint32_t pc_ibpb_set; \ void *pc_mds_buf; \ void *pc_mds_buf64; \ uint32_t pc_pad[4]; \ uint8_t pc_mds_tmp[64]; \ u_int pc_ipi_bitmap; \ struct amd64tss pc_common_tss; \ struct user_segment_descriptor pc_gdt[NGDT]; \ void *pc_smp_tlb_pmap; \ uint64_t pc_smp_tlb_addr1; \ uint64_t pc_smp_tlb_addr2; \ uint32_t pc_smp_tlb_gen; \ u_int pc_smp_tlb_op; \ uint64_t pc_ucr3_load_mask; \ char __pad[2916] /* pad to UMA_PCPU_ALLOC_SIZE */ #define PC_DBREG_CMD_NONE 0 #define PC_DBREG_CMD_LOAD 1 #ifdef _KERNEL #define MONITOR_STOPSTATE_RUNNING 0 #define MONITOR_STOPSTATE_STOPPED 1 #if defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) /* * Evaluates to the byte offset of the per-cpu variable name. */ #define __pcpu_offset(name) \ __offsetof(struct pcpu, name) /* * Evaluates to the type of the per-cpu variable name. */ #define __pcpu_type(name) \ __typeof(((struct pcpu *)0)->name) /* * Evaluates to the address of the per-cpu variable name. */ #define __PCPU_PTR(name) __extension__ ({ \ __pcpu_type(name) *__p; \ \ __asm __volatile("movq %%gs:%1,%0; addq %2,%0" \ : "=r" (__p) \ : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace))), \ "i" (__pcpu_offset(name))); \ \ __p; \ }) /* * Evaluates to the value of the per-cpu variable name. */ #define __PCPU_GET(name) __extension__ ({ \ __pcpu_type(name) __res; \ struct __s { \ u_char __b[MIN(sizeof(__pcpu_type(name)), 8)]; \ } __s; \ \ if (sizeof(__res) == 1 || sizeof(__res) == 2 || \ sizeof(__res) == 4 || sizeof(__res) == 8) { \ __asm __volatile("mov %%gs:%1,%0" \ : "=r" (__s) \ : "m" (*(struct __s *)(__pcpu_offset(name)))); \ *(struct __s *)(void *)&__res = __s; \ } else { \ __res = *__PCPU_PTR(name); \ } \ __res; \ }) /* * Adds the value to the per-cpu counter name. The implementation * must be atomic with respect to interrupts. */ #define __PCPU_ADD(name, val) do { \ __pcpu_type(name) __val; \ struct __s { \ u_char __b[MIN(sizeof(__pcpu_type(name)), 8)]; \ } __s; \ \ __val = (val); \ if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ sizeof(__val) == 4 || sizeof(__val) == 8) { \ __s = *(struct __s *)(void *)&__val; \ __asm __volatile("add %1,%%gs:%0" \ : "=m" (*(struct __s *)(__pcpu_offset(name))) \ : "r" (__s)); \ } else \ *__PCPU_PTR(name) += __val; \ } while (0) -/* - * Increments the value of the per-cpu counter name. The implementation - * must be atomic with respect to interrupts. - */ -#define __PCPU_INC(name) do { \ - CTASSERT(sizeof(__pcpu_type(name)) == 1 || \ - sizeof(__pcpu_type(name)) == 2 || \ - sizeof(__pcpu_type(name)) == 4 || \ - sizeof(__pcpu_type(name)) == 8); \ - if (sizeof(__pcpu_type(name)) == 1) { \ - __asm __volatile("incb %%gs:%0" \ - : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ - : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ - } else if (sizeof(__pcpu_type(name)) == 2) { \ - __asm __volatile("incw %%gs:%0" \ - : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ - : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ - } else if (sizeof(__pcpu_type(name)) == 4) { \ - __asm __volatile("incl %%gs:%0" \ - : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ - : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ - } else if (sizeof(__pcpu_type(name)) == 8) { \ - __asm __volatile("incq %%gs:%0" \ - : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ - : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ - } \ -} while (0) - /* * Sets the value of the per-cpu variable name to value val. */ #define __PCPU_SET(name, val) { \ __pcpu_type(name) __val; \ struct __s { \ u_char __b[MIN(sizeof(__pcpu_type(name)), 8)]; \ } __s; \ \ __val = (val); \ if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ sizeof(__val) == 4 || sizeof(__val) == 8) { \ __s = *(struct __s *)(void *)&__val; \ __asm __volatile("mov %1,%%gs:%0" \ : "=m" (*(struct __s *)(__pcpu_offset(name))) \ : "r" (__s)); \ } else { \ *__PCPU_PTR(name) = __val; \ } \ } #define get_pcpu() __extension__ ({ \ struct pcpu *__pc; \ \ __asm __volatile("movq %%gs:%1,%0" \ : "=r" (__pc) \ : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace)))); \ __pc; \ }) #define PCPU_GET(member) __PCPU_GET(pc_ ## member) #define PCPU_ADD(member, val) __PCPU_ADD(pc_ ## member, val) -#define PCPU_INC(member) __PCPU_INC(pc_ ## member) #define PCPU_PTR(member) __PCPU_PTR(pc_ ## member) #define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val) #define IS_BSP() (PCPU_GET(cpuid) == 0) #define zpcpu_offset_cpu(cpu) ((uintptr_t)&__pcpu[0] + UMA_PCPU_ALLOC_SIZE * cpu) #define zpcpu_base_to_offset(base) (void *)((uintptr_t)(base) - (uintptr_t)&__pcpu[0]) #define zpcpu_offset_to_base(base) (void *)((uintptr_t)(base) + (uintptr_t)&__pcpu[0]) #define zpcpu_sub_protected(base, n) do { \ ZPCPU_ASSERT_PROTECTED(); \ zpcpu_sub(base, n); \ } while (0) #define zpcpu_set_protected(base, n) do { \ __typeof(*base) __n = (n); \ ZPCPU_ASSERT_PROTECTED(); \ switch (sizeof(*base)) { \ case 4: \ __asm __volatile("movl\t%1,%%gs:(%0)" \ : : "r" (base), "ri" (__n) : "memory", "cc"); \ break; \ case 8: \ __asm __volatile("movq\t%1,%%gs:(%0)" \ : : "r" (base), "ri" (__n) : "memory", "cc"); \ break; \ default: \ *zpcpu_get(base) = __n; \ } \ } while (0); #define zpcpu_add(base, n) do { \ __typeof(*base) __n = (n); \ CTASSERT(sizeof(*base) == 4 || sizeof(*base) == 8); \ switch (sizeof(*base)) { \ case 4: \ __asm __volatile("addl\t%1,%%gs:(%0)" \ : : "r" (base), "ri" (__n) : "memory", "cc"); \ break; \ case 8: \ __asm __volatile("addq\t%1,%%gs:(%0)" \ : : "r" (base), "ri" (__n) : "memory", "cc"); \ break; \ } \ } while (0) #define zpcpu_add_protected(base, n) do { \ ZPCPU_ASSERT_PROTECTED(); \ zpcpu_add(base, n); \ } while (0) #define zpcpu_sub(base, n) do { \ __typeof(*base) __n = (n); \ CTASSERT(sizeof(*base) == 4 || sizeof(*base) == 8); \ switch (sizeof(*base)) { \ case 4: \ __asm __volatile("subl\t%1,%%gs:(%0)" \ : : "r" (base), "ri" (__n) : "memory", "cc"); \ break; \ case 8: \ __asm __volatile("subq\t%1,%%gs:(%0)" \ : : "r" (base), "ri" (__n) : "memory", "cc"); \ break; \ } \ } while (0); #else /* !__GNUCLIKE_ASM || !__GNUCLIKE___TYPEOF */ #error "this file needs to be ported to your compiler" #endif /* __GNUCLIKE_ASM && __GNUCLIKE___TYPEOF */ #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ diff --git a/sys/arm/include/pcpu.h b/sys/arm/include/pcpu.h index 4d609b10bf73..2353b1d06b17 100644 --- a/sys/arm/include/pcpu.h +++ b/sys/arm/include/pcpu.h @@ -1,146 +1,145 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1999 Luoqi Chen * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27 * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #ifdef _KERNEL #include #include #define ALT_STACK_SIZE 128 struct vmspace; #endif /* _KERNEL */ /* Branch predictor hardening method */ #define PCPU_BP_HARDEN_KIND_NONE 0 #define PCPU_BP_HARDEN_KIND_BPIALL 1 #define PCPU_BP_HARDEN_KIND_ICIALLU 2 #define PCPU_MD_FIELDS \ unsigned int pc_vfpsid; \ unsigned int pc_vfpmvfr0; \ unsigned int pc_vfpmvfr1; \ struct pmap *pc_curpmap; \ struct mtx pc_cmap_lock; \ void *pc_cmap1_pte2p; \ void *pc_cmap2_pte2p; \ caddr_t pc_cmap1_addr; \ caddr_t pc_cmap2_addr; \ vm_offset_t pc_qmap_addr; \ void *pc_qmap_pte2p; \ unsigned int pc_dbreg[32]; \ int pc_dbreg_cmd; \ int pc_bp_harden_kind; \ uint32_t pc_original_actlr; \ uint64_t pc_clock; \ uint32_t pc_mpidr; \ char __pad[135] #ifdef _KERNEL #define PC_DBREG_CMD_NONE 0 #define PC_DBREG_CMD_LOAD 1 struct pcb; struct pcpu; extern struct pcpu *pcpup; #define CPU_MASK (0xf) #ifndef SMP #define get_pcpu() (pcpup) #else #define get_pcpu() __extension__ ({ \ int id; \ __asm __volatile("mrc p15, 0, %0, c0, c0, 5" : "=r" (id)); \ (pcpup + (id & CPU_MASK)); \ }) #endif static inline struct thread * get_curthread(void) { void *ret; __asm __volatile("mrc p15, 0, %0, c13, c0, 4" : "=r" (ret)); return (ret); } static inline void set_curthread(struct thread *td) { __asm __volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (td)); } static inline void * get_tls(void) { void *tls; /* TPIDRURW contains the authoritative value. */ __asm __volatile("mrc p15, 0, %0, c13, c0, 2" : "=r" (tls)); return (tls); } static inline void set_tls(void *tls) { /* * Update both TPIDRURW and TPIDRURO. TPIDRURW needs to be written * first to ensure that a context switch between the two writes will * still give the desired result of updating both. */ __asm __volatile( "mcr p15, 0, %0, c13, c0, 2\n" "mcr p15, 0, %0, c13, c0, 3\n" : : "r" (tls)); } #define curthread get_curthread() #define PCPU_GET(member) (get_pcpu()->pc_ ## member) #define PCPU_ADD(member, value) (get_pcpu()->pc_ ## member += (value)) -#define PCPU_INC(member) PCPU_ADD(member, 1) #define PCPU_PTR(member) (&get_pcpu()->pc_ ## member) #define PCPU_SET(member,value) (get_pcpu()->pc_ ## member = (value)) void pcpu0_init(void); #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h index ce4dc46e1609..09f6361c651c 100644 --- a/sys/arm64/include/pcpu.h +++ b/sys/arm64/include/pcpu.h @@ -1,82 +1,81 @@ /*- * Copyright (c) 1999 Luoqi Chen * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27 * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #include #include #define ALT_STACK_SIZE 128 typedef int (*pcpu_bp_harden)(void); typedef int (*pcpu_ssbd)(int); struct debug_monitor_state; #define PCPU_MD_FIELDS \ u_int pc_acpi_id; /* ACPI CPU id */ \ u_int pc_midr; /* stored MIDR value */ \ uint64_t pc_clock; \ pcpu_bp_harden pc_bp_harden; \ pcpu_ssbd pc_ssbd; \ struct pmap *pc_curpmap; \ struct pmap *pc_curvmpmap; \ u_int pc_bcast_tlbi_workaround; \ u_int pc_mpidr; /* stored MPIDR value */ \ char __pad[201] #ifdef _KERNEL struct pcb; struct pcpu; register struct pcpu *pcpup __asm ("x18"); #define get_pcpu() pcpup static inline struct thread * get_curthread(void) { struct thread *td; __asm __volatile("ldr %0, [x18]" : "=&r"(td)); return (td); } #define curthread get_curthread() #define PCPU_GET(member) (pcpup->pc_ ## member) #define PCPU_ADD(member, value) (pcpup->pc_ ## member += (value)) -#define PCPU_INC(member) PCPU_ADD(member, 1) #define PCPU_PTR(member) (&pcpup->pc_ ## member) #define PCPU_SET(member,value) (pcpup->pc_ ## member = (value)) #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h index 9e65d179dc36..a4c7968ea85f 100644 --- a/sys/i386/include/pcpu.h +++ b/sys/i386/include/pcpu.h @@ -1,241 +1,217 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #ifndef _SYS_CDEFS_H_ #error "sys/cdefs.h is a prerequisite for this file" #endif #include #include #include #include struct monitorbuf { int idle_state; /* Used by cpu_idle_mwait. */ int stop_state; /* Used by cpustop_handler. */ char padding[128 - (2 * sizeof(int))]; }; _Static_assert(sizeof(struct monitorbuf) == 128, "2x cache line"); /* * The SMP parts are setup in pmap.c and machdep.c for the BSP, and * pmap.c and mp_machdep.c sets up the data for the AP's to "see" when * they awake. The reason for doing it via a struct is so that an * array of pointers to each CPU's data can be set up for things like * "check curproc on all other processors" */ #define PCPU_MD_FIELDS \ struct monitorbuf pc_monitorbuf __aligned(128); /* cache line */\ struct pcpu *pc_prvspace; /* Self-reference */ \ struct pmap *pc_curpmap; \ struct segment_descriptor pc_common_tssd; \ struct segment_descriptor *pc_tss_gdt; \ struct segment_descriptor *pc_fsgs_gdt; \ struct i386tss *pc_common_tssp; \ u_int pc_kesp0; \ u_int pc_trampstk; \ int pc_currentldt; \ u_int pc_acpi_id; /* ACPI CPU id */ \ u_int pc_apic_id; \ int pc_private_tss; /* Flag indicating private tss*/\ u_int pc_cmci_mask; /* MCx banks for CMCI */ \ u_int pc_vcpu_id; /* Xen vCPU ID */ \ struct mtx pc_cmap_lock; \ void *pc_cmap_pte1; \ void *pc_cmap_pte2; \ caddr_t pc_cmap_addr1; \ caddr_t pc_cmap_addr2; \ vm_offset_t pc_qmap_addr; /* KVA for temporary mappings */\ vm_offset_t pc_copyout_maddr; \ vm_offset_t pc_copyout_saddr; \ struct mtx pc_copyout_mlock; \ struct sx pc_copyout_slock; \ char *pc_copyout_buf; \ vm_offset_t pc_pmap_eh_va; \ caddr_t pc_pmap_eh_ptep; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ uint32_t pc_ibpb_set; \ void *pc_mds_buf; \ void *pc_mds_buf64; \ uint32_t pc_pad[4]; \ uint8_t pc_mds_tmp[64]; \ u_int pc_ipi_bitmap; \ char __pad[3518] #ifdef _KERNEL #define MONITOR_STOPSTATE_RUNNING 0 #define MONITOR_STOPSTATE_STOPPED 1 #if defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) /* * Evaluates to the byte offset of the per-cpu variable name. */ #define __pcpu_offset(name) \ __offsetof(struct pcpu, name) /* * Evaluates to the type of the per-cpu variable name. */ #define __pcpu_type(name) \ __typeof(((struct pcpu *)0)->name) /* * Evaluates to the address of the per-cpu variable name. */ #define __PCPU_PTR(name) __extension__ ({ \ __pcpu_type(name) *__p; \ \ __asm __volatile("movl %%fs:%1,%0; addl %2,%0" \ : "=r" (__p) \ : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace))), \ "i" (__pcpu_offset(name))); \ \ __p; \ }) /* * Evaluates to the value of the per-cpu variable name. */ #define __PCPU_GET(name) __extension__ ({ \ __pcpu_type(name) __res; \ struct __s { \ u_char __b[MIN(sizeof(__res), 4)]; \ } __s; \ \ if (sizeof(__res) == 1 || sizeof(__res) == 2 || \ sizeof(__res) == 4) { \ __asm __volatile("mov %%fs:%1,%0" \ : "=r" (__s) \ : "m" (*(struct __s *)(__pcpu_offset(name)))); \ *(struct __s *)(void *)&__res = __s; \ } else { \ __res = *__PCPU_PTR(name); \ } \ __res; \ }) /* * Adds a value of the per-cpu counter name. The implementation * must be atomic with respect to interrupts. */ #define __PCPU_ADD(name, val) do { \ __pcpu_type(name) __val; \ struct __s { \ u_char __b[MIN(sizeof(__val), 4)]; \ } __s; \ \ __val = (val); \ if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ sizeof(__val) == 4) { \ __s = *(struct __s *)(void *)&__val; \ __asm __volatile("add %1,%%fs:%0" \ : "=m" (*(struct __s *)(__pcpu_offset(name))) \ : "r" (__s)); \ } else \ *__PCPU_PTR(name) += __val; \ } while (0) -/* - * Increments the value of the per-cpu counter name. The implementation - * must be atomic with respect to interrupts. - */ -#define __PCPU_INC(name) do { \ - CTASSERT(sizeof(__pcpu_type(name)) == 1 || \ - sizeof(__pcpu_type(name)) == 2 || \ - sizeof(__pcpu_type(name)) == 4); \ - if (sizeof(__pcpu_type(name)) == 1) { \ - __asm __volatile("incb %%fs:%0" \ - : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ - : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ - } else if (sizeof(__pcpu_type(name)) == 2) { \ - __asm __volatile("incw %%fs:%0" \ - : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ - : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ - } else if (sizeof(__pcpu_type(name)) == 4) { \ - __asm __volatile("incl %%fs:%0" \ - : "=m" (*(__pcpu_type(name) *)(__pcpu_offset(name)))\ - : "m" (*(__pcpu_type(name) *)(__pcpu_offset(name))));\ - } \ -} while (0) - /* * Sets the value of the per-cpu variable name to value val. */ #define __PCPU_SET(name, val) do { \ __pcpu_type(name) __val; \ struct __s { \ u_char __b[MIN(sizeof(__val), 4)]; \ } __s; \ \ __val = (val); \ if (sizeof(__val) == 1 || sizeof(__val) == 2 || \ sizeof(__val) == 4) { \ __s = *(struct __s *)(void *)&__val; \ __asm __volatile("mov %1,%%fs:%0" \ : "=m" (*(struct __s *)(__pcpu_offset(name))) \ : "r" (__s)); \ } else { \ *__PCPU_PTR(name) = __val; \ } \ } while (0) #define get_pcpu() __extension__ ({ \ struct pcpu *__pc; \ \ __asm __volatile("movl %%fs:%1,%0" \ : "=r" (__pc) \ : "m" (*(struct pcpu *)(__pcpu_offset(pc_prvspace)))); \ __pc; \ }) #define PCPU_GET(member) __PCPU_GET(pc_ ## member) #define PCPU_ADD(member, val) __PCPU_ADD(pc_ ## member, val) -#define PCPU_INC(member) __PCPU_INC(pc_ ## member) #define PCPU_PTR(member) __PCPU_PTR(pc_ ## member) #define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val) #define IS_BSP() (PCPU_GET(cpuid) == 0) #else /* defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */ #error "this file needs to be ported to your compiler" #endif /* __GNUCLIKE_ASM etc. */ #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ diff --git a/sys/mips/include/pcpu.h b/sys/mips/include/pcpu.h index fea594729e90..879f2d8afb90 100644 --- a/sys/mips/include/pcpu.h +++ b/sys/mips/include/pcpu.h @@ -1,100 +1,99 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1999 Luoqi Chen * Copyright (c) Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: src/sys/alpha/include/pcpu.h,v 1.15 2004/11/05 19:16:44 jhb * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #include #include #define PCPU_MD_COMMON_FIELDS \ pd_entry_t *pc_segbase; /* curthread segbase */ \ struct pmap *pc_curpmap; /* pmap of curthread */ \ u_int32_t pc_next_asid; /* next ASID to alloc */ \ u_int32_t pc_asid_generation; /* current ASID generation */ \ u_int pc_pending_ipis; /* IPIs pending to this CPU */ \ struct pcpu *pc_self; /* globally-uniqe self pointer */ #ifdef __mips_n64 #define PCPU_MD_MIPS64_FIELDS \ PCPU_MD_COMMON_FIELDS \ char __pad[245] #else #define PCPU_MD_MIPS32_FIELDS \ PCPU_MD_COMMON_FIELDS \ pt_entry_t *pc_cmap1_ptep; /* PTE for copy window 1 KVA */ \ pt_entry_t *pc_cmap2_ptep; /* PTE for copy window 2 KVA */ \ vm_offset_t pc_cmap1_addr; /* KVA page for copy window 1 */ \ vm_offset_t pc_cmap2_addr; /* KVA page for copy window 2 */ \ vm_offset_t pc_qmap_addr; /* KVA page for temporary mappings */ \ pt_entry_t *pc_qmap_ptep; /* PTE for temporary mapping KVA */ \ char __pad[97] #endif #ifdef __mips_n64 #define PCPU_MD_FIELDS PCPU_MD_MIPS64_FIELDS #else #define PCPU_MD_FIELDS PCPU_MD_MIPS32_FIELDS #endif #ifdef _KERNEL extern char pcpu_space[MAXCPU][PAGE_SIZE * 2]; #define PCPU_ADDR(cpu) (struct pcpu *)(pcpu_space[(cpu)]) extern struct pcpu *pcpup; #define PCPUP pcpup /* * Since we use a wired TLB entry to map the same VA to a different * physical page for each CPU, get_pcpu() must use the pc_self * field to obtain a globally-unique pointer. */ #define get_pcpu() (PCPUP->pc_self) #define PCPU_ADD(member, value) (PCPUP->pc_ ## member += (value)) #define PCPU_GET(member) (PCPUP->pc_ ## member) -#define PCPU_INC(member) PCPU_ADD(member, 1) #define PCPU_PTR(member) (&PCPUP->pc_ ## member) #define PCPU_SET(member,value) (PCPUP->pc_ ## member = (value)) #define PCPU_LAZY_INC(member) (++PCPUP->pc_ ## member) #ifdef SMP /* * Instantiate the wired TLB entry at PCPU_TLB_ENTRY to map 'pcpu' at 'pcpup'. */ void mips_pcpu_tlb_init(struct pcpu *pcpu); #endif #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h index 31c4b2c47c70..288e12c1f2ab 100644 --- a/sys/powerpc/include/pcpu.h +++ b/sys/powerpc/include/pcpu.h @@ -1,177 +1,176 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1999 Luoqi Chen * Copyright (c) Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #include #include #include struct pmap; struct pvo_entry; #define CPUSAVE_LEN 9 #define PCPU_MD_COMMON_FIELDS \ int pc_inside_intr; \ struct pmap *pc_curpmap; /* current pmap */ \ struct thread *pc_fputhread; /* current fpu user */ \ struct thread *pc_vecthread; /* current vec user */ \ struct thread *pc_htmthread; /* current htm user */ \ uintptr_t pc_hwref; \ int pc_bsp; \ volatile int pc_awake; \ uint32_t pc_ipimask; \ uint32_t pc_flags; /* cpu feature flags */ \ register_t pc_tempsave[CPUSAVE_LEN]; \ register_t pc_disisave[CPUSAVE_LEN]; \ register_t pc_dbsave[CPUSAVE_LEN]; \ void *pc_restore; \ vm_offset_t pc_qmap_addr; #define PCPU_MD_AIM32_FIELDS \ struct pvo_entry *qmap_pvo; \ struct mtx qmap_lock; \ char __pad[128]; #define PCPU_MD_AIM64_FIELDS \ struct slb slb[64]; \ struct slb **userslb; \ register_t slbsave[18]; \ uint8_t slbstack[1024]; \ struct pvo_entry *qmap_pvo; \ struct mtx qmap_lock; \ uint64_t opal_hmi_flags; \ char __pad[1337]; #ifdef __powerpc64__ #define PCPU_MD_AIM_FIELDS PCPU_MD_AIM64_FIELDS #else #define PCPU_MD_AIM_FIELDS PCPU_MD_AIM32_FIELDS #endif /* CPU feature flags, can be used for cached flow control. */ #define PC_FLAG_NOSRS 0x80000000 #define BOOKE_CRITSAVE_LEN (CPUSAVE_LEN + 2) #define BOOKE_TLB_MAXNEST 4 #define BOOKE_TLB_SAVELEN 16 #define BOOKE_TLBSAVE_LEN (BOOKE_TLB_SAVELEN * BOOKE_TLB_MAXNEST) #ifdef __powerpc64__ #define BOOKE_PCPU_PAD 901 #else #define BOOKE_PCPU_PAD 365 #endif #define PCPU_MD_BOOKE_FIELDS \ register_t critsave[BOOKE_CRITSAVE_LEN]; \ register_t mchksave[CPUSAVE_LEN]; \ register_t tlbsave[BOOKE_TLBSAVE_LEN]; \ register_t tlb_level; \ uintptr_t *tlb_lock; \ int tid_next; \ char __pad[BOOKE_PCPU_PAD]; /* Definitions for register offsets within the exception tmp save areas */ #define CPUSAVE_R27 0 /* where r27 gets saved */ #define CPUSAVE_R28 1 /* where r28 gets saved */ #define CPUSAVE_R29 2 /* where r29 gets saved */ #define CPUSAVE_R30 3 /* where r30 gets saved */ #define CPUSAVE_R31 4 /* where r31 gets saved */ #define CPUSAVE_AIM_DAR 5 /* where SPR_DAR gets saved */ #define CPUSAVE_AIM_DSISR 6 /* where SPR_DSISR gets saved */ #define CPUSAVE_BOOKE_DEAR 5 /* where SPR_DEAR gets saved */ #define CPUSAVE_BOOKE_ESR 6 /* where SPR_ESR gets saved */ #define CPUSAVE_SRR0 7 /* where SRR0 gets saved */ #define CPUSAVE_SRR1 8 /* where SRR1 gets saved */ #define BOOKE_CRITSAVE_SRR0 9 /* where real SRR0 gets saved (critical) */ #define BOOKE_CRITSAVE_SRR1 10 /* where real SRR0 gets saved (critical) */ /* Book-E TLBSAVE is more elaborate */ #define TLBSAVE_BOOKE_LR 0 #define TLBSAVE_BOOKE_CR 1 #define TLBSAVE_BOOKE_SRR0 2 #define TLBSAVE_BOOKE_SRR1 3 #define TLBSAVE_BOOKE_R20 4 #define TLBSAVE_BOOKE_R21 5 #define TLBSAVE_BOOKE_R22 6 #define TLBSAVE_BOOKE_R23 7 #define TLBSAVE_BOOKE_R24 8 #define TLBSAVE_BOOKE_R25 9 #define TLBSAVE_BOOKE_R26 10 #define TLBSAVE_BOOKE_R27 11 #define TLBSAVE_BOOKE_R28 12 #define TLBSAVE_BOOKE_R29 13 #define TLBSAVE_BOOKE_R30 14 #define TLBSAVE_BOOKE_R31 15 #define PCPU_MD_FIELDS \ PCPU_MD_COMMON_FIELDS \ union { \ struct { \ PCPU_MD_AIM_FIELDS \ } pc_aim; \ struct { \ PCPU_MD_BOOKE_FIELDS \ } pc_booke; \ } #ifdef _KERNEL #define pcpup (get_pcpu()) static __inline __pure2 struct thread * __curthread(void) { struct thread *td; #ifdef __powerpc64__ __asm __volatile("mr %0,13" : "=r"(td)); #else __asm __volatile("mr %0,2" : "=r"(td)); #endif return (td); } #define curthread (__curthread()) #define PCPU_GET(member) (pcpup->pc_ ## member) /* * XXX The implementation of this operation should be made atomic * with respect to preemption. */ #define PCPU_ADD(member, value) (pcpup->pc_ ## member += (value)) -#define PCPU_INC(member) PCPU_ADD(member, 1) #define PCPU_PTR(member) (&pcpup->pc_ ## member) #define PCPU_SET(member,value) (pcpup->pc_ ## member = (value)) #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */ diff --git a/sys/riscv/include/pcpu.h b/sys/riscv/include/pcpu.h index 5068596462fc..af5eb6fd5c41 100644 --- a/sys/riscv/include/pcpu.h +++ b/sys/riscv/include/pcpu.h @@ -1,88 +1,87 @@ /*- * Copyright (c) 1999 Luoqi Chen * Copyright (c) 2015-2016 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27 * $FreeBSD$ */ #ifndef _MACHINE_PCPU_H_ #define _MACHINE_PCPU_H_ #include #include #define ALT_STACK_SIZE 128 #define PCPU_MD_FIELDS \ struct pmap *pc_curpmap; /* Currently active pmap */ \ uint32_t pc_pending_ipis; /* IPIs pending to this CPU */ \ uint32_t pc_hart; /* Hart ID */ \ char __pad[56] /* Pad to factor of PAGE_SIZE */ #ifdef _KERNEL struct pcb; struct pcpu; static inline struct pcpu * get_pcpu(void) { struct pcpu *pcpu; __asm __volatile("mv %0, tp" : "=&r"(pcpu)); return (pcpu); } static inline struct thread * get_curthread(void) { struct thread *td; __asm __volatile("ld %0, 0(tp)" : "=&r"(td)); return (td); } #define curthread get_curthread() #define PCPU_GET(member) (get_pcpu()->pc_ ## member) #define PCPU_ADD(member, value) (get_pcpu()->pc_ ## member += (value)) -#define PCPU_INC(member) PCPU_ADD(member, 1) #define PCPU_PTR(member) (&get_pcpu()->pc_ ## member) #define PCPU_SET(member,value) (get_pcpu()->pc_ ## member = (value)) #endif /* _KERNEL */ #endif /* !_MACHINE_PCPU_H_ */