Index: sys/amd64/include/cpufunc.h =================================================================== --- sys/amd64/include/cpufunc.h +++ sys/amd64/include/cpufunc.h @@ -41,6 +41,8 @@ #ifndef _MACHINE_CPUFUNC_H_ #define _MACHINE_CPUFUNC_H_ +#define CRITICAL_INLINE + #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif Index: sys/kern/kern_switch.c =================================================================== --- sys/kern/kern_switch.c +++ sys/kern/kern_switch.c @@ -198,18 +198,13 @@ * penalty imposed by the membar could, then, produce slower code than * the function call itself, for most cases. */ -void -critical_enter(void) -{ - struct thread *td; - td = curthread; - td->td_critnest++; - CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, - (long)td->td_proc->p_pid, td->td_name, td->td_critnest); -} +#ifndef CRITICAL_INLINE static void __noinline +#else +void +#endif critical_exit_preempt(void) { struct thread *td; @@ -240,6 +235,18 @@ thread_unlock(td); } +#ifndef CRITICAL_INLINE +void +critical_enter(void) +{ + struct thread *td; + + td = curthread; + td->td_critnest++; + CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td, + (long)td->td_proc->p_pid, td->td_name, td->td_critnest); +} + void critical_exit(void) { @@ -255,6 +262,7 @@ CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td, (long)td->td_proc->p_pid, td->td_name, td->td_critnest); } +#endif /************************************************************************ * SYSTEM RUN QUEUE manipulations and tests * Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c +++ sys/kern/kern_thread.c @@ -77,13 +77,13 @@ * structures. */ #ifdef __amd64__ -_Static_assert(offsetof(struct thread, td_flags) == 0xfc, +_Static_assert(offsetof(struct thread, td_flags) == 0x10c, "struct thread KBI td_flags"); -_Static_assert(offsetof(struct thread, td_pflags) == 0x104, +_Static_assert(offsetof(struct thread, td_pflags) == 0x114, "struct thread KBI td_pflags"); -_Static_assert(offsetof(struct thread, td_frame) == 0x470, +_Static_assert(offsetof(struct thread, td_frame) == 0x478, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x518, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x520, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0xb0, "struct proc KBI p_flag"); @@ -97,13 +97,13 @@ "struct proc KBI p_emuldata"); #endif #ifdef __i386__ -_Static_assert(offsetof(struct thread, td_flags) == 0x98, +_Static_assert(offsetof(struct thread, td_flags) == 0xa8, "struct thread KBI td_flags"); -_Static_assert(offsetof(struct thread, td_pflags) == 0xa0, +_Static_assert(offsetof(struct thread, td_pflags) == 0xb0, "struct thread KBI td_pflags"); -_Static_assert(offsetof(struct thread, td_frame) == 0x2e8, +_Static_assert(offsetof(struct thread, td_frame) == 0x2f0, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x334, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x338, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0x68, "struct proc KBI p_flag"); @@ -117,6 +117,15 @@ "struct proc KBI p_emuldata"); #endif +#ifdef CRITICAL_INLINE +#include + +CTASSERT(offsetof(struct thread, td_owepreempt) == + offsetof(struct thread_lite, td_owepreempt)); +CTASSERT(offsetof(struct thread, td_critnest) == + offsetof(struct thread_lite, td_critnest)); +#endif + SDT_PROVIDER_DECLARE(proc); SDT_PROBE_DEFINE(proc, , , lwp__exit); Index: sys/sys/proc.h =================================================================== --- sys/sys/proc.h +++ sys/sys/proc.h @@ -254,8 +254,15 @@ u_char td_lend_user_pri; /* (t) Lend user pri. */ /* Cleared during fork1() */ + volatile u_char td_owepreempt; /* (k*) Preempt on last critical_exit */ + volatile u_int td_critnest; /* (k*) Critical section nest level. */ #define td_startzero td_epochnest u_char td_epochnest; /* (k) Epoch nest counter. */ + short td_locks; /* (k) Debug: count of non-spin locks */ + short td_rw_rlocks; /* (k) Count of rwlock read locks. */ + short td_sx_slocks; /* (k) Count of sx shared locks. */ + short td_lk_slocks; /* (k) Count of lockmgr shared locks. */ + short td_stopsched; /* (k) Scheduler stopped. */ int td_flags; /* (t) TDF_* flags. */ int td_inhibitors; /* (t) Why can not run. */ int td_pflags; /* (k) Private thread (TDP_*) flags. */ @@ -263,13 +270,7 @@ int td_sqqueue; /* (t) Sleepqueue queue blocked on. */ void *td_wchan; /* (t) Sleep address. */ const char *td_wmesg; /* (t) Reason for sleep. */ - volatile u_char td_owepreempt; /* (k*) Preempt on last critical_exit */ u_char td_tsqueue; /* (t) Turnstile queue blocked on. */ - short td_locks; /* (k) Debug: count of non-spin locks */ - short td_rw_rlocks; /* (k) Count of rwlock read locks. */ - short td_sx_slocks; /* (k) Count of sx shared locks. */ - short td_lk_slocks; /* (k) Count of lockmgr shared locks. */ - short td_stopsched; /* (k) Scheduler stopped. */ struct turnstile *td_blocked; /* (t) Lock thread is blocked on. */ const char *td_lockname; /* (t) Name of lock blocked on. */ LIST_HEAD(, turnstile) td_contested; /* (q) Contested locks. */ @@ -355,7 +356,6 @@ struct vm_object *td_kstack_obj;/* (a) Kstack object. */ vm_offset_t td_kstack; /* (a) Kernel VA of kstack. */ int td_kstack_pages; /* (a) Size of the kstack. */ - volatile u_int td_critnest; /* (k*) Critical section nest level. */ struct mdthread td_md; /* (k) Any machine-dependent fields. */ struct kaudit_record *td_ar; /* (k) Active audit record, if any. */ struct lpohead td_lprof[2]; /* (a) lock profiling objects. */ Index: sys/sys/systm.h =================================================================== --- sys/sys/systm.h +++ sys/sys/systm.h @@ -46,6 +46,9 @@ #include #include #include /* for people using printf mainly */ +#ifdef CRITICAL_INLINE +#include +#endif __NULLABILITY_PRAGMA_PUSH @@ -214,8 +217,36 @@ void cpu_boot(int); void cpu_flush_dcache(void *, size_t); void cpu_rootconf(void); -void critical_enter(void); -void critical_exit(void); + +#ifdef CRITICAL_INLINE +void critical_exit_preempt(void); + +static __inline void +critical_enter(void) +{ + + curthread_lite->td_critnest++; + __compiler_membar(); +} + +static __inline void +critical_exit(void) +{ + struct thread_lite *td; + + td = curthread_lite; + KASSERT(td->td_critnest != 0, + ("critical_exit: td_critnest == 0")); + td->td_critnest--; + __compiler_membar(); + if (__predict_false(td->td_owepreempt)) + critical_exit_preempt(); +} +#else +void critical_enter(void); +void critical_exit(void); +#endif + void init_param1(void); void init_param2(long physpages); void init_static_kenv(char *, size_t); Index: sys/sys/thread_lite.h =================================================================== --- /dev/null +++ sys/sys/thread_lite.h @@ -0,0 +1,19 @@ +#ifndef _SYS_THREAD_LITE_H_ +#define _SYS_THREAD_LITE_H_ + +#include + +#ifdef _LP64 +#define TD_OWEPREEMPT_OFFSET 0xf9 +#else +#define TD_OWEPREEMPT_OFFSET 0x95 +#endif + +struct thread_lite { + char pad[TD_OWEPREEMPT_OFFSET]; + u_char td_owepreempt; + int td_critnest; +}; + +#define curthread_lite ((struct thread_lite *)curthread) +#endif