Index: sys/amd64/include/cpufunc.h =================================================================== --- sys/amd64/include/cpufunc.h +++ sys/amd64/include/cpufunc.h @@ -489,6 +489,21 @@ return (data); } +static __inline void +load_cr8(u_long data) +{ + __asm __volatile("movq %0,%%cr8" : : "r" (data)); +} + +static __inline u_long +rcr8(void) +{ + u_long data; + + __asm __volatile("movq %%cr8,%0" : "=r" (data)); + return (data); +} + static __inline u_long rxcr(u_int reg) { @@ -999,6 +1014,7 @@ void load_cr0(u_long cr0); void load_cr3(u_long cr3); void load_cr4(u_long cr4); +void load_cr8(u_long cr8); void load_dr0(uint64_t dr0); void load_dr1(uint64_t dr1); void load_dr2(uint64_t dr2); @@ -1018,6 +1034,7 @@ u_long rcr2(void); u_long rcr3(void); u_long rcr4(void); +u_long rcr8(void); uint64_t rdmsr(u_int msr); uint32_t rdmsr32(u_int msr); uint64_t rdpmc(u_int pmc); Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3779,6 +3779,7 @@ kern/linker_if.m standard kern/md4c.c optional netsmb kern/md5c.c standard +kern/mutex_irql.c standard kern/p1003_1b.c standard kern/posix4_mib.c standard kern/sched_4bsd.c optional sched_4bsd Index: sys/kern/mutex_irql.c =================================================================== --- /dev/null +++ sys/kern/mutex_irql.c @@ -0,0 +1,128 @@ +#include + +#include +#include +#include + +#define X86_EFLAGS_IF 0x200 + +// ----------------------------------------------------------------------------- +void mtx_init_irql(struct mtx_irql* mtx, const char* name, uint32_t irql) +{ + mtx->irql = irql; + mtx->iflag_n_irql = 0; + mtx->lock = 0; + mtx->name = name; +} + +// ----------------------------------------------------------------------------- +void mtx_destroy_irql(struct mtx_irql* mtx) +{ + mtx->name = 0; +} + +// ----------------------------------------------------------------------------- +void mtx_lock_irql(struct mtx_irql* mtx) +{ + // reset flags + uint32_t iflag_n_irql = 0; + + // interrupts already disabled? + register_t eFlags = read_rflags(); + if (eFlags & X86_EFLAGS_IF) + { + // no, then raise irql + u_int irql = rcr8(); + if (irql < mtx->irql) + { + load_cr8(mtx->irql); + iflag_n_irql = X86_EFLAGS_IF | (irql & 0xF); + } + } + + // spin on lock variable + while (atomic_swap_long(&(mtx->lock), 1)) + { + ia32_pause(); + } + + // save flags + mtx->iflag_n_irql = iflag_n_irql; +} + + +// ----------------------------------------------------------------------------- +void mtx_unlock_irql(struct mtx_irql* mtx) +{ + // get flags and delete + uint32_t iflag_n_irql = mtx->iflag_n_irql; + mtx->iflag_n_irql = 0; + + // release lock variable + atomic_store_long(&(mtx->lock), 0); + + // restore irql if "IF" flags is set + if (iflag_n_irql & X86_EFLAGS_IF) + { + load_cr8(iflag_n_irql & 0xF); + } +} + + +// ----------------------------------------------------------------------------- + +// ----------------------------------------------------------------------------- +void mtx_init_un(struct mtx_un* mtx, const char* name, uint32_t irql) +{ + if (irql == 0) + { + mtx->isbsd = 1; + mtx_init(&(mtx->mtx_bsd), name, 0, MTX_SPIN); + } + else + { + mtx->isbsd = 0; + mtx_init_irql(&(mtx->mtx_irql), name, irql); + } +} + +// ----------------------------------------------------------------------------- +void mtx_destroy_un(struct mtx_un* mtx) +{ + if (mtx->isbsd != 0) + mtx_destroy(&(mtx->mtx_bsd)); + else + mtx_destroy_irql(&(mtx->mtx_irql)); +} + +// ----------------------------------------------------------------------------- +void mtx_lock_un(struct mtx_un* mtx) +{ + if (mtx->isbsd != 0) + mtx_lock_spin(&(mtx->mtx_bsd)); + else + mtx_lock_irql(&(mtx->mtx_irql)); +} + +// ----------------------------------------------------------------------------- +void mtx_unlock_un(struct mtx_un* mtx) +{ + if (mtx->isbsd != 0) + mtx_unlock_spin(&(mtx->mtx_bsd)); + else + mtx_unlock_irql(&(mtx->mtx_irql)); +} + +// ----------------------------------------------------------------------------- +void mtx_sysinit_un(const void* udata) +{ + const struct mtx_un_args* pArgs = (const struct mtx_un_args*) udata; + mtx_init_un(pArgs->ma_mtx, pArgs->ma_name, pArgs->ma_irql); +} + +// ----------------------------------------------------------------------------- +void mtx_sysuninit_un(const void* udata) +{ + const struct mtx_un_args* pArgs = (const struct mtx_un_args*) udata; + mtx_destroy_un(pArgs->ma_mtx); +} Index: sys/sys/mutex_irql.h =================================================================== --- /dev/null +++ sys/sys/mutex_irql.h @@ -0,0 +1,69 @@ +/* + * + */ +#include + +#include +#include +#include +#include + +// interrupt level for X86/X64 with RT environment +#define MTX_IRQL_RT_LEVEL 0xE + +// ----------------------------------------------------------------------------- +struct mtx_irql +{ + uint32_t irql; + volatile uint32_t iflag_n_irql; + volatile u_long lock; + const char* name; +}; + +// ----------------------------------------------------------------------------- +void mtx_init_irql(struct mtx_irql* mtx, const char* name, uint32_t irql); +void mtx_destroy_irql(struct mtx_irql* mtx); +void mtx_lock_irql(struct mtx_irql* mtx); +void mtx_unlock_irql(struct mtx_irql* mtx); + +// ----------------------------------------------------------------------------- +struct mtx_un +{ + uint32_t isbsd; + union + { + struct mtx mtx_bsd; + struct mtx_irql mtx_irql; + }; +}; + +// ----------------------------------------------------------------------------- +void mtx_init_un(struct mtx_un* mtx, const char* name, uint32_t irql); +void mtx_destroy_un(struct mtx_un* mtx); +void mtx_lock_un(struct mtx_un* mtx); +void mtx_unlock_un(struct mtx_un* mtx); + +// ----------------------------------------------------------------------------- +void mtx_sysinit_un(const void* udata); +void mtx_sysuninit_un(const void* udata); + +// ----------------------------------------------------------------------------- +struct mtx_un_args +{ + struct mtx_un* ma_mtx; + const char* ma_name; + uint32_t ma_irql; +}; + +#define MTX_UN_SYSINIT(name, mtx, desc, irql) \ + static struct mtx_un_args name##_args = { \ + (mtx), \ + (desc), \ + (irql) \ + }; \ + SYSINIT(name##_mtx_un_sysinit, \ + SI_SUB_LOCK, SI_ORDER_MIDDLE, \ + mtx_sysinit_un, &name##_args); \ + SYSUNINIT(name##_mtx_un_sysuninit, \ + SI_SUB_LOCK, SI_ORDER_MIDDLE, \ + mtx_sysuninit_un, &name##_args) Index: sys/x86/isa/atrtc.c =================================================================== --- sys/x86/isa/atrtc.c +++ sys/x86/isa/atrtc.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -62,13 +63,35 @@ #include #endif +/* tunable to configure lock below to use irq-level instead of disable-int */ +static int atrtc_use_high_irql = 0; +SYSCTL_INT(_machdep, OID_AUTO, atrtc_use_high_irql, CTLFLAG_RDTUN, + &atrtc_use_high_irql, 0, "lock with high irql instead of disabling interupts" ); + /* * atrtc_lock protects low-level access to individual hardware registers. * atrtc_time_lock protects the entire sequence of accessing multiple registers * to read or write the date and time. */ -static struct mtx atrtc_lock; -MTX_SYSINIT(atrtc_lock_init, &atrtc_lock, "atrtc", MTX_SPIN); +/*static*/ struct mtx_un atrtc_lock; + +/** + * init and cleanup routines for rtc (spin-)lock. this is done with dedicated + * routines to read tuneable for it +*/ +static void atrtc_lock_init(const void* udata) +{ + int irql = (atrtc_use_high_irql == 0) ? 0 : MTX_IRQL_RT_LEVEL; + mtx_init_un(&atrtc_lock, "atrtc", irql); +} +SYSINIT(atcrtc_lock_un_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, atrtc_lock_init, 0); + +static void atrtc_lock_destroy(const void* udata) +{ + mtx_destroy_un(&atrtc_lock); +} +SYSUNINIT(atrtc_lock_un_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, atrtc_lock_destroy, 0); + /* Force RTC enabled/disabled. */ static int atrtc_enabled = -1; @@ -77,6 +100,7 @@ struct mtx atrtc_time_lock; MTX_SYSINIT(atrtc_time_lock_init, &atrtc_time_lock, "atrtc_time", MTX_DEF); +/* some global constants */ int atrtcclock_disable = 0; static int rtc_reg = -1; @@ -124,9 +148,9 @@ { u_char val; - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); val = rtcin_locked(reg); - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); return (val); } @@ -134,19 +158,19 @@ writertc(int reg, u_char val) { - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); rtcout_locked(reg, val); - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); } static void atrtc_start(void) { - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); rtcout_locked(RTC_STATUSA, rtc_statusa); rtcout_locked(RTC_STATUSB, RTCSB_24HR); - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); } static void @@ -162,10 +186,10 @@ { rtc_statusb |= RTCSB_PINTR; - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); } static void @@ -173,10 +197,10 @@ { rtc_statusb &= ~RTCSB_PINTR; - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); } void @@ -184,13 +208,13 @@ { /* Restore all of the RTC's "status" (actually, control) registers. */ - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); rtcin_locked(RTC_STATUSA); /* dummy to get rtc_reg set */ rtcout_locked(RTC_STATUSB, RTCSB_24HR); rtcout_locked(RTC_STATUSA, rtc_statusa); rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); } /********************************************************************** @@ -535,7 +559,7 @@ clock_dbgprint_bcd(dev, CLOCK_DBG_WRITE, &bct); mtx_lock(&atrtc_time_lock); - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); /* Disable RTC updates and interrupts. */ rtcout_locked(RTC_STATUSB, RTCSB_HALT | RTCSB_24HR); @@ -558,7 +582,7 @@ rtcout_locked(RTC_STATUSB, rtc_statusb); rtcin_locked(RTC_INTR); - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); mtx_unlock(&atrtc_time_lock); return (0); @@ -585,7 +609,7 @@ mtx_lock(&atrtc_time_lock); while (rtcin(RTC_STATUSA) & RTCSA_TUP) continue; - mtx_lock_spin(&atrtc_lock); + mtx_lock_un(&atrtc_lock); bct.sec = rtcin_locked(RTC_SEC); bct.min = rtcin_locked(RTC_MIN); bct.hour = rtcin_locked(RTC_HRS); @@ -595,7 +619,7 @@ #ifdef USE_RTC_CENTURY bct.year |= rtcin_locked(RTC_CENTURY) << 8; #endif - mtx_unlock_spin(&atrtc_lock); + mtx_unlock_un(&atrtc_lock); mtx_unlock(&atrtc_time_lock); /* dow is unused in timespec conversion and we have no nsec info. */ bct.dow = 0;