Index: sys/kern/kern_shutdown.c =================================================================== --- sys/kern/kern_shutdown.c +++ sys/kern/kern_shutdown.c @@ -822,6 +822,17 @@ #endif /* KASSERT_PANIC_OPTIONAL */ #endif +#ifdef INVARIANTS +void +refcnt_default_assert(void *obj, long count, const char *exp, const char *file, + int line, const char *func) +{ + + panic("refcount assertion %s failed; count %ld (%p) detected at %s:%d (%s)", + exp, count, obj, file, line, func); +} +#endif + /* * Panic is called on unresolvable fatal errors. It prints "panic: mesg", * and then reboots. If we are called twice, then we avoid trying to sync Index: sys/sys/_refcount.h =================================================================== --- /dev/null +++ sys/sys/_refcount.h @@ -0,0 +1,14 @@ +#ifndef _SYS__REFCOUNT_H_ +#define _SYS__REFCOUNT_H_ + +#define REFCNT_TYPE(TYPE) \ +struct refcnt##TYPE { \ + u_##TYPE count; \ +}; \ + \ +typedef struct refcnt##TYPE refcnt##TYPE##_t; \ + +REFCNT_TYPE(int); +REFCNT_TYPE(long); + +#endif /* !_SYS__REFCOUNT_H_ */ Index: sys/sys/refcount.h =================================================================== --- sys/sys/refcount.h +++ sys/sys/refcount.h @@ -31,6 +31,7 @@ #define __SYS_REFCOUNT_H__ #include +#include #ifdef _KERNEL #include @@ -209,4 +210,301 @@ return refcount_release_if_gt(count, 1); } + +/* + * refcnt##type implementation. + * + * In contrast to refcount_*, routines below don't support runtime overflow + * detection nor flags of any sort. + */ + +#ifdef INVARIANTS +void refcnt_default_assert(void *, long, const char *, const char *, int, + const char *); +#define REFCNT_DEBUG_ARGS , __FILE__, __LINE__, __func__ +#define REFCNT_ASSERT_DEF_ARGS \ + , void (*assert_fail)(void *, long, const char *, const char *, int,\ + const char *), void *obj, const char *file, int line, const char *func +#define REFCNT_ASSERT_ARGS \ + , assert_fail, obj, file, line, func +#define REFCNT_ASSERT(exp) do { \ + if (__predict_false(!(exp))) \ + assert_fail(obj, old, #exp, file, line, func); \ +} while (0) + +#define REFCNT_ASSERTS(TYPE) \ +static __inline void \ +__refcnt##TYPE##_add(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + TYPE old; \ + \ + old = atomic_fetchadd_##TYPE(&cnt->count, n); \ + REFCNT_ASSERT(old >= 0 && old + n > 0); \ +} \ + \ +static __inline u_##TYPE \ +__refcnt##TYPE##_sub(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + TYPE old; \ + \ + atomic_thread_fence_rel(); \ + old = atomic_fetchadd_##TYPE(&cnt->count, -n); \ + REFCNT_ASSERT(old >= n); \ + return (old); \ +} \ +static __inline void \ +__refcnt##TYPE##_acquiren_nz(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + TYPE old; \ + \ + old = atomic_fetchadd_##TYPE(&cnt->count, n); \ + REFCNT_ASSERT(old > 0 && old + n > 0); \ +} +#else +#define refcnt_default_assert NULL +#define REFCNT_DEBUG_ARGS +#define REFCNT_ASSERT_DEF_ARGS \ + , void (*assert_fail)(void *, long, const char *, int, const char *),\ + void *obj +#define REFCNT_ASSERT_ARGS \ + , assert_fail, obj +#define REFCNT_ASSERT(exp) do { } while (0) + +#define REFCNT_ASSERTS(TYPE) \ +static __inline void \ +__refcnt##TYPE##_add(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + atomic_add_##TYPE(&cnt->count, n); \ +} \ + \ +static __inline u_##TYPE \ +__refcnt##TYPE##_sub(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + TYPE old; \ + \ + atomic_thread_fence_rel(); \ + old = atomic_fetchadd_##TYPE(&cnt->count, -n); \ + return (old); \ +} \ + \ +static __inline void \ +__refcnt##TYPE##_acquiren_nz(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + atomic_add_##TYPE(&cnt->count, n); \ +} +#endif + +#define REFCNT_IMPL(TYPE) \ +static __inline void \ +refcnt##TYPE##_init(refcnt##TYPE##_t *cnt, u_##TYPE n) \ +{ \ + \ + atomic_store_##TYPE(&cnt->count, n); \ +} \ + \ +static __inline u_##TYPE \ +refcnt##TYPE##_read(refcnt##TYPE##_t *cnt) \ +{ \ + \ + return (atomic_load_##TYPE(&cnt->count)); \ +} \ + \ +static __inline u_##TYPE \ +refcnt##TYPE##_clear(refcnt##TYPE##_t *cnt) \ +{ \ + \ + return (atomic_swap_##TYPE(&cnt->count, 0)); \ +} \ + \ +static __inline void \ +__refcnt##TYPE##_acquiren(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + __refcnt##TYPE##_add(cnt, n REFCNT_ASSERT_ARGS); \ +} \ + \ +static __inline void \ +__refcnt##TYPE##_acquire(refcnt##TYPE##_t *cnt REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + __refcnt##TYPE##_acquiren(cnt, 1 REFCNT_ASSERT_ARGS); \ +} \ + \ +static __inline bool \ +__refcnt##TYPE##_releasen(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + TYPE old; \ + \ + old = __refcnt##TYPE##_sub(cnt, n REFCNT_ASSERT_ARGS); \ + if (__predict_false(old == n)) { \ + atomic_thread_fence_acq(); \ + return (true); \ + } \ + return (false); \ +} \ + \ +static __inline bool \ +__refcnt##TYPE##_release(refcnt##TYPE##_t *cnt REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + return (__refcnt##TYPE##_releasen(cnt, 1 REFCNT_ASSERT_ARGS)); \ +} \ + \ +static __inline __result_use_check bool \ +__refcnt##TYPE##_acquire_if_gt(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + TYPE old; \ + \ + old = refcnt##TYPE##_read(cnt); \ + for (;;) { \ + REFCNT_ASSERT(old >= 0); \ + if (old <= n) \ + return (false); \ + if (atomic_fcmpset_##TYPE(&cnt->count, &old, old + 1)) \ + return (true); \ + } \ +} \ + \ +static __inline __result_use_check bool \ +__refcnt##TYPE##_release_if_gt(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + TYPE old; \ + \ + old = refcnt##TYPE##_read(cnt); \ + for (;;) { \ + REFCNT_ASSERT(old > 0); \ + if (old <= n) \ + return (false); \ + if (atomic_fcmpset_rel_##TYPE(&cnt->count, &old, old - 1))\ + return (true); \ + } \ +} \ + \ +static __inline __result_use_check TYPE \ +__refcnt##TYPE##_acquiren_ret(refcnt##TYPE##_t *cnt, u_##TYPE n \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + return (atomic_fetchadd_##TYPE(&cnt->count, n)); \ +} \ + \ +static __inline __result_use_check TYPE \ +__refcnt##TYPE##_acquire_ret(refcnt##TYPE##_t *cnt \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + return (__refcnt##TYPE##_acquiren_ret(cnt, 1 \ + REFCNT_ASSERT_ARGS)); \ +} \ + \ +static __inline void \ +__refcnt##TYPE##_acquire_nz(refcnt##TYPE##_t *cnt \ + REFCNT_ASSERT_DEF_ARGS) \ +{ \ + \ + __refcnt##TYPE##_acquiren_nz(cnt, 1 REFCNT_ASSERT_ARGS); \ +} + +REFCNT_ASSERTS(int); +REFCNT_IMPL(int); + +REFCNT_ASSERTS(long); +REFCNT_IMPL(long); + +#define _refcntint_acquiren(cnt, n, f, a) \ + __refcntint_acquiren(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntint_acquire(cnt, f, a) \ + __refcntint_acquire(cnt, f, a REFCNT_DEBUG_ARGS) +#define _refcntint_acquiren_ret(cnt, n, f, a) \ + __refcntint_acquiren_ret(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntint_acquire_ret(cnt, f, a) \ + __refcntint_acquire_ret(cnt, f, a REFCNT_DEBUG_ARGS) +#define _refcntint_acquiren_nz(cnt, n, f, a) \ + __refcntint_acquiren_nz(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntint_acquire_nz(cnt, f, a) \ + __refcntint_acquire_nz(cnt, f, a REFCNT_DEBUG_ARGS) +#define _refcntint_releasen(cnt, n, f, a) \ + __predict_false(__refcntint_releasen(cnt, n, f, a REFCNT_DEBUG_ARGS)) +#define _refcntint_release(cnt, f, a) \ + __predict_false(__refcntint_release(cnt, f, a REFCNT_DEBUG_ARGS)) +#define _refcntint_acquire_if_gt(cnt, n, f, a) \ + __refcntint_acquire_if_gt(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntint_release_if_gt(cnt, n, f, a) \ + __refcntint_release_if_gt(cnt, n, f, a REFCNT_DEBUG_ARGS) + +#define _refcntlong_acquiren(cnt, n, f, a) \ + __refcntlong_acquiren(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntlong_acquire(cnt, f, a) \ + __refcntlong_acquire(cnt, f, a REFCNT_DEBUG_ARGS) +#define _refcntlong_acquiren_ret(cnt, n, f, a) \ + __refcntlong_acquiren_ret(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntlong_acquire_ret(cnt, f, a) \ + __refcntlong_acquire_ret(cnt, f, a REFCNT_DEBUG_ARGS) +#define _refcntlong_acquiren_nz(cnt, n, f, a) \ + __refcntlong_acquiren_nz(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntlong_acquire_nz(cnt, f, a) \ + __refcntlong_acquire_nz(cnt, f, a REFCNT_DEBUG_ARGS) +#define _refcntlong_releasen(cnt, n, f, a) \ + __predict_false(__refcntlong_releasen(cnt, n, f, a REFCNT_DEBUG_ARGS)) +#define _refcntlong_release(cnt, f, a) \ + __predict_false(__refcntlong_release(cnt, f, a REFCNT_DEBUG_ARGS)) +#define _refcntlong_acquire_if_gt(cnt, n, f, a) \ + __refcntlong_acquire_if_gt(cnt, n, f, a REFCNT_DEBUG_ARGS) +#define _refcntlong_release_if_gt(cnt, n, f, a) \ + __refcntlong_release_if_gt(cnt, n, f, a REFCNT_DEBUG_ARGS) + +#define refcntint_acquiren(cnt, n) \ + _refcntint_acquiren(cnt, n, refcnt_default_assert, cnt) +#define refcntint_acquire(cnt) \ + _refcntint_acquire(cnt, refcnt_default_assert, cnt) +#define refcntint_acquiren_ret(cnt, n) \ + _refcntint_acquiren_ret(cnt, n, refcnt_default_assert, cnt) +#define refcntint_acquire_ret(cnt) \ + _refcntint_acquire_ret(cnt, refcnt_default_assert, cnt) +#define refcntint_acquiren_nz(cnt, n) \ + _refcntint_acquiren_nz(cnt, n, refcnt_default_assert, cnt) +#define refcntint_acquire_nz(cnt) \ + _refcntint_acquire_nz(cnt, refcnt_default_assert, cnt) +#define refcntint_releasen(cnt, n) \ + _refcntint_releasen(cnt, n, refcnt_default_assert, cnt) +#define refcntint_release(cnt) \ + _refcntint_release(cnt, refcnt_default_assert, cnt) +#define refcntint_acquire_if_gt(cnt, n) \ + _refcntint_acquire_if_gt(cnt, n, refcnt_default_assert, cnt) +#define refcntint_release_if_gt(cnt, n) \ + _refcntint_release_if_gt(cnt, n, refcnt_default_assert, cnt) + +#define refcntlong_acquiren(cnt, n) \ + _refcntlong_acquiren(cnt, n, refcnt_default_assert, cnt) +#define refcntlong_acquire(cnt) \ + _refcntlong_acquire(cnt, refcnt_default_assert, cnt) +#define refcntlong_acquiren_ret(cnt, n) \ + _refcntlong_acquiren_ret(cnt, n, refcnt_default_assert, cnt) +#define refcntlong_acquire_ret(cnt) \ + _refcntlong_acquire_ret(cnt, refcnt_default_assert, cnt) +#define refcntlong_acquiren_nz(cnt, n) \ + _refcntlong_acquiren_nz(cnt, n, refcnt_default_assert, cnt) +#define refcntlong_acquire_nz(cnt) \ + _refcntlong_acquire_nz(cnt, refcnt_default_assert, cnt) +#define refcntlong_releasen(cnt, n) \ + _refcntlong_releasen(cnt, n, refcnt_default_assert, cnt) +#define refcntlong_release(cnt) \ + _refcntlong_release(cnt, refcnt_default_assert, cnt) +#define refcntlong_acquire_if_gt(cnt, n) \ + _refcntlong_acquire_if_gt(cnt, n, refcnt_default_assert, cnt) +#define refcntlong_release_if_gt(cnt, n) \ + _refcntlong_release_if_gt(cnt, n, refcnt_default_assert, cnt) + #endif /* ! __SYS_REFCOUNT_H__ */