Index: vendor-sys/ck/20161230/include/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_pr.h (revision 310844) @@ -0,0 +1,1219 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_H +#define CK_PR_H + +#include +#include +#include +#include +#include + +#ifndef CK_USE_CC_BUILTINS +#if defined(__x86_64__) +#include "gcc/x86_64/ck_pr.h" +#elif defined(__x86__) +#include "gcc/x86/ck_pr.h" +#elif defined(__sparcv9__) +#include "gcc/sparcv9/ck_pr.h" +#elif defined(__ppc64__) +#include "gcc/ppc64/ck_pr.h" +#elif defined(__ppc__) +#include "gcc/ppc/ck_pr.h" +#elif defined(__arm__) +#include "gcc/arm/ck_pr.h" +#elif defined(__aarch64__) +#include "gcc/aarch64/ck_pr.h" +#elif !defined(__GNUC__) +#error Your platform is unsupported +#endif +#endif /* !CK_USE_CC_BUILTINS */ + +#if defined(__GNUC__) +#include "gcc/ck_pr.h" +#endif + +#define CK_PR_FENCE_EMIT(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_##T(void) \ + { \ + ck_pr_fence_strict_##T(); \ + return; \ + } +#define CK_PR_FENCE_NOOP(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_##T(void) \ + { \ + ck_pr_barrier(); \ + return; \ + } + +/* + * None of the currently supported platforms allow for data-dependent + * load ordering. + */ +CK_PR_FENCE_NOOP(load_depends) +#define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends + +/* + * In memory models where atomic operations do not have serializing + * effects, atomic read-modify-write operations are modeled as stores. + */ +#if defined(CK_MD_RMO) +/* + * Only stores to the same location have a global + * ordering. + */ +CK_PR_FENCE_EMIT(atomic) +CK_PR_FENCE_EMIT(atomic_load) +CK_PR_FENCE_EMIT(atomic_store) +CK_PR_FENCE_EMIT(store_atomic) +CK_PR_FENCE_EMIT(load_atomic) +CK_PR_FENCE_EMIT(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_EMIT(load) +CK_PR_FENCE_EMIT(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_EMIT(acquire) +CK_PR_FENCE_EMIT(release) +CK_PR_FENCE_EMIT(acqrel) +CK_PR_FENCE_EMIT(lock) +CK_PR_FENCE_EMIT(unlock) +#elif defined(CK_MD_PSO) +/* + * Anything can be re-ordered with respect to stores. + * Otherwise, loads are executed in-order. + */ +CK_PR_FENCE_EMIT(atomic) +CK_PR_FENCE_NOOP(atomic_load) +CK_PR_FENCE_EMIT(atomic_store) +CK_PR_FENCE_EMIT(store_atomic) +CK_PR_FENCE_NOOP(load_atomic) +CK_PR_FENCE_EMIT(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_NOOP(load) +CK_PR_FENCE_EMIT(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_EMIT(acquire) +CK_PR_FENCE_EMIT(release) +CK_PR_FENCE_EMIT(acqrel) +CK_PR_FENCE_EMIT(lock) +CK_PR_FENCE_EMIT(unlock) +#elif defined(CK_MD_TSO) +/* + * Only loads are re-ordered and only with respect to + * prior stores. Atomic operations are serializing. + */ +CK_PR_FENCE_NOOP(atomic) +CK_PR_FENCE_NOOP(atomic_load) +CK_PR_FENCE_NOOP(atomic_store) +CK_PR_FENCE_NOOP(store_atomic) +CK_PR_FENCE_NOOP(load_atomic) +CK_PR_FENCE_NOOP(load_store) +CK_PR_FENCE_EMIT(store_load) +CK_PR_FENCE_NOOP(load) +CK_PR_FENCE_NOOP(store) +CK_PR_FENCE_EMIT(memory) +CK_PR_FENCE_NOOP(acquire) +CK_PR_FENCE_NOOP(release) +CK_PR_FENCE_NOOP(acqrel) +CK_PR_FENCE_NOOP(lock) +CK_PR_FENCE_NOOP(unlock) +#else +#error "No memory model has been defined." +#endif /* CK_MD_TSO */ + +#undef CK_PR_FENCE_EMIT +#undef CK_PR_FENCE_NOOP + +#ifndef CK_F_PR_RFO +#define CK_F_PR_RFO +CK_CC_INLINE static void +ck_pr_rfo(const void *m) +{ + + (void)m; + return; +} +#endif /* CK_F_PR_RFO */ + +#define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ + ck_pr_md_store_##TYPE( \ + ((void)sizeof(*(DST) = (VAL)), (DST)), \ + (VAL)) + +#define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) +#define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) +#ifndef CK_PR_DISABLE_DOUBLE +#define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) +#endif +#define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) +#define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) +#define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) +#define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) +#define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) + +#define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) + +#ifdef CK_F_PR_LOAD_64 +#define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) +#endif /* CK_F_PR_LOAD_64 */ + +#define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) +#define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) + +#define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) +#define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) +#ifndef CK_PR_DISABLE_DOUBLE +#define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) +#endif +#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) +#define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) +#define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) +#define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) +#define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) + +#ifdef CK_F_PR_LOAD_64 +#define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) +#endif /* CK_F_PR_LOAD_64 */ + +#define CK_PR_BIN(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T value) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P value), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ + } + +#define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_CHAR +CK_PR_BIN_S(add, char, char, +) +#endif /* CK_F_PR_ADD_CHAR */ + +#ifndef CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_CHAR +CK_PR_BIN_S(sub, char, char, -) +#endif /* CK_F_PR_SUB_CHAR */ + +#ifndef CK_F_PR_AND_CHAR +#define CK_F_PR_AND_CHAR +CK_PR_BIN_S(and, char, char, &) +#endif /* CK_F_PR_AND_CHAR */ + +#ifndef CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_CHAR +CK_PR_BIN_S(xor, char, char, ^) +#endif /* CK_F_PR_XOR_CHAR */ + +#ifndef CK_F_PR_OR_CHAR +#define CK_F_PR_OR_CHAR +CK_PR_BIN_S(or, char, char, |) +#endif /* CK_F_PR_OR_CHAR */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_ADD_INT +#define CK_F_PR_ADD_INT +CK_PR_BIN_S(add, int, int, +) +#endif /* CK_F_PR_ADD_INT */ + +#ifndef CK_F_PR_SUB_INT +#define CK_F_PR_SUB_INT +CK_PR_BIN_S(sub, int, int, -) +#endif /* CK_F_PR_SUB_INT */ + +#ifndef CK_F_PR_AND_INT +#define CK_F_PR_AND_INT +CK_PR_BIN_S(and, int, int, &) +#endif /* CK_F_PR_AND_INT */ + +#ifndef CK_F_PR_XOR_INT +#define CK_F_PR_XOR_INT +CK_PR_BIN_S(xor, int, int, ^) +#endif /* CK_F_PR_XOR_INT */ + +#ifndef CK_F_PR_OR_INT +#define CK_F_PR_OR_INT +CK_PR_BIN_S(or, int, int, |) +#endif /* CK_F_PR_OR_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_ADD_DOUBLE +#define CK_F_PR_ADD_DOUBLE +CK_PR_BIN_S(add, double, double, +) +#endif /* CK_F_PR_ADD_DOUBLE */ + +#ifndef CK_F_PR_SUB_DOUBLE +#define CK_F_PR_SUB_DOUBLE +CK_PR_BIN_S(sub, double, double, -) +#endif /* CK_F_PR_SUB_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_ADD_UINT +#define CK_F_PR_ADD_UINT +CK_PR_BIN_S(add, uint, unsigned int, +) +#endif /* CK_F_PR_ADD_UINT */ + +#ifndef CK_F_PR_SUB_UINT +#define CK_F_PR_SUB_UINT +CK_PR_BIN_S(sub, uint, unsigned int, -) +#endif /* CK_F_PR_SUB_UINT */ + +#ifndef CK_F_PR_AND_UINT +#define CK_F_PR_AND_UINT +CK_PR_BIN_S(and, uint, unsigned int, &) +#endif /* CK_F_PR_AND_UINT */ + +#ifndef CK_F_PR_XOR_UINT +#define CK_F_PR_XOR_UINT +CK_PR_BIN_S(xor, uint, unsigned int, ^) +#endif /* CK_F_PR_XOR_UINT */ + +#ifndef CK_F_PR_OR_UINT +#define CK_F_PR_OR_UINT +CK_PR_BIN_S(or, uint, unsigned int, |) +#endif /* CK_F_PR_OR_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_PTR +CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) +#endif /* CK_F_PR_ADD_PTR */ + +#ifndef CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_PTR +CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) +#endif /* CK_F_PR_SUB_PTR */ + +#ifndef CK_F_PR_AND_PTR +#define CK_F_PR_AND_PTR +CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) +#endif /* CK_F_PR_AND_PTR */ + +#ifndef CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_PTR +CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) +#endif /* CK_F_PR_XOR_PTR */ + +#ifndef CK_F_PR_OR_PTR +#define CK_F_PR_OR_PTR +CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) +#endif /* CK_F_PR_OR_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_ADD_64 +#define CK_F_PR_ADD_64 +CK_PR_BIN_S(add, 64, uint64_t, +) +#endif /* CK_F_PR_ADD_64 */ + +#ifndef CK_F_PR_SUB_64 +#define CK_F_PR_SUB_64 +CK_PR_BIN_S(sub, 64, uint64_t, -) +#endif /* CK_F_PR_SUB_64 */ + +#ifndef CK_F_PR_AND_64 +#define CK_F_PR_AND_64 +CK_PR_BIN_S(and, 64, uint64_t, &) +#endif /* CK_F_PR_AND_64 */ + +#ifndef CK_F_PR_XOR_64 +#define CK_F_PR_XOR_64 +CK_PR_BIN_S(xor, 64, uint64_t, ^) +#endif /* CK_F_PR_XOR_64 */ + +#ifndef CK_F_PR_OR_64 +#define CK_F_PR_OR_64 +CK_PR_BIN_S(or, 64, uint64_t, |) +#endif /* CK_F_PR_OR_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_ADD_32 +#define CK_F_PR_ADD_32 +CK_PR_BIN_S(add, 32, uint32_t, +) +#endif /* CK_F_PR_ADD_32 */ + +#ifndef CK_F_PR_SUB_32 +#define CK_F_PR_SUB_32 +CK_PR_BIN_S(sub, 32, uint32_t, -) +#endif /* CK_F_PR_SUB_32 */ + +#ifndef CK_F_PR_AND_32 +#define CK_F_PR_AND_32 +CK_PR_BIN_S(and, 32, uint32_t, &) +#endif /* CK_F_PR_AND_32 */ + +#ifndef CK_F_PR_XOR_32 +#define CK_F_PR_XOR_32 +CK_PR_BIN_S(xor, 32, uint32_t, ^) +#endif /* CK_F_PR_XOR_32 */ + +#ifndef CK_F_PR_OR_32 +#define CK_F_PR_OR_32 +CK_PR_BIN_S(or, 32, uint32_t, |) +#endif /* CK_F_PR_OR_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_ADD_16 +#define CK_F_PR_ADD_16 +CK_PR_BIN_S(add, 16, uint16_t, +) +#endif /* CK_F_PR_ADD_16 */ + +#ifndef CK_F_PR_SUB_16 +#define CK_F_PR_SUB_16 +CK_PR_BIN_S(sub, 16, uint16_t, -) +#endif /* CK_F_PR_SUB_16 */ + +#ifndef CK_F_PR_AND_16 +#define CK_F_PR_AND_16 +CK_PR_BIN_S(and, 16, uint16_t, &) +#endif /* CK_F_PR_AND_16 */ + +#ifndef CK_F_PR_XOR_16 +#define CK_F_PR_XOR_16 +CK_PR_BIN_S(xor, 16, uint16_t, ^) +#endif /* CK_F_PR_XOR_16 */ + +#ifndef CK_F_PR_OR_16 +#define CK_F_PR_OR_16 +CK_PR_BIN_S(or, 16, uint16_t, |) +#endif /* CK_F_PR_OR_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_ADD_8 +#define CK_F_PR_ADD_8 +CK_PR_BIN_S(add, 8, uint8_t, +) +#endif /* CK_F_PR_ADD_8 */ + +#ifndef CK_F_PR_SUB_8 +#define CK_F_PR_SUB_8 +CK_PR_BIN_S(sub, 8, uint8_t, -) +#endif /* CK_F_PR_SUB_8 */ + +#ifndef CK_F_PR_AND_8 +#define CK_F_PR_AND_8 +CK_PR_BIN_S(and, 8, uint8_t, &) +#endif /* CK_F_PR_AND_8 */ + +#ifndef CK_F_PR_XOR_8 +#define CK_F_PR_XOR_8 +CK_PR_BIN_S(xor, 8, uint8_t, ^) +#endif /* CK_F_PR_XOR_8 */ + +#ifndef CK_F_PR_OR_8 +#define CK_F_PR_OR_8 +CK_PR_BIN_S(or, 8, uint8_t, |) +#endif /* CK_F_PR_OR_8 */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_BIN_S +#undef CK_PR_BIN + +#define CK_PR_BTX(K, S, M, T, P, C, R) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(M *target, unsigned int offset) \ + { \ + T previous; \ + C punt; \ + punt = ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, (C)previous, \ + (C)(previous P (R ((T)1 << offset))), &previous) == false) \ + ck_pr_stall(); \ + return ((previous >> offset) & 1); \ + } + +#define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_BTC_INT +#define CK_F_PR_BTC_INT +CK_PR_BTX_S(btc, int, int, ^,) +#endif /* CK_F_PR_BTC_INT */ + +#ifndef CK_F_PR_BTR_INT +#define CK_F_PR_BTR_INT +CK_PR_BTX_S(btr, int, int, &, ~) +#endif /* CK_F_PR_BTR_INT */ + +#ifndef CK_F_PR_BTS_INT +#define CK_F_PR_BTS_INT +CK_PR_BTX_S(bts, int, int, |,) +#endif /* CK_F_PR_BTS_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_BTC_UINT +#define CK_F_PR_BTC_UINT +CK_PR_BTX_S(btc, uint, unsigned int, ^,) +#endif /* CK_F_PR_BTC_UINT */ + +#ifndef CK_F_PR_BTR_UINT +#define CK_F_PR_BTR_UINT +CK_PR_BTX_S(btr, uint, unsigned int, &, ~) +#endif /* CK_F_PR_BTR_UINT */ + +#ifndef CK_F_PR_BTS_UINT +#define CK_F_PR_BTS_UINT +CK_PR_BTX_S(bts, uint, unsigned int, |,) +#endif /* CK_F_PR_BTS_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_PTR +CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) +#endif /* CK_F_PR_BTC_PTR */ + +#ifndef CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_PTR +CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) +#endif /* CK_F_PR_BTR_PTR */ + +#ifndef CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_PTR +CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) +#endif /* CK_F_PR_BTS_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_BTC_64 +#define CK_F_PR_BTC_64 +CK_PR_BTX_S(btc, 64, uint64_t, ^,) +#endif /* CK_F_PR_BTC_64 */ + +#ifndef CK_F_PR_BTR_64 +#define CK_F_PR_BTR_64 +CK_PR_BTX_S(btr, 64, uint64_t, &, ~) +#endif /* CK_F_PR_BTR_64 */ + +#ifndef CK_F_PR_BTS_64 +#define CK_F_PR_BTS_64 +CK_PR_BTX_S(bts, 64, uint64_t, |,) +#endif /* CK_F_PR_BTS_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_BTC_32 +#define CK_F_PR_BTC_32 +CK_PR_BTX_S(btc, 32, uint32_t, ^,) +#endif /* CK_F_PR_BTC_32 */ + +#ifndef CK_F_PR_BTR_32 +#define CK_F_PR_BTR_32 +CK_PR_BTX_S(btr, 32, uint32_t, &, ~) +#endif /* CK_F_PR_BTR_32 */ + +#ifndef CK_F_PR_BTS_32 +#define CK_F_PR_BTS_32 +CK_PR_BTX_S(bts, 32, uint32_t, |,) +#endif /* CK_F_PR_BTS_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_BTC_16 +#define CK_F_PR_BTC_16 +CK_PR_BTX_S(btc, 16, uint16_t, ^,) +#endif /* CK_F_PR_BTC_16 */ + +#ifndef CK_F_PR_BTR_16 +#define CK_F_PR_BTR_16 +CK_PR_BTX_S(btr, 16, uint16_t, &, ~) +#endif /* CK_F_PR_BTR_16 */ + +#ifndef CK_F_PR_BTS_16 +#define CK_F_PR_BTS_16 +CK_PR_BTX_S(bts, 16, uint16_t, |,) +#endif /* CK_F_PR_BTS_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#undef CK_PR_BTX_S +#undef CK_PR_BTX + +#define CK_PR_UNARY(K, X, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + ck_pr_##X##_##S(target, (T)1); \ + return; \ + } + +#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous P 1), \ + &previous) == false) \ + ck_pr_stall(); \ + *zero = previous == (T)Z; \ + return; \ + } + +#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) +#define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR +CK_PR_UNARY_S(inc, add, char, char) +#endif /* CK_F_PR_INC_CHAR */ + +#ifndef CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_CHAR_ZERO +CK_PR_UNARY_Z_S(inc, char, char, +, -1) +#endif /* CK_F_PR_INC_CHAR_ZERO */ + +#ifndef CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR +CK_PR_UNARY_S(dec, sub, char, char) +#endif /* CK_F_PR_DEC_CHAR */ + +#ifndef CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_CHAR_ZERO +CK_PR_UNARY_Z_S(dec, char, char, -, 1) +#endif /* CK_F_PR_DEC_CHAR_ZERO */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_INC_INT +#define CK_F_PR_INC_INT +CK_PR_UNARY_S(inc, add, int, int) +#endif /* CK_F_PR_INC_INT */ + +#ifndef CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_INT_ZERO +CK_PR_UNARY_Z_S(inc, int, int, +, -1) +#endif /* CK_F_PR_INC_INT_ZERO */ + +#ifndef CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT +CK_PR_UNARY_S(dec, sub, int, int) +#endif /* CK_F_PR_DEC_INT */ + +#ifndef CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_INT_ZERO +CK_PR_UNARY_Z_S(dec, int, int, -, 1) +#endif /* CK_F_PR_DEC_INT_ZERO */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_INC_DOUBLE +#define CK_F_PR_INC_DOUBLE +CK_PR_UNARY_S(inc, add, double, double) +#endif /* CK_F_PR_INC_DOUBLE */ + +#ifndef CK_F_PR_DEC_DOUBLE +#define CK_F_PR_DEC_DOUBLE +CK_PR_UNARY_S(dec, sub, double, double) +#endif /* CK_F_PR_DEC_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT +CK_PR_UNARY_S(inc, add, uint, unsigned int) +#endif /* CK_F_PR_INC_UINT */ + +#ifndef CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_INC_UINT_ZERO +CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) +#endif /* CK_F_PR_INC_UINT_ZERO */ + +#ifndef CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT +CK_PR_UNARY_S(dec, sub, uint, unsigned int) +#endif /* CK_F_PR_DEC_UINT */ + +#ifndef CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_DEC_UINT_ZERO +CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) +#endif /* CK_F_PR_DEC_UINT_ZERO */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR +CK_PR_UNARY(inc, add, ptr, void, uintptr_t) +#endif /* CK_F_PR_INC_PTR */ + +#ifndef CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_PTR_ZERO +CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) +#endif /* CK_F_PR_INC_PTR_ZERO */ + +#ifndef CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR +CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) +#endif /* CK_F_PR_DEC_PTR */ + +#ifndef CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_PTR_ZERO +CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) +#endif /* CK_F_PR_DEC_PTR_ZERO */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_INC_64 +#define CK_F_PR_INC_64 +CK_PR_UNARY_S(inc, add, 64, uint64_t) +#endif /* CK_F_PR_INC_64 */ + +#ifndef CK_F_PR_INC_64_ZERO +#define CK_F_PR_INC_64_ZERO +CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) +#endif /* CK_F_PR_INC_64_ZERO */ + +#ifndef CK_F_PR_DEC_64 +#define CK_F_PR_DEC_64 +CK_PR_UNARY_S(dec, sub, 64, uint64_t) +#endif /* CK_F_PR_DEC_64 */ + +#ifndef CK_F_PR_DEC_64_ZERO +#define CK_F_PR_DEC_64_ZERO +CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) +#endif /* CK_F_PR_DEC_64_ZERO */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_INC_32 +#define CK_F_PR_INC_32 +CK_PR_UNARY_S(inc, add, 32, uint32_t) +#endif /* CK_F_PR_INC_32 */ + +#ifndef CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_32_ZERO +CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) +#endif /* CK_F_PR_INC_32_ZERO */ + +#ifndef CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32 +CK_PR_UNARY_S(dec, sub, 32, uint32_t) +#endif /* CK_F_PR_DEC_32 */ + +#ifndef CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_32_ZERO +CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) +#endif /* CK_F_PR_DEC_32_ZERO */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_INC_16 +#define CK_F_PR_INC_16 +CK_PR_UNARY_S(inc, add, 16, uint16_t) +#endif /* CK_F_PR_INC_16 */ + +#ifndef CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_16_ZERO +CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) +#endif /* CK_F_PR_INC_16_ZERO */ + +#ifndef CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16 +CK_PR_UNARY_S(dec, sub, 16, uint16_t) +#endif /* CK_F_PR_DEC_16 */ + +#ifndef CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_16_ZERO +CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) +#endif /* CK_F_PR_DEC_16_ZERO */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_INC_8 +#define CK_F_PR_INC_8 +CK_PR_UNARY_S(inc, add, 8, uint8_t) +#endif /* CK_F_PR_INC_8 */ + +#ifndef CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_8_ZERO +CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) +#endif /* CK_F_PR_INC_8_ZERO */ + +#ifndef CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8 +CK_PR_UNARY_S(dec, sub, 8, uint8_t) +#endif /* CK_F_PR_DEC_8 */ + +#ifndef CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_8_ZERO +CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) +#endif /* CK_F_PR_DEC_8_ZERO */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_UNARY_Z_S +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_Z +#undef CK_PR_UNARY + +#define CK_PR_N(K, S, M, T, P, C) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(P previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return; \ + } + +#define CK_PR_N_Z(S, M, T, C) \ + CK_CC_INLINE static void \ + ck_pr_neg_##S##_zero(M *target, bool *zero) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(-previous), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + *zero = previous == 0; \ + return; \ + } + +#define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) +#define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_CHAR +CK_PR_N_S(not, char, char, ~) +#endif /* CK_F_PR_NOT_CHAR */ + +#ifndef CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR +CK_PR_N_S(neg, char, char, -) +#endif /* CK_F_PR_NEG_CHAR */ + +#ifndef CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_CHAR_ZERO +CK_PR_N_Z_S(char, char) +#endif /* CK_F_PR_NEG_CHAR_ZERO */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_NOT_INT +#define CK_F_PR_NOT_INT +CK_PR_N_S(not, int, int, ~) +#endif /* CK_F_PR_NOT_INT */ + +#ifndef CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT +CK_PR_N_S(neg, int, int, -) +#endif /* CK_F_PR_NEG_INT */ + +#ifndef CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_INT_ZERO +CK_PR_N_Z_S(int, int) +#endif /* CK_F_PR_NEG_INT_ZERO */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_NEG_DOUBLE +#define CK_F_PR_NEG_DOUBLE +CK_PR_N_S(neg, double, double, -) +#endif /* CK_F_PR_NEG_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_NOT_UINT +#define CK_F_PR_NOT_UINT +CK_PR_N_S(not, uint, unsigned int, ~) +#endif /* CK_F_PR_NOT_UINT */ + +#ifndef CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT +CK_PR_N_S(neg, uint, unsigned int, -) +#endif /* CK_F_PR_NEG_UINT */ + +#ifndef CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NEG_UINT_ZERO +CK_PR_N_Z_S(uint, unsigned int) +#endif /* CK_F_PR_NEG_UINT_ZERO */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_PTR +CK_PR_N(not, ptr, void, uintptr_t, ~, void *) +#endif /* CK_F_PR_NOT_PTR */ + +#ifndef CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR +CK_PR_N(neg, ptr, void, uintptr_t, -, void *) +#endif /* CK_F_PR_NEG_PTR */ + +#ifndef CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_PTR_ZERO +CK_PR_N_Z(ptr, void, uintptr_t, void *) +#endif /* CK_F_PR_NEG_PTR_ZERO */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_NOT_64 +#define CK_F_PR_NOT_64 +CK_PR_N_S(not, 64, uint64_t, ~) +#endif /* CK_F_PR_NOT_64 */ + +#ifndef CK_F_PR_NEG_64 +#define CK_F_PR_NEG_64 +CK_PR_N_S(neg, 64, uint64_t, -) +#endif /* CK_F_PR_NEG_64 */ + +#ifndef CK_F_PR_NEG_64_ZERO +#define CK_F_PR_NEG_64_ZERO +CK_PR_N_Z_S(64, uint64_t) +#endif /* CK_F_PR_NEG_64_ZERO */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_NOT_32 +#define CK_F_PR_NOT_32 +CK_PR_N_S(not, 32, uint32_t, ~) +#endif /* CK_F_PR_NOT_32 */ + +#ifndef CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32 +CK_PR_N_S(neg, 32, uint32_t, -) +#endif /* CK_F_PR_NEG_32 */ + +#ifndef CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_32_ZERO +CK_PR_N_Z_S(32, uint32_t) +#endif /* CK_F_PR_NEG_32_ZERO */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_NOT_16 +#define CK_F_PR_NOT_16 +CK_PR_N_S(not, 16, uint16_t, ~) +#endif /* CK_F_PR_NOT_16 */ + +#ifndef CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16 +CK_PR_N_S(neg, 16, uint16_t, -) +#endif /* CK_F_PR_NEG_16 */ + +#ifndef CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_16_ZERO +CK_PR_N_Z_S(16, uint16_t) +#endif /* CK_F_PR_NEG_16_ZERO */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_NOT_8 +#define CK_F_PR_NOT_8 +CK_PR_N_S(not, 8, uint8_t, ~) +#endif /* CK_F_PR_NOT_8 */ + +#ifndef CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8 +CK_PR_N_S(neg, 8, uint8_t, -) +#endif /* CK_F_PR_NEG_8 */ + +#ifndef CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_8_ZERO +CK_PR_N_Z_S(8, uint8_t) +#endif /* CK_F_PR_NEG_8_ZERO */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_N_Z_S +#undef CK_PR_N_S +#undef CK_PR_N_Z +#undef CK_PR_N + +#define CK_PR_FAA(S, M, T, C) \ + CK_CC_INLINE static C \ + ck_pr_faa_##S(M *target, T delta) \ + { \ + T previous; \ + C punt; \ + punt = (C)ck_pr_md_load_##S(target); \ + previous = (T)punt; \ + while (ck_pr_cas_##S##_value(target, \ + (C)previous, \ + (C)(previous + delta), \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return ((C)previous); \ + } + +#define CK_PR_FAS(S, M, C) \ + CK_CC_INLINE static C \ + ck_pr_fas_##S(M *target, C update) \ + { \ + C previous; \ + previous = ck_pr_md_load_##S(target); \ + while (ck_pr_cas_##S##_value(target, \ + previous, \ + update, \ + &previous) == false) \ + ck_pr_stall(); \ + \ + return (previous); \ + } + +#define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) +#define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) + +#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) + +#ifndef CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_CHAR +CK_PR_FAA_S(char, char) +#endif /* CK_F_PR_FAA_CHAR */ + +#ifndef CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_CHAR +CK_PR_FAS_S(char, char) +#endif /* CK_F_PR_FAS_CHAR */ + +#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ + +#if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) + +#ifndef CK_F_PR_FAA_INT +#define CK_F_PR_FAA_INT +CK_PR_FAA_S(int, int) +#endif /* CK_F_PR_FAA_INT */ + +#ifndef CK_F_PR_FAS_INT +#define CK_F_PR_FAS_INT +CK_PR_FAS_S(int, int) +#endif /* CK_F_PR_FAS_INT */ + +#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ + +#if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ + !defined(CK_PR_DISABLE_DOUBLE) + +#ifndef CK_F_PR_FAA_DOUBLE +#define CK_F_PR_FAA_DOUBLE +CK_PR_FAA_S(double, double) +#endif /* CK_F_PR_FAA_DOUBLE */ + +#ifndef CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FAS_DOUBLE +CK_PR_FAS_S(double, double) +#endif /* CK_F_PR_FAS_DOUBLE */ + +#endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ + +#if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) + +#ifndef CK_F_PR_FAA_UINT +#define CK_F_PR_FAA_UINT +CK_PR_FAA_S(uint, unsigned int) +#endif /* CK_F_PR_FAA_UINT */ + +#ifndef CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_UINT +CK_PR_FAS_S(uint, unsigned int) +#endif /* CK_F_PR_FAS_UINT */ + +#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ + +#if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) + +#ifndef CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_PTR +CK_PR_FAA(ptr, void, uintptr_t, void *) +#endif /* CK_F_PR_FAA_PTR */ + +#ifndef CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_PTR +CK_PR_FAS(ptr, void, void *) +#endif /* CK_F_PR_FAS_PTR */ + +#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ + +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) + +#ifndef CK_F_PR_FAA_64 +#define CK_F_PR_FAA_64 +CK_PR_FAA_S(64, uint64_t) +#endif /* CK_F_PR_FAA_64 */ + +#ifndef CK_F_PR_FAS_64 +#define CK_F_PR_FAS_64 +CK_PR_FAS_S(64, uint64_t) +#endif /* CK_F_PR_FAS_64 */ + +#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ + +#if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) + +#ifndef CK_F_PR_FAA_32 +#define CK_F_PR_FAA_32 +CK_PR_FAA_S(32, uint32_t) +#endif /* CK_F_PR_FAA_32 */ + +#ifndef CK_F_PR_FAS_32 +#define CK_F_PR_FAS_32 +CK_PR_FAS_S(32, uint32_t) +#endif /* CK_F_PR_FAS_32 */ + +#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ + +#if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) + +#ifndef CK_F_PR_FAA_16 +#define CK_F_PR_FAA_16 +CK_PR_FAA_S(16, uint16_t) +#endif /* CK_F_PR_FAA_16 */ + +#ifndef CK_F_PR_FAS_16 +#define CK_F_PR_FAS_16 +CK_PR_FAS_S(16, uint16_t) +#endif /* CK_F_PR_FAS_16 */ + +#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) + +#ifndef CK_F_PR_FAA_8 +#define CK_F_PR_FAA_8 +CK_PR_FAA_S(8, uint8_t) +#endif /* CK_F_PR_FAA_8 */ + +#ifndef CK_F_PR_FAS_8 +#define CK_F_PR_FAS_8 +CK_PR_FAS_S(8, uint8_t) +#endif /* CK_F_PR_FAS_8 */ + +#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ + +#undef CK_PR_FAA_S +#undef CK_PR_FAS_S +#undef CK_PR_FAA +#undef CK_PR_FAS + +#endif /* CK_PR_H */ Property changes on: vendor-sys/ck/20161230/include/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr.h (revision 310844) @@ -0,0 +1,227 @@ +/* + * Copyright 2009-2016 Samy Al Bahra. + * Copyright 2013-2016 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_AARCH64_H +#define CK_PR_AARCH64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#define CK_DMB_SY __asm __volatile("dmb ish" : : "r" (0) : "memory") +#define CK_DMB_LD __asm __volatile("dmb ishld" : : "r" (0) : "memory") +#define CK_DMB_ST __asm __volatile("dmb ishst" : : "r" (0) : "memory") + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + I; \ + } + +CK_PR_FENCE(atomic, CK_DMB_ST) +CK_PR_FENCE(atomic_store, CK_DMB_ST) +CK_PR_FENCE(atomic_load, CK_DMB_SY) +CK_PR_FENCE(store_atomic, CK_DMB_ST) +CK_PR_FENCE(load_atomic, CK_DMB_SY) +CK_PR_FENCE(store, CK_DMB_ST) +CK_PR_FENCE(store_load, CK_DMB_SY) +CK_PR_FENCE(load, CK_DMB_LD) +CK_PR_FENCE(load_store, CK_DMB_SY) +CK_PR_FENCE(memory, CK_DMB_SY) +CK_PR_FENCE(acquire, CK_DMB_SY) +CK_PR_FENCE(release, CK_DMB_SY) +CK_PR_FENCE(acqrel, CK_DMB_SY) +CK_PR_FENCE(lock, CK_DMB_SY) +CK_PR_FENCE(unlock, CK_DMB_SY) + +#undef CK_PR_FENCE + +#undef CK_DMB_SI +#undef CK_DMB_LD +#undef CK_DMB_ST + +#define CK_PR_LOAD(S, M, T, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %w0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } +#define CK_PR_LOAD_64(S, M, T, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } + + +CK_PR_LOAD_64(ptr, void, void *, "ldr") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, I) +#define CK_PR_LOAD_S_64(S, T, I) CK_PR_LOAD_64(S, T, T, I) + +CK_PR_LOAD_S_64(64, uint64_t, "ldr") +CK_PR_LOAD_S(32, uint32_t, "ldr") +CK_PR_LOAD_S(16, uint16_t, "ldrh") +CK_PR_LOAD_S(8, uint8_t, "ldrb") +CK_PR_LOAD_S(uint, unsigned int, "ldr") +CK_PR_LOAD_S(int, int, "ldr") +CK_PR_LOAD_S(short, short, "ldrh") +CK_PR_LOAD_S(char, char, "ldrb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S_64(double, double, "ldr") +#endif + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD_S_64 +#undef CK_PR_LOAD +#undef CK_PR_LAOD_64 + +#define CK_PR_STORE(S, M, T, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %w1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } +#define CK_PR_STORE_64(S, M, T, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE_64(ptr, void, const void *, "str") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, I) +#define CK_PR_STORE_S_64(S, T, I) CK_PR_STORE_64(S, T, T, I) + +CK_PR_STORE_S_64(64, uint64_t, "str") +CK_PR_STORE_S(32, uint32_t, "str") +CK_PR_STORE_S(16, uint16_t, "strh") +CK_PR_STORE_S(8, uint8_t, "strb") +CK_PR_STORE_S(uint, unsigned int, "str") +CK_PR_STORE_S(int, int, "str") +CK_PR_STORE_S(short, short, "strh") +CK_PR_STORE_S(char, char, "strb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE_S_64(double, double, "str") +#endif + +#undef CK_PR_STORE_S +#undef CK_PR_STORE_S_64 +#undef CK_PR_STORE +#undef CK_PR_STORE_64 + +#ifdef CK_MD_LSE_ENABLE +#include "ck_pr_lse.h" +#else +#include "ck_pr_llsc.h" +#endif + +/* + * ck_pr_neg_*() functions can only be implemented via LL/SC, as there are no + * LSE alternatives. + */ +#define CK_PR_NEG(N, M, T, W, R) \ + CK_CC_INLINE static void \ + ck_pr_neg_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + "neg %" R "0, %" R "0;" \ + "stxr" W " %w1, %" R "0, [%2];" \ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_NEG(ptr, void, void *, "", "") +CK_PR_NEG(64, uint64_t, uint64_t, "", "") + +#define CK_PR_NEG_S(S, T, W) \ + CK_PR_NEG(S, T, T, W, "w") \ + +CK_PR_NEG_S(32, uint32_t, "") +CK_PR_NEG_S(uint, unsigned int, "") +CK_PR_NEG_S(int, int, "") +CK_PR_NEG_S(16, uint16_t, "h") +CK_PR_NEG_S(8, uint8_t, "b") +CK_PR_NEG_S(short, short, "h") +CK_PR_NEG_S(char, char, "b") + +#undef CK_PR_NEG_S +#undef CK_PR_NEG + +#endif /* CK_PR_AARCH64_H */ + Property changes on: vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_llsc.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_llsc.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_llsc.h (revision 310844) @@ -0,0 +1,352 @@ +/* + * Copyright 2009-2016 Samy Al Bahra. + * Copyright 2013-2016 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_AARCH64_LLSC_H +#define CK_PR_AARCH64_LLSC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +CK_CC_INLINE static bool +ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2]) +{ + uint64_t tmp1, tmp2; + + __asm__ __volatile__("1:" + "ldxp %0, %1, [%4];" + "mov %2, %0;" + "mov %3, %1;" + "eor %0, %0, %5;" + "eor %1, %1, %6;" + "orr %1, %0, %1;" + "mov %w0, #0;" + "cbnz %1, 2f;" + "stxp %w0, %7, %8, [%4];" + "cbnz %w0, 1b;" + "mov %w0, #1;" + "2:" + : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1]) + : "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1]) + : "cc", "memory"); + + return (tmp1); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) +{ + return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set), + CK_CPP_CAST(uint64_t *, value))); +} + +CK_CC_INLINE static bool +ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) +{ + uint64_t tmp1, tmp2; + + __asm__ __volatile__("1:" + "ldxp %0, %1, [%2];" + "eor %0, %0, %3;" + "eor %1, %1, %4;" + "orr %1, %0, %1;" + "mov %w0, #0;" + "cbnz %1, 2f;" + "stxp %w0, %5, %6, [%2];" + "cbnz %w0, 1b;" + "mov %w0, #1;" + "2:" + : "=&r" (tmp1), "=&r" (tmp2) + : "r" (target), "r" (compare[0]), "r" (compare[1]), "r" (set[0]), "r" (set[1]) + : "cc", "memory"); + + return (tmp1); +} +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *target, void *compare, void *set) +{ + return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set))); +} + + +#define CK_PR_CAS(N, M, T, W, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + "cmp %" R "0, %" R "4;" \ + "b.ne 2f;" \ + "stxr" W " %w1, %" R "3, [%2];" \ + "cbnz %w1, 1b;" \ + "2:" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *(T *)value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(M *target, T compare, T set) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__( \ + "1:" \ + "ldxr" W " %" R "0, [%2];" \ + "cmp %" R "0, %" R "4;" \ + "b.ne 2f;" \ + "stxr" W " %w1, %" R "3, [%2];" \ + "cbnz %w1, 1b;" \ + "2:" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(ptr, void, void *, "", "") + +#define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R) +CK_PR_CAS_S(64, uint64_t, "", "") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_S(double, double, "", "") +#endif +CK_PR_CAS_S(32, uint32_t, "", "w") +CK_PR_CAS_S(uint, unsigned int, "", "w") +CK_PR_CAS_S(int, int, "", "w") +CK_PR_CAS_S(16, uint16_t, "h", "w") +CK_PR_CAS_S(8, uint8_t, "b", "w") +CK_PR_CAS_S(short, short, "h", "w") +CK_PR_CAS_S(char, char, "b", "w") + + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W, R) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + "stxr" W " %w1, %" R "3, [%2];"\ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(64, uint64_t, uint64_t, "", "") +CK_PR_FAS(32, uint32_t, uint32_t, "", "w") +CK_PR_FAS(ptr, void, void *, "", "") +CK_PR_FAS(int, int, int, "", "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "", "w") +CK_PR_FAS(16, uint16_t, uint16_t, "h", "w") +CK_PR_FAS(8, uint8_t, uint8_t, "b", "w") +CK_PR_FAS(short, short, short, "h", "w") +CK_PR_FAS(char, char, char, "b", "w") + + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W, R) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];" \ + I ";" \ + "stxr" W " %w1, %" R "0, [%2];" \ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "", "") +CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "", "") +CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "", "") +CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "add %0, %0, #1", "", "") +CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "sub %0, %0, #1", "", "") +CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mvn %0, %0", "", "") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "add %w0, %w0, #1", W, "w") \ + CK_PR_UNARY(dec, S, T, T, "sub %w0, %w0, #1", W, "w") \ + CK_PR_UNARY(not, S, T, T, "mvn %w0, %w0", W, "w") \ + +CK_PR_UNARY_S(32, uint32_t, "") +CK_PR_UNARY_S(uint, unsigned int, "") +CK_PR_UNARY_S(int, int, "") +CK_PR_UNARY_S(16, uint16_t, "h") +CK_PR_UNARY_S(8, uint8_t, "b") +CK_PR_UNARY_S(short, short, "h") +CK_PR_UNARY_S(char, char, "b") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W, R) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + T tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %" R "0, [%2];"\ + I " %" R "0, %" R "0, %" R "3;" \ + "stxr" W " %w1, %" R "0, [%2];" \ + "cbnz %w1, 1b;" \ + : "=&r" (previous), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "", "") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "") +CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "", "") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "", "") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "") +CK_PR_BINARY(and, 64, uint64_t, uint64_t, "and", "", "") +CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "") +CK_PR_BINARY(or, 64, uint64_t, uint64_t, "orr", "", "") +CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "sub", "", "") +CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W, "w") \ + CK_PR_BINARY(add, S, T, T, "add", W, "w") \ + CK_PR_BINARY(or, S, T, T, "orr", W, "w") \ + CK_PR_BINARY(sub, S, T, T, "sub", W, "w") \ + CK_PR_BINARY(xor, S, T, T, "eor", W, "w") + +CK_PR_BINARY_S(32, uint32_t, "") +CK_PR_BINARY_S(uint, unsigned int, "") +CK_PR_BINARY_S(int, int, "") +CK_PR_BINARY_S(16, uint16_t, "h") +CK_PR_BINARY_S(8, uint8_t, "b") +CK_PR_BINARY_S(short, short, "h") +CK_PR_BINARY_S(char, char, "b") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r, tmp; + + __asm__ __volatile__("1:" + "ldxr %0, [%3];" + "add %1, %4, %0;" + "stxr %w2, %1, [%3];" + "cbnz %w2, 1b;" + : "=&r" (previous), + "=&r" (r), + "=&r" (tmp) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +CK_CC_INLINE static uint64_t +ck_pr_faa_64(uint64_t *target, uint64_t delta) +{ + uint64_t previous, r, tmp; + + __asm__ __volatile__("1:" + "ldxr %0, [%3];" + "add %1, %4, %0;" + "stxr %w2, %1, [%3];" + "cbnz %w2, 1b;" + : "=&r" (previous), + "=&r" (r), + "=&r" (tmp) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r, tmp; \ + __asm__ __volatile__("1:" \ + "ldxr" W " %w0, [%3];" \ + "add %w1, %w4, %w0;" \ + "stxr" W " %w2, %w1, [%3];" \ + "cbnz %w2, 1b;" \ + : "=&r" (previous), \ + "=&r" (r), \ + "=&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "") +CK_PR_FAA(uint, unsigned int, "") +CK_PR_FAA(int, int, "") +CK_PR_FAA(16, uint16_t, "h") +CK_PR_FAA(8, uint8_t, "b") +CK_PR_FAA(short, short, "h") +CK_PR_FAA(char, char, "b") + +#undef CK_PR_FAA + +#endif /* CK_PR_AARCH64_LLSC_H */ Property changes on: vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_llsc.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_lse.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_lse.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_lse.h (revision 310844) @@ -0,0 +1,298 @@ +/* + * Copyright 2009-2016 Samy Al Bahra. + * Copyright 2013-2016 Olivier Houchard. + * Copyright 2016 Alexey Kopytov. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_AARCH64_LSE_H +#define CK_PR_AARCH64_LSE_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +CK_CC_INLINE static bool +ck_pr_cas_64_2_value(uint64_t target[2], uint64_t compare[2], uint64_t set[2], uint64_t value[2]) +{ + uint64_t tmp1; + uint64_t tmp2; + register uint64_t x0 __asm__ ("x0") = compare[0]; + register uint64_t x1 __asm__ ("x1") = compare[1]; + register uint64_t x2 __asm__ ("x2") = set[0]; + register uint64_t x3 __asm__ ("x3") = set[1]; + + __asm__ __volatile__("casp %0, %1, %4, %5, [%6];" + "eor %2, %0, %7;" + "eor %3, %1, %8;" + "orr %2, %2, %3;" + : "+&r" (x0), "+&r" (x1), "=&r" (tmp1), "=&r" (tmp2) + : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1]) + : "memory"); + + value[0] = x0; + value[1] = x1; + + return (!!tmp1); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) +{ + return (ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set), + CK_CPP_CAST(uint64_t *, value))); +} + +CK_CC_INLINE static bool +ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) +{ + register uint64_t x0 __asm__ ("x0") = compare[0]; + register uint64_t x1 __asm__ ("x1") = compare[1]; + register uint64_t x2 __asm__ ("x2") = set[0]; + register uint64_t x3 __asm__ ("x3") = set[1]; + + __asm__ __volatile__("casp %0, %1, %2, %3, [%4];" + "eor %0, %0, %5;" + "eor %1, %1, %6;" + "orr %0, %0, %1;" + : "+&r" (x0), "+&r" (x1) + : "r" (x2), "r" (x3), "r" (target), "r" (compare[0]), "r" (compare[1]) + : "memory"); + + return (!!x0); +} +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *target, void *compare, void *set) +{ + return (ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, target), + CK_CPP_CAST(uint64_t *, compare), + CK_CPP_CAST(uint64_t *, set))); +} + + +#define CK_PR_CAS(N, M, T, W, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ + { \ + *(T *)value = compare; \ + __asm__ __volatile__( \ + "cas" W " %" R "0, %" R "2, [%1];" \ + : "+&r" (*(T *)value) \ + : "r" (target), \ + "r" (set) \ + : "memory"); \ + return (*(T *)value == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(M *target, T compare, T set) \ + { \ + T previous = compare; \ + __asm__ __volatile__( \ + "cas" W " %" R "0, %" R "2, [%1];" \ + : "+&r" (previous) \ + : "r" (target), \ + "r" (set) \ + : "memory"); \ + return (previous == compare); \ + } + +CK_PR_CAS(ptr, void, void *, "", "") + +#define CK_PR_CAS_S(N, M, W, R) CK_PR_CAS(N, M, M, W, R) +CK_PR_CAS_S(64, uint64_t, "", "") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_S(double, double, "", "") +#endif +CK_PR_CAS_S(32, uint32_t, "", "w") +CK_PR_CAS_S(uint, unsigned int, "", "w") +CK_PR_CAS_S(int, int, "", "w") +CK_PR_CAS_S(16, uint16_t, "h", "w") +CK_PR_CAS_S(8, uint8_t, "b", "w") +CK_PR_CAS_S(short, short, "h", "w") +CK_PR_CAS_S(char, char, "b", "w") + + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W, R) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__( \ + "swp" W " %" R "2, %" R "0, [%1];" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return (previous); \ + } + +CK_PR_FAS(64, uint64_t, uint64_t, "", "") +CK_PR_FAS(32, uint32_t, uint32_t, "", "w") +CK_PR_FAS(ptr, void, void *, "", "") +CK_PR_FAS(int, int, int, "", "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "", "w") +CK_PR_FAS(16, uint16_t, uint16_t, "h", "w") +CK_PR_FAS(8, uint8_t, uint8_t, "b", "w") +CK_PR_FAS(short, short, short, "h", "w") +CK_PR_FAS(char, char, char, "b", "w") + + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W, R, S) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + __asm__ __volatile__(I ";" \ + "st" S W " " R "0, [%0];" \ + : \ + : "r" (target) \ + : "x0", "memory"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "mov x0, 1", "", "x", "add") +CK_PR_UNARY(dec, ptr, void, void *, "mov x0, -1", "", "x", "add") +CK_PR_UNARY(not, ptr, void, void *, "mov x0, -1", "", "x", "eor") +CK_PR_UNARY(inc, 64, uint64_t, uint64_t, "mov x0, 1", "", "x", "add") +CK_PR_UNARY(dec, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "add") +CK_PR_UNARY(not, 64, uint64_t, uint64_t, "mov x0, -1", "", "x", "eor") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "mov w0, 1", W, "w", "add") \ + CK_PR_UNARY(dec, S, T, T, "mov w0, -1", W, "w", "add") \ + CK_PR_UNARY(not, S, T, T, "mov w0, -1", W, "w", "eor") \ + +CK_PR_UNARY_S(32, uint32_t, "") +CK_PR_UNARY_S(uint, unsigned int, "") +CK_PR_UNARY_S(int, int, "") +CK_PR_UNARY_S(16, uint16_t, "h") +CK_PR_UNARY_S(8, uint8_t, "b") +CK_PR_UNARY_S(short, short, "h") +CK_PR_UNARY_S(char, char, "b") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, S, W, R, I) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + __asm__ __volatile__(I ";" \ + "st" S W " %" R "0, [%1];" \ + : "+&r" (delta) \ + : "r" (target) \ + : "memory"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "clr", "", "", "mvn %0, %0") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "", "", "") +CK_PR_BINARY(or, ptr, void, uintptr_t, "set", "", "", "") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "add", "", "", "neg %0, %0") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "", "", "") +CK_PR_BINARY(and, 64, uint64_t, uint64_t, "clr", "", "", "mvn %0, %0") +CK_PR_BINARY(add, 64, uint64_t, uint64_t, "add", "", "", "") +CK_PR_BINARY(or, 64, uint64_t, uint64_t, "set", "", "", "") +CK_PR_BINARY(sub, 64, uint64_t, uint64_t, "add", "", "", "neg %0, %0") +CK_PR_BINARY(xor, 64, uint64_t, uint64_t, "eor", "", "", "") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "clr", W, "w", "mvn %w0, %w0") \ + CK_PR_BINARY(add, S, T, T, "add", W, "w", "") \ + CK_PR_BINARY(or, S, T, T, "set", W, "w", "") \ + CK_PR_BINARY(sub, S, T, T, "add", W, "w", "neg %w0, %w0") \ + CK_PR_BINARY(xor, S, T, T, "eor", W, "w", "") + +CK_PR_BINARY_S(32, uint32_t, "") +CK_PR_BINARY_S(uint, unsigned int, "") +CK_PR_BINARY_S(int, int, "") +CK_PR_BINARY_S(16, uint16_t, "h") +CK_PR_BINARY_S(8, uint8_t, "b") +CK_PR_BINARY_S(short, short, "h") +CK_PR_BINARY_S(char, char, "b") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous; + + __asm__ __volatile__( + "ldadd %2, %0, [%1];" + : "=r" (previous) + : "r" (target), + "r" (delta) + : "memory"); + + return (void *)(previous); +} + +CK_CC_INLINE static uint64_t +ck_pr_faa_64(uint64_t *target, uint64_t delta) +{ + uint64_t previous; + + __asm__ __volatile__( + "ldadd %2, %0, [%1];" + : "=r" (previous) + : "r" (target), + "r" (delta) + : "memory"); + + return (previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__( \ + "ldadd" W " %w2, %w0, [%1];" \ + : "=r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "") +CK_PR_FAA(uint, unsigned int, "") +CK_PR_FAA(int, int, "") +CK_PR_FAA(16, uint16_t, "h") +CK_PR_FAA(8, uint8_t, "b") +CK_PR_FAA(short, short, "h") +CK_PR_FAA(char, char, "b") + +#undef CK_PR_FAA + +#endif /* CK_PR_AARCH64_LSE_H */ Property changes on: vendor-sys/ck/20161230/include/gcc/aarch64/ck_pr_lse.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/aarch64/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/aarch64/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/aarch64/ck_f_pr.h (revision 310844) @@ -0,0 +1,167 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_SHORT +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_SHORT +#define CK_F_PR_AND_UINT +#define CK_F_PR_BARRIER +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_64_2 +#define CK_F_PR_CAS_64_2_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_SHORT +#define CK_F_PR_CAS_SHORT_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_SHORT +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_SHORT +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_SHORT +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_ATOMIC +#define CK_F_PR_FENCE_ATOMIC_LOAD +#define CK_F_PR_FENCE_ATOMIC_STORE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_ATOMIC +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_LOAD_STORE +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STORE_ATOMIC +#define CK_F_PR_FENCE_STORE_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC +#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC +#define CK_F_PR_FENCE_STRICT_LOAD_STORE +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC +#define CK_F_PR_FENCE_STRICT_STORE_LOAD +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_SHORT +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_SHORT +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_SHORT +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_SHORT +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_SHORT +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_SHORT +#define CK_F_PR_XOR_UINT Property changes on: vendor-sys/ck/20161230/include/gcc/aarch64/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/arm/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/arm/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/arm/ck_pr.h (revision 310844) @@ -0,0 +1,563 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2013-2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_ARM_H +#define CK_PR_ARM_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_ISB __asm __volatile("isb" : : "r" (0) : "memory") +#define CK_DMB __asm __volatile("dmb" : : "r" (0) : "memory") +#define CK_DSB __asm __volatile("dsb" : : "r" (0) : "memory") +/* FreeBSD's toolchain doesn't accept dmb st, so use the opcode instead */ +#ifdef __FreeBSD__ +#define CK_DMB_ST __asm __volatile(".word 0xf57ff05e" : : "r" (0) : "memory") +#else +#define CK_DMB_ST __asm __volatile("dmb st" : : "r" (0) : "memory") +#endif /* __FreeBSD__ */ +#else +/* armv6 doesn't have dsb/dmb/isb, and no way to wait only for stores */ +#define CK_ISB \ + __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") +#define CK_DSB \ + __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") +#define CK_DMB \ + __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory") +#define CK_DMB_ST CK_DMB +#endif + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + I; \ + } + +CK_PR_FENCE(atomic, CK_DMB_ST) +CK_PR_FENCE(atomic_store, CK_DMB_ST) +CK_PR_FENCE(atomic_load, CK_DMB_ST) +CK_PR_FENCE(store_atomic, CK_DMB_ST) +CK_PR_FENCE(load_atomic, CK_DMB) +CK_PR_FENCE(store, CK_DMB_ST) +CK_PR_FENCE(store_load, CK_DMB) +CK_PR_FENCE(load, CK_DMB) +CK_PR_FENCE(load_store, CK_DMB) +CK_PR_FENCE(memory, CK_DMB) +CK_PR_FENCE(acquire, CK_DMB) +CK_PR_FENCE(release, CK_DMB) +CK_PR_FENCE(acqrel, CK_DMB) +CK_PR_FENCE(lock, CK_DMB) +CK_PR_FENCE(unlock, CK_DMB) + +#undef CK_PR_FENCE + +#undef CK_ISB +#undef CK_DSB +#undef CK_DMB +#undef CK_DMB_ST + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + long r = 0; \ + __asm__ __volatile__(I " %0, [%1];" \ + : "=r" (r) \ + : "r" (target) \ + : "memory"); \ + return ((T)r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint32_t, "ldr") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(32, uint32_t, "ldr") +CK_PR_LOAD_S(16, uint16_t, "ldrh") +CK_PR_LOAD_S(8, uint8_t, "ldrb") +CK_PR_LOAD_S(uint, unsigned int, "ldr") +CK_PR_LOAD_S(int, int, "ldr") +CK_PR_LOAD_S(short, short, "ldrh") +CK_PR_LOAD_S(char, char, "ldrb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) + +#define CK_PR_DOUBLE_LOAD(T, N) \ +CK_CC_INLINE static T \ +ck_pr_md_load_##N(const T *target) \ +{ \ + register T ret; \ + \ + __asm __volatile("ldrexd %0, [%1]" \ + : "=&r" (ret) \ + : "r" (target) \ + : "memory", "cc"); \ + return (ret); \ +} + +CK_PR_DOUBLE_LOAD(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_LOAD(double, double) +#endif +#undef CK_PR_DOUBLE_LOAD +#endif + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, [%0]" \ + : \ + : "r" (target), \ + "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint32_t, "str") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(32, uint32_t, "str") +CK_PR_STORE_S(16, uint16_t, "strh") +CK_PR_STORE_S(8, uint8_t, "strb") +CK_PR_STORE_S(uint, unsigned int, "str") +CK_PR_STORE_S(int, int, "str") +CK_PR_STORE_S(short, short, "strh") +CK_PR_STORE_S(char, char, "strb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) + +#define CK_PR_DOUBLE_STORE(T, N) \ +CK_CC_INLINE static void \ +ck_pr_md_store_##N(const T *target, T value) \ +{ \ + T tmp; \ + uint32_t flag; \ + __asm __volatile("1: \n" \ + "ldrexd %0, [%2]\n" \ + "strexd %1, %3, [%2]\n" \ + "teq %1, #0\n" \ + "it ne \n" \ + "bne 1b\n" \ + : "=&r" (tmp), "=&r" (flag) \ + : "r" (target), "r" (value) \ + : "memory", "cc"); \ +} + +CK_PR_DOUBLE_STORE(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_STORE(double, double) +#endif + +#undef CK_PR_DOUBLE_STORE + +#define CK_PR_DOUBLE_CAS_VALUE(T, N) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ +{ \ + T previous; \ + int tmp; \ + \ + __asm__ __volatile__("1:" \ + "ldrexd %0, [%4];" \ + "cmp %Q0, %Q2;" \ + "ittt eq;" \ + "cmpeq %R0, %R2;" \ + "strexdeq %1, %3, [%4];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + :"=&r" (previous), "=&r" (tmp) \ + : "r" (compare), "r" (set) , \ + "r"(target) \ + : "memory", "cc"); \ + *value = previous; \ + return (*value == compare); \ +} + +CK_PR_DOUBLE_CAS_VALUE(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_CAS_VALUE(double, double) +#endif + +#undef CK_PR_DOUBLE_CAS_VALUE + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *target, void *compare, void *set, void *value) +{ + uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); + uint32_t *_set = CK_CPP_CAST(uint32_t *, set); + uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); + uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); + + return (ck_pr_cas_64_value(CK_CPP_CAST(uint64_t *, target), + __compare, + __set, + CK_CPP_CAST(uint64_t *, value))); +} + +#define CK_PR_DOUBLE_CAS(T, N) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##N(T *target, T compare, T set) \ +{ \ + int ret; \ + T tmp; \ + \ + __asm__ __volatile__("1:" \ + "mov %0, #0;" \ + "ldrexd %1, [%4];" \ + "cmp %Q1, %Q2;" \ + "itttt eq;" \ + "cmpeq %R1, %R2;" \ + "strexdeq %1, %3, [%4];" \ + "moveq %0, #1;" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + : "=&r" (ret), "=&r" (tmp) \ + : "r" (compare), "r" (set) , \ + "r"(target) \ + : "memory", "cc"); \ + \ + return (ret); \ +} + +CK_PR_DOUBLE_CAS(uint64_t, 64) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_DOUBLE_CAS(double, double) +#endif + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *target, void *compare, void *set) +{ + uint32_t *_compare = CK_CPP_CAST(uint32_t *, compare); + uint32_t *_set = CK_CPP_CAST(uint32_t *, set); + uint64_t __compare = ((uint64_t)_compare[0]) | ((uint64_t)_compare[1] << 32); + uint64_t __set = ((uint64_t)_set[0]) | ((uint64_t)_set[1] << 32); + return (ck_pr_cas_64(CK_CPP_CAST(uint64_t *, target), + __compare, + __set)); +} + +#endif + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) +{ + void *previous, *tmp; + __asm__ __volatile__("1:" + "ldrex %0, [%2];" + "cmp %0, %4;" + "itt eq;" + "strexeq %1, %3, [%2];" + "cmpeq %1, #1;" + "beq 1b;" + : "=&r" (previous), + "=&r" (tmp) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + *(void **)value = previous; + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + void *previous, *tmp; + __asm__ __volatile__("1:" + "ldrex %0, [%2];" + "cmp %0, %4;" + "itt eq;" + "strexeq %1, %3, [%2];" + "cmpeq %1, #1;" + "beq 1b;" + : "=&r" (previous), + "=&r" (tmp) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + return (previous == compare); +} + +#define CK_PR_CAS(N, T, W) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + T previous = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "cmp %0, %4;" \ + "itt eq;" \ + "strex" W "eq %1, %3, [%2];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + /* \ + * Using "+&" instead of "=&" to avoid bogus \ + * clang warnings. \ + */ \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + T previous = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "cmp %0, %4;" \ + "itt eq;" \ + "strex" W "eq %1, %3, [%2];" \ + "cmpeq %1, #1;" \ + "beq 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(32, uint32_t, "") +CK_PR_CAS(uint, unsigned int, "") +CK_PR_CAS(int, int, "") +CK_PR_CAS(16, uint16_t, "h") +CK_PR_CAS(8, uint8_t, "b") +CK_PR_CAS(short, short, "h") +CK_PR_CAS(char, char, "b") + + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + "strex" W " %1, %3, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(32, uint32_t, uint32_t, "") +CK_PR_FAS(ptr, void, void *, "") +CK_PR_FAS(int, int, int, "") +CK_PR_FAS(uint, unsigned int, unsigned int, "") +CK_PR_FAS(16, uint16_t, uint16_t, "h") +CK_PR_FAS(8, uint8_t, uint8_t, "b") +CK_PR_FAS(short, short, short, "h") +CK_PR_FAS(char, char, char, "b") + + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + I ";" \ + "strex" W " %1, %0, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "add %0, %0, #1", "") +CK_PR_UNARY(dec, ptr, void, void *, "sub %0, %0, #1", "") +CK_PR_UNARY(not, ptr, void, void *, "mvn %0, %0", "") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "add %0, %0, #1", W) \ + CK_PR_UNARY(dec, S, T, T, "sub %0, %0, #1", W) \ + CK_PR_UNARY(not, S, T, T, "mvn %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) \ + +CK_PR_UNARY_S(32, uint32_t, "") +CK_PR_UNARY_S(uint, unsigned int, "") +CK_PR_UNARY_S(int, int, "") +CK_PR_UNARY_S(16, uint16_t, "h") +CK_PR_UNARY_S(8, uint8_t, "b") +CK_PR_UNARY_S(short, short, "h") +CK_PR_UNARY_S(char, char, "b") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous = 0; \ + T tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%2];" \ + I " %0, %0, %3;" \ + "strex" W " %1, %0, [%2];" \ + "cmp %1, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "") +CK_PR_BINARY(or, ptr, void, uintptr_t, "orr", "") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "eor", "") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "orr", W) \ + CK_PR_BINARY(sub, S, T, T, "sub", W) \ + CK_PR_BINARY(xor, S, T, T, "eor", W) + +CK_PR_BINARY_S(32, uint32_t, "") +CK_PR_BINARY_S(uint, unsigned int, "") +CK_PR_BINARY_S(int, int, "") +CK_PR_BINARY_S(16, uint16_t, "h") +CK_PR_BINARY_S(8, uint8_t, "b") +CK_PR_BINARY_S(short, short, "h") +CK_PR_BINARY_S(char, char, "b") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r, tmp; + + __asm__ __volatile__("1:" + "ldrex %0, [%3];" + "add %1, %4, %0;" + "strex %2, %1, [%3];" + "cmp %2, #0;" + "bne 1b;" + : "=&r" (previous), + "=&r" (r), + "=&r" (tmp) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous = 0, r = 0, tmp = 0; \ + __asm__ __volatile__("1:" \ + "ldrex" W " %0, [%3];" \ + "add %1, %4, %0;" \ + "strex" W " %2, %1, [%3];" \ + "cmp %2, #0;" \ + "bne 1b;" \ + : "+&r" (previous), \ + "+&r" (r), \ + "+&r" (tmp) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "") +CK_PR_FAA(uint, unsigned int, "") +CK_PR_FAA(int, int, "") +CK_PR_FAA(16, uint16_t, "h") +CK_PR_FAA(8, uint8_t, "b") +CK_PR_FAA(short, short, "h") +CK_PR_FAA(char, char, "b") + +#undef CK_PR_FAA + +#endif /* CK_PR_ARM_H */ + Property changes on: vendor-sys/ck/20161230/include/gcc/arm/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/arm/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/arm/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/arm/ck_f_pr.h (revision 310844) @@ -0,0 +1,162 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_SHORT +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_SHORT +#define CK_F_PR_AND_UINT +#define CK_F_PR_BARRIER +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_VALUE +#endif +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#endif +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_SHORT +#define CK_F_PR_CAS_SHORT_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_SHORT +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_SHORT +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_SHORT +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_ATOMIC +#define CK_F_PR_FENCE_ATOMIC_LOAD +#define CK_F_PR_FENCE_ATOMIC_STORE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_ATOMIC +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_LOAD_STORE +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STORE_ATOMIC +#define CK_F_PR_FENCE_STORE_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC +#define CK_F_PR_FENCE_STRICT_ATOMIC_LOAD +#define CK_F_PR_FENCE_STRICT_ATOMIC_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_ATOMIC +#define CK_F_PR_FENCE_STRICT_LOAD_STORE +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_FENCE_STRICT_STORE_ATOMIC +#define CK_F_PR_FENCE_STRICT_STORE_LOAD +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_SHORT +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_DOUBLE +#endif +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_SHORT +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_SHORT +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_SHORT +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_DOUBLE +#endif +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_SHORT +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_SHORT +#define CK_F_PR_XOR_UINT Property changes on: vendor-sys/ck/20161230/include/gcc/arm/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/ck_pr.h (revision 310844) @@ -0,0 +1,297 @@ +/* + * Copyright 2010 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_GCC_H +#define CK_PR_GCC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include + +CK_CC_INLINE static void +ck_pr_barrier(void) +{ + + __asm__ __volatile__("" ::: "memory"); + return; +} + +#ifndef CK_F_PR +#define CK_F_PR + +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +#define CK_PR_ACCESS(x) (*(volatile __typeof__(x) *)&(x)) + +#define CK_PR_LOAD(S, M, T) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + ck_pr_barrier(); \ + r = CK_PR_ACCESS(*(const T *)target); \ + ck_pr_barrier(); \ + return (r); \ + } \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + ck_pr_barrier(); \ + CK_PR_ACCESS(*(T *)target) = v; \ + ck_pr_barrier(); \ + return; \ + } + +CK_CC_INLINE static void * +ck_pr_md_load_ptr(const void *target) +{ + void *r; + + ck_pr_barrier(); + r = CK_CC_DECONST_PTR(CK_PR_ACCESS(target)); + ck_pr_barrier(); + + return r; +} + +CK_CC_INLINE static void +ck_pr_md_store_ptr(void *target, const void *v) +{ + + ck_pr_barrier(); + CK_PR_ACCESS(target) = CK_CC_DECONST_PTR(v); + ck_pr_barrier(); + return; +} + +#define CK_PR_LOAD_S(S, T) CK_PR_LOAD(S, T, T) + +CK_PR_LOAD_S(char, char) +CK_PR_LOAD_S(uint, unsigned int) +CK_PR_LOAD_S(int, int) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double) +#endif +CK_PR_LOAD_S(64, uint64_t) +CK_PR_LOAD_S(32, uint32_t) +CK_PR_LOAD_S(16, uint16_t) +CK_PR_LOAD_S(8, uint8_t) + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + ck_pr_barrier(); +} + +/* + * Load and store fences are equivalent to full fences in the GCC port. + */ +#define CK_PR_FENCE(T) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __sync_synchronize(); \ + } + +CK_PR_FENCE(atomic) +CK_PR_FENCE(atomic_atomic) +CK_PR_FENCE(atomic_load) +CK_PR_FENCE(atomic_store) +CK_PR_FENCE(store_atomic) +CK_PR_FENCE(load_atomic) +CK_PR_FENCE(load) +CK_PR_FENCE(load_load) +CK_PR_FENCE(load_store) +CK_PR_FENCE(store) +CK_PR_FENCE(store_store) +CK_PR_FENCE(store_load) +CK_PR_FENCE(memory) +CK_PR_FENCE(acquire) +CK_PR_FENCE(release) +CK_PR_FENCE(acqrel) +CK_PR_FENCE(lock) +CK_PR_FENCE(unlock) + +#undef CK_PR_FENCE + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + z = __sync_bool_compare_and_swap((T *)target, compare, set); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *) + +#define CK_PR_CAS_S(S, T) CK_PR_CAS(S, T, T) + +CK_PR_CAS_S(char, char) +CK_PR_CAS_S(int, int) +CK_PR_CAS_S(uint, unsigned int) +CK_PR_CAS_S(64, uint64_t) +CK_PR_CAS_S(32, uint32_t) +CK_PR_CAS_S(16, uint16_t) +CK_PR_CAS_S(8, uint8_t) + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *v) +{ + set = __sync_val_compare_and_swap((void **)target, compare, set); + *(void **)v = set; + return (set == compare); +} + +#define CK_PR_CAS_O(S, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(T *target, T compare, T set, T *v) \ + { \ + set = __sync_val_compare_and_swap(target, compare, set);\ + *v = set; \ + return (set == compare); \ + } + +CK_PR_CAS_O(char, char) +CK_PR_CAS_O(int, int) +CK_PR_CAS_O(uint, unsigned int) +CK_PR_CAS_O(64, uint64_t) +CK_PR_CAS_O(32, uint32_t) +CK_PR_CAS_O(16, uint16_t) +CK_PR_CAS_O(8, uint8_t) + +#undef CK_PR_CAS_O + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + d = __sync_fetch_and_add((T *)target, d); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, void *) + +#define CK_PR_FAA_S(S, T) CK_PR_FAA(S, T, T) + +CK_PR_FAA_S(char, char) +CK_PR_FAA_S(uint, unsigned int) +CK_PR_FAA_S(int, int) +CK_PR_FAA_S(64, uint64_t) +CK_PR_FAA_S(32, uint32_t) +CK_PR_FAA_S(16, uint16_t) +CK_PR_FAA_S(8, uint8_t) + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + d = __sync_fetch_and_##K((T *)target, d); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T) CK_PR_BINARY(K, S, T, T) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, void *) \ + CK_PR_BINARY_S(K, char, char) \ + CK_PR_BINARY_S(K, int, int) \ + CK_PR_BINARY_S(K, uint, unsigned int) \ + CK_PR_BINARY_S(K, 64, uint64_t) \ + CK_PR_BINARY_S(K, 32, uint32_t) \ + CK_PR_BINARY_S(K, 16, uint16_t) \ + CK_PR_BINARY_S(K, 8, uint8_t) + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +#define CK_PR_UNARY(S, M, T) \ + CK_CC_INLINE static void \ + ck_pr_inc_##S(M *target) \ + { \ + ck_pr_add_##S(target, (T)1); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_pr_dec_##S(M *target) \ + { \ + ck_pr_sub_##S(target, (T)1); \ + return; \ + } + +#define CK_PR_UNARY_S(S, M) CK_PR_UNARY(S, M, M) + +CK_PR_UNARY(ptr, void, void *) +CK_PR_UNARY_S(char, char) +CK_PR_UNARY_S(int, int) +CK_PR_UNARY_S(uint, unsigned int) +CK_PR_UNARY_S(64, uint64_t) +CK_PR_UNARY_S(32, uint32_t) +CK_PR_UNARY_S(16, uint16_t) +CK_PR_UNARY_S(8, uint8_t) + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY +#endif /* !CK_F_PR */ +#endif /* CK_PR_GCC_H */ Property changes on: vendor-sys/ck/20161230/include/gcc/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/ppc64/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/ppc64/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/ppc64/ck_pr.h (revision 310844) @@ -0,0 +1,427 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_PPC64_H +#define CK_PR_PPC64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * This bounces the hardware thread from low to medium + * priority. I am unsure of the benefits of this approach + * but it is used by the Linux kernel. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("or 1, 1, 1;" + "or 2, 2, 2;" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +/* + * These are derived from: + * http://www.ibm.com/developerworks/systems/articles/powerpc.html + */ +CK_PR_FENCE(atomic, "lwsync") +CK_PR_FENCE(atomic_store, "lwsync") +CK_PR_FENCE(atomic_load, "sync") +CK_PR_FENCE(store_atomic, "lwsync") +CK_PR_FENCE(load_atomic, "lwsync") +CK_PR_FENCE(store, "lwsync") +CK_PR_FENCE(store_load, "sync") +CK_PR_FENCE(load, "lwsync") +CK_PR_FENCE(load_store, "lwsync") +CK_PR_FENCE(memory, "sync") +CK_PR_FENCE(acquire, "lwsync") +CK_PR_FENCE(release, "lwsync") +CK_PR_FENCE(acqrel, "lwsync") +CK_PR_FENCE(lock, "lwsync") +CK_PR_FENCE(unlock, "lwsync") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I "%U1%X1 %0, %1" \ + : "=r" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint64_t, "ld") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(64, uint64_t, "ld") +CK_PR_LOAD_S(32, uint32_t, "lwz") +CK_PR_LOAD_S(16, uint16_t, "lhz") +CK_PR_LOAD_S(8, uint8_t, "lbz") +CK_PR_LOAD_S(uint, unsigned int, "lwz") +CK_PR_LOAD_S(int, int, "lwz") +CK_PR_LOAD_S(short, short, "lhz") +CK_PR_LOAD_S(char, char, "lbz") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double, "ld") +#endif + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I "%U0%X0 %1, %0" \ + : "=m" (*(C *)target) \ + : "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint64_t, "std") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(64, uint64_t, "std") +CK_PR_STORE_S(32, uint32_t, "stw") +CK_PR_STORE_S(16, uint16_t, "sth") +CK_PR_STORE_S(8, uint8_t, "stb") +CK_PR_STORE_S(uint, unsigned int, "stw") +CK_PR_STORE_S(int, int, "stw") +CK_PR_STORE_S(short, short, "sth") +CK_PR_STORE_S(char, char, "stb") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE_S(double, double, "std") +#endif + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +CK_CC_INLINE static bool +ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) +{ + uint64_t previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + *value = previous; + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value) +{ + void *previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + ck_pr_md_store_ptr(value, previous); + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) +{ + uint64_t previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + return (previous == compare); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + void *previous; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %1;" + "cmpd 0, %0, %3;" + "bne- 2f;" + "stdcx. %2, 0, %1;" + "bne- 1b;" + "2:" + : "=&r" (previous) + : "r" (target), + "r" (set), + "r" (compare) + : "memory", "cc"); + + return (previous == compare); +} + +#define CK_PR_CAS(N, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(32, uint32_t) +CK_PR_CAS(uint, unsigned int) +CK_PR_CAS(int, int) + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + "st" W "cx. %2, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(64, uint64_t, uint64_t, "d") +CK_PR_FAS(32, uint32_t, uint32_t, "w") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_FAS(double, double, double, "d") +#endif +CK_PR_FAS(ptr, void, void *, "d") +CK_PR_FAS(int, int, int, "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "w") + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I ";" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d") +CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d") +CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ + CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ + CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) + +CK_PR_UNARY_S(64, uint64_t, "d") +CK_PR_UNARY_S(32, uint32_t, "w") +CK_PR_UNARY_S(uint, unsigned int, "w") +CK_PR_UNARY_S(int, int, "w") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I " %0, %2, %0;" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d") +CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "or", W) \ + CK_PR_BINARY(sub, S, T, T, "subf", W) \ + CK_PR_BINARY(xor, S, T, T, "xor", W) + +CK_PR_BINARY_S(64, uint64_t, "d") +CK_PR_BINARY_S(32, uint32_t, "w") +CK_PR_BINARY_S(uint, unsigned int, "w") +CK_PR_BINARY_S(int, int, "w") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r; + + __asm__ __volatile__("1:" + "ldarx %0, 0, %2;" + "add %1, %3, %0;" + "stdcx. %1, 0, %2;" + "bne- 1b;" + : "=&r" (previous), + "=&r" (r) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %2;" \ + "add %1, %3, %0;" \ + "st" W "cx. %1, 0, %2;" \ + "bne- 1b;" \ + : "=&r" (previous), \ + "=&r" (r) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(64, uint64_t, "d") +CK_PR_FAA(32, uint32_t, "w") +CK_PR_FAA(uint, unsigned int, "w") +CK_PR_FAA(int, int, "w") + +#undef CK_PR_FAA + +#endif /* CK_PR_PPC64_H */ Property changes on: vendor-sys/ck/20161230/include/gcc/ppc64/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/ppc64/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/ppc64/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/ppc64/ck_f_pr.h (revision 310844) @@ -0,0 +1,97 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + Property changes on: vendor-sys/ck/20161230/include/gcc/ppc64/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr.h (revision 310844) @@ -0,0 +1,585 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_X86_64_H +#define CK_PR_X86_64_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Support for TSX extensions. + */ +#ifdef CK_MD_RTM_ENABLE +#include "ck_pr_rtm.h" +#endif + +/* Minimum requirements for the CK_PR interface are met. */ +#define CK_F_PR + +#ifdef CK_MD_UMP +#define CK_PR_LOCK_PREFIX +#else +#define CK_PR_LOCK_PREFIX "lock " +#endif + +/* + * Prevent speculative execution in busy-wait loops (P4 <=) + * or "predefined delay". + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + __asm__ __volatile__("pause" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +CK_PR_FENCE(atomic, "sfence") +CK_PR_FENCE(atomic_store, "sfence") +CK_PR_FENCE(atomic_load, "mfence") +CK_PR_FENCE(store_atomic, "sfence") +CK_PR_FENCE(load_atomic, "mfence") +CK_PR_FENCE(load, "lfence") +CK_PR_FENCE(load_store, "mfence") +CK_PR_FENCE(store, "sfence") +CK_PR_FENCE(store_load, "mfence") +CK_PR_FENCE(memory, "mfence") +CK_PR_FENCE(release, "mfence") +CK_PR_FENCE(acquire, "mfence") +CK_PR_FENCE(acqrel, "mfence") +CK_PR_FENCE(lock, "mfence") +CK_PR_FENCE(unlock, "mfence") + +#undef CK_PR_FENCE + +/* + * Read for ownership. Older compilers will generate the 32-bit + * 3DNow! variant which is binary compatible with x86-64 variant + * of prefetchw. + */ +#ifndef CK_F_PR_RFO +#define CK_F_PR_RFO +CK_CC_INLINE static void +ck_pr_rfo(const void *m) +{ + + __asm__ __volatile__("prefetchw (%0)" + : + : "r" (m) + : "memory"); + + return; +} +#endif /* CK_F_PR_RFO */ + +/* + * Atomic fetch-and-store operations. + */ +#define CK_PR_FAS(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_fas_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, %1" \ + : "+m" (*(C *)target), \ + "+q" (v) \ + : \ + : "memory"); \ + return v; \ + } + +CK_PR_FAS(ptr, void, void *, char, "xchgq") + +#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I) + +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_FAS_S(double, double, "xchgq") +#endif +CK_PR_FAS_S(char, char, "xchgb") +CK_PR_FAS_S(uint, unsigned int, "xchgl") +CK_PR_FAS_S(int, int, "xchgl") +CK_PR_FAS_S(64, uint64_t, "xchgq") +CK_PR_FAS_S(32, uint32_t, "xchgl") +CK_PR_FAS_S(16, uint16_t, "xchgw") +CK_PR_FAS_S(8, uint8_t, "xchgb") + +#undef CK_PR_FAS_S +#undef CK_PR_FAS + +/* + * Atomic load-from-memory operations. + */ +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " %1, %0" \ + : "=q" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, char, "movq") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(char, char, "movb") +CK_PR_LOAD_S(uint, unsigned int, "movl") +CK_PR_LOAD_S(int, int, "movl") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_LOAD_S(double, double, "movq") +#endif +CK_PR_LOAD_S(64, uint64_t, "movq") +CK_PR_LOAD_S(32, uint32_t, "movl") +CK_PR_LOAD_S(16, uint16_t, "movw") +CK_PR_LOAD_S(8, uint8_t, "movb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +CK_CC_INLINE static void +ck_pr_load_64_2(const uint64_t target[2], uint64_t v[2]) +{ + __asm__ __volatile__("movq %%rdx, %%rcx;" + "movq %%rax, %%rbx;" + CK_PR_LOCK_PREFIX "cmpxchg16b %2;" + : "=a" (v[0]), + "=d" (v[1]) + : "m" (*(const uint64_t *)target) + : "rbx", "rcx", "memory", "cc"); + return; +} + +CK_CC_INLINE static void +ck_pr_load_ptr_2(const void *t, void *v) +{ + ck_pr_load_64_2(CK_CPP_CAST(const uint64_t *, t), + CK_CPP_CAST(uint64_t *, v)); + return; +} + +#define CK_PR_LOAD_2(S, W, T) \ + CK_CC_INLINE static void \ + ck_pr_md_load_##S##_##W(const T t[2], T v[2]) \ + { \ + ck_pr_load_64_2((const uint64_t *)(const void *)t, \ + (uint64_t *)(void *)v); \ + return; \ + } + +CK_PR_LOAD_2(char, 16, char) +CK_PR_LOAD_2(int, 4, int) +CK_PR_LOAD_2(uint, 4, unsigned int) +CK_PR_LOAD_2(32, 4, uint32_t) +CK_PR_LOAD_2(16, 8, uint16_t) +CK_PR_LOAD_2(8, 16, uint8_t) + +#undef CK_PR_LOAD_2 + +/* + * Atomic store-to-memory operations. + */ +#define CK_PR_STORE_IMM(S, M, T, C, I, K) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : K "q" (v) \ + : "memory"); \ + return; \ + } + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : "q" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE_IMM(ptr, void, const void *, char, "movq", CK_CC_IMM_U32) +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_STORE(double, double, double, double, "movq") +#endif + +#define CK_PR_STORE_S(S, T, I, K) CK_PR_STORE_IMM(S, T, T, T, I, K) + +CK_PR_STORE_S(char, char, "movb", CK_CC_IMM_S32) +CK_PR_STORE_S(int, int, "movl", CK_CC_IMM_S32) +CK_PR_STORE_S(uint, unsigned int, "movl", CK_CC_IMM_U32) +CK_PR_STORE_S(64, uint64_t, "movq", CK_CC_IMM_U32) +CK_PR_STORE_S(32, uint32_t, "movl", CK_CC_IMM_U32) +CK_PR_STORE_S(16, uint16_t, "movw", CK_CC_IMM_U32) +CK_PR_STORE_S(8, uint8_t, "movb", CK_CC_IMM_U32) + +#undef CK_PR_STORE_S +#undef CK_PR_STORE_IMM +#undef CK_PR_STORE + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target), \ + "+q" (d) \ + : \ + : "memory", "cc"); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, uintptr_t, char, "xaddq") + +#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I) + +CK_PR_FAA_S(char, char, "xaddb") +CK_PR_FAA_S(uint, unsigned int, "xaddl") +CK_PR_FAA_S(int, int, "xaddl") +CK_PR_FAA_S(64, uint64_t, "xaddq") +CK_PR_FAA_S(32, uint32_t, "xaddl") +CK_PR_FAA_S(16, uint16_t, "xaddw") +CK_PR_FAA_S(8, uint8_t, "xaddb") + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only unary operations. + */ +#define CK_PR_UNARY(K, S, T, C, I) \ + CK_PR_UNARY_R(K, S, T, C, I) \ + CK_PR_UNARY_V(K, S, T, C, I) + +#define CK_PR_UNARY_R(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(T *target) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \ + : "+m" (*(C *)target) \ + : \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_UNARY_V(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(T *target, bool *r) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \ + : "+m" (*(C *)target), \ + "=m" (*r) \ + : \ + : "memory", "cc"); \ + return; \ + } + + +#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_UNARY(K, ptr, void, char, #K "q") \ + CK_PR_UNARY_S(K, char, char, #K "b") \ + CK_PR_UNARY_S(K, int, int, #K "l") \ + CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_UNARY_S(K, 64, uint64_t, #K "q") \ + CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_UNARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(inc) +CK_PR_GENERATE(dec) +CK_PR_GENERATE(neg) + +/* not does not affect condition flags. */ +#undef CK_PR_UNARY_V +#define CK_PR_UNARY_V(a, b, c, d, e) +CK_PR_GENERATE(not) + +#undef CK_PR_GENERATE +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_V +#undef CK_PR_UNARY_R +#undef CK_PR_UNARY + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T, C, I, O) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target) \ + : O "q" (d) \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T, I, O) CK_PR_BINARY(K, S, T, T, T, I, O) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "q", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, char, char, #K "b", CK_CC_IMM_S32) \ + CK_PR_BINARY_S(K, int, int, #K "l", CK_CC_IMM_S32) \ + CK_PR_BINARY_S(K, uint, unsigned int, #K "l", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 64, uint64_t, #K "q", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 32, uint32_t, #K "l", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 16, uint16_t, #K "w", CK_CC_IMM_U32) \ + CK_PR_BINARY_S(K, 8, uint8_t, #K "b", CK_CC_IMM_U32) + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \ + : "+m" (*(C *)target), \ + "=a" (z) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *, char, "cmpxchgq") + +#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I) + +CK_PR_CAS_S(char, char, "cmpxchgb") +CK_PR_CAS_S(int, int, "cmpxchgl") +CK_PR_CAS_S(uint, unsigned int, "cmpxchgl") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_S(double, double, "cmpxchgq") +#endif +CK_PR_CAS_S(64, uint64_t, "cmpxchgq") +CK_PR_CAS_S(32, uint32_t, "cmpxchgl") +CK_PR_CAS_S(16, uint16_t, "cmpxchgw") +CK_PR_CAS_S(8, uint8_t, "cmpxchgb") + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +#define CK_PR_CAS_O(S, M, T, C, I, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \ + "mov %% " R ", %2;" \ + "setz %1;" \ + : "+m" (*(C *)target), \ + "=a" (z), \ + "=m" (*(C *)v) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS_O(ptr, void, void *, char, "q", "rax") + +#define CK_PR_CAS_O_S(S, T, I, R) \ + CK_PR_CAS_O(S, T, T, T, I, R) + +CK_PR_CAS_O_S(char, char, "b", "al") +CK_PR_CAS_O_S(int, int, "l", "eax") +CK_PR_CAS_O_S(uint, unsigned int, "l", "eax") +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_O_S(double, double, "q", "rax") +#endif +CK_PR_CAS_O_S(64, uint64_t, "q", "rax") +CK_PR_CAS_O_S(32, uint32_t, "l", "eax") +CK_PR_CAS_O_S(16, uint16_t, "w", "ax") +CK_PR_CAS_O_S(8, uint8_t, "b", "al") + +#undef CK_PR_CAS_O_S +#undef CK_PR_CAS_O + +/* + * Contrary to C-interface, alignment requirements are that of uint64_t[2]. + */ +CK_CC_INLINE static bool +ck_pr_cas_64_2(uint64_t target[2], uint64_t compare[2], uint64_t set[2]) +{ + bool z; + + __asm__ __volatile__("movq 0(%4), %%rax;" + "movq 8(%4), %%rdx;" + CK_PR_LOCK_PREFIX "cmpxchg16b %0; setz %1" + : "+m" (*target), + "=q" (z) + : "b" (set[0]), + "c" (set[1]), + "q" (compare) + : "memory", "cc", "%rax", "%rdx"); + return z; +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2(void *t, void *c, void *s) +{ + return ck_pr_cas_64_2(CK_CPP_CAST(uint64_t *, t), + CK_CPP_CAST(uint64_t *, c), + CK_CPP_CAST(uint64_t *, s)); +} + +CK_CC_INLINE static bool +ck_pr_cas_64_2_value(uint64_t target[2], + uint64_t compare[2], + uint64_t set[2], + uint64_t v[2]) +{ + bool z; + + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg16b %0;" + "setz %3" + : "+m" (*target), + "=a" (v[0]), + "=d" (v[1]), + "=q" (z) + : "a" (compare[0]), + "d" (compare[1]), + "b" (set[0]), + "c" (set[1]) + : "memory", "cc"); + return z; +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_2_value(void *t, void *c, void *s, void *v) +{ + return ck_pr_cas_64_2_value(CK_CPP_CAST(uint64_t *,t), + CK_CPP_CAST(uint64_t *,c), + CK_CPP_CAST(uint64_t *,s), + CK_CPP_CAST(uint64_t *,v)); +} + +#define CK_PR_CAS_V(S, W, T) \ +CK_CC_INLINE static bool \ +ck_pr_cas_##S##_##W(T t[W], T c[W], T s[W]) \ +{ \ + return ck_pr_cas_64_2((uint64_t *)(void *)t, \ + (uint64_t *)(void *)c, \ + (uint64_t *)(void *)s); \ +} \ +CK_CC_INLINE static bool \ +ck_pr_cas_##S##_##W##_value(T *t, T c[W], T s[W], T *v) \ +{ \ + return ck_pr_cas_64_2_value((uint64_t *)(void *)t, \ + (uint64_t *)(void *)c, \ + (uint64_t *)(void *)s, \ + (uint64_t *)(void *)v); \ +} + +#ifndef CK_PR_DISABLE_DOUBLE +CK_PR_CAS_V(double, 2, double) +#endif +CK_PR_CAS_V(char, 16, char) +CK_PR_CAS_V(int, 4, int) +CK_PR_CAS_V(uint, 4, unsigned int) +CK_PR_CAS_V(32, 4, uint32_t) +CK_PR_CAS_V(16, 8, uint16_t) +CK_PR_CAS_V(8, 16, uint8_t) + +#undef CK_PR_CAS_V + +/* + * Atomic bit test operations. + */ +#define CK_PR_BT(K, S, T, P, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(T *target, unsigned int b) \ + { \ + bool c; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \ + : "+m" (*(C *)target), \ + "=q" (c) \ + : "q" ((P)b) \ + : "memory", "cc"); \ + return c; \ + } + +#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BT(K, ptr, void, uint64_t, char, #K "q %2, %0") \ + CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \ + CK_PR_BT_S(K, int, int, #K "l %2, %0") \ + CK_PR_BT_S(K, 64, uint64_t, #K "q %2, %0") \ + CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \ + CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0") + +CK_PR_GENERATE(btc) +CK_PR_GENERATE(bts) +CK_PR_GENERATE(btr) + +#undef CK_PR_GENERATE +#undef CK_PR_BT + +#endif /* CK_PR_X86_64_H */ + Property changes on: vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/x86_64/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/x86_64/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/x86_64/ck_f_pr.h (revision 310844) @@ -0,0 +1,202 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_BTC_16 +#define CK_F_PR_BTC_32 +#define CK_F_PR_BTC_64 +#define CK_F_PR_BTC_INT +#define CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_UINT +#define CK_F_PR_BTR_16 +#define CK_F_PR_BTR_32 +#define CK_F_PR_BTR_64 +#define CK_F_PR_BTR_INT +#define CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_UINT +#define CK_F_PR_BTS_16 +#define CK_F_PR_BTS_32 +#define CK_F_PR_BTS_64 +#define CK_F_PR_BTS_INT +#define CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_8 +#define CK_F_PR_CAS_16_8_VALUE +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_4 +#define CK_F_PR_CAS_32_4_VALUE +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_2 +#define CK_F_PR_CAS_64_2_VALUE +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_16 +#define CK_F_PR_CAS_8_16_VALUE +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_16 +#define CK_F_PR_CAS_CHAR_16_VALUE +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_4 +#define CK_F_PR_CAS_INT_4_VALUE +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_2 +#define CK_F_PR_CAS_PTR_2_VALUE +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_DOUBLE +#define CK_F_PR_CAS_DOUBLE_2 +#define CK_F_PR_CAS_DOUBLE_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_4 +#define CK_F_PR_CAS_UINT_4_VALUE +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_64_ZERO +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_64 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_DOUBLE +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_64_ZERO +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_16_8 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_32_4 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_64_2 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_8_16 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_CHAR_16 +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_INT_4 +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_PTR_2 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_LOAD_UINT_4 +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_64 +#define CK_F_PR_NEG_64_ZERO +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_64 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + Property changes on: vendor-sys/ck/20161230/include/gcc/x86_64/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr_rtm.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr_rtm.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr_rtm.h (revision 310844) @@ -0,0 +1,109 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Copyright (c) 2012,2013 Intel Corporation + * Author: Andi Kleen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: (1) source code distributions + * retain the above copyright notice and this paragraph in its entirety, (2) + * distributions including binary code include the above copyright notice and + * this paragraph in its entirety in the documentation or other materials + * provided with the distribution + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +#ifndef CK_PR_X86_64_RTM_H +#define CK_PR_X86_64_RTM_H + +#ifndef CK_PR_X86_64_H +#error Do not include this file directly, use ck_pr.h +#endif + +#define CK_F_PR_RTM + +#include +#include + +#define CK_PR_RTM_STARTED (~0U) +#define CK_PR_RTM_EXPLICIT (1 << 0) +#define CK_PR_RTM_RETRY (1 << 1) +#define CK_PR_RTM_CONFLICT (1 << 2) +#define CK_PR_RTM_CAPACITY (1 << 3) +#define CK_PR_RTM_DEBUG (1 << 4) +#define CK_PR_RTM_NESTED (1 << 5) +#define CK_PR_RTM_CODE(x) (((x) >> 24) & 0xFF) + +CK_CC_INLINE static unsigned int +ck_pr_rtm_begin(void) +{ + unsigned int r = CK_PR_RTM_STARTED; + + __asm__ __volatile__(".byte 0xc7,0xf8;" + ".long 0;" + : "+a" (r) + : + : "memory"); + + return r; +} + +CK_CC_INLINE static void +ck_pr_rtm_end(void) +{ + + __asm__ __volatile__(".byte 0x0f,0x01,0xd5" ::: "memory"); + return; +} + +CK_CC_INLINE static void +ck_pr_rtm_abort(const unsigned int status) +{ + + __asm__ __volatile__(".byte 0xc6,0xf8,%P0" :: "i" (status) : "memory"); + return; +} + +CK_CC_INLINE static bool +ck_pr_rtm_test(void) +{ + bool r; + + __asm__ __volatile__(".byte 0x0f,0x01,0xd6;" + "setnz %0" + : "=r" (r) + : + : "memory"); + + return r; +} + +#endif /* CK_PR_X86_64_RTM_H */ + Property changes on: vendor-sys/ck/20161230/include/gcc/x86_64/ck_pr_rtm.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/ck_cc.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/ck_cc.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/ck_cc.h (revision 310844) @@ -0,0 +1,142 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_GCC_CC_H +#define CK_GCC_CC_H + +#include + +#ifdef __SUNPRO_C +#define CK_CC_UNUSED +#define CK_CC_USED +#define CK_CC_IMM +#define CK_CC_IMM_U32 +#else +#define CK_CC_UNUSED __attribute__((unused)) +#define CK_CC_USED __attribute__((used)) +#define CK_CC_IMM "i" +#if defined(__x86_64__) || defined(__x86__) +#define CK_CC_IMM_U32 "Z" +#define CK_CC_IMM_S32 "e" +#else +#define CK_CC_IMM_U32 CK_CC_IMM +#define CK_CC_IMM_S32 CK_CC_IMM +#endif /* __x86_64__ || __x86__ */ +#endif + +#ifdef __OPTIMIZE__ +#define CK_CC_INLINE CK_CC_UNUSED inline +#else +#define CK_CC_INLINE CK_CC_UNUSED +#endif + +#define CK_CC_FORCE_INLINE CK_CC_UNUSED __attribute__((always_inline)) inline +#define CK_CC_RESTRICT __restrict__ + +/* + * Packed attribute. + */ +#define CK_CC_PACKED __attribute__((packed)) + +/* + * Weak reference. + */ +#define CK_CC_WEAKREF __attribute__((weakref)) + +/* + * Alignment attribute. + */ +#define CK_CC_ALIGN(B) __attribute__((aligned(B))) + +/* + * Cache align. + */ +#define CK_CC_CACHELINE CK_CC_ALIGN(CK_MD_CACHELINE) + +/* + * These are functions which should be avoided. + */ +#ifdef __freestanding__ +#pragma GCC poison malloc free +#endif + +/* + * Branch execution hints. + */ +#define CK_CC_LIKELY(x) (__builtin_expect(!!(x), 1)) +#define CK_CC_UNLIKELY(x) (__builtin_expect(!!(x), 0)) + +/* + * Some compilers are overly strict regarding aliasing semantics. + * Unfortunately, in many cases it makes more sense to pay aliasing + * cost rather than overly expensive register spillage. + */ +#define CK_CC_ALIASED __attribute__((__may_alias__)) + +/* + * Compile-time typeof + */ +#define CK_CC_TYPEOF(X, DEFAULT) __typeof__(X) + +/* + * Portability wrappers for bitwise ops. + */ + +#define CK_F_CC_FFS +#define CK_F_CC_CLZ +#define CK_F_CC_CTZ +#define CK_F_CC_POPCOUNT + +CK_CC_INLINE static int +ck_cc_ffs(unsigned int x) +{ + + return __builtin_ffs(x); +} + +CK_CC_INLINE static int +ck_cc_clz(unsigned int x) +{ + + return __builtin_clz(x); +} + +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + + return __builtin_ctz(x); +} + +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + + return __builtin_popcount(x); +} + +#endif /* CK_GCC_CC_H */ Property changes on: vendor-sys/ck/20161230/include/gcc/ck_cc.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/ck_f_pr.h (revision 310844) @@ -0,0 +1,105 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_64 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_64 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_64 +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_64 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_64 +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_64 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_64 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_64 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT Property changes on: vendor-sys/ck/20161230/include/gcc/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/ppc/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/ppc/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/ppc/ck_f_pr.h (revision 310844) @@ -0,0 +1,79 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_UINT +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_UINT +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_SHORT +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_SHORT +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + Property changes on: vendor-sys/ck/20161230/include/gcc/ppc/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/ppc/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/ppc/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/ppc/ck_pr.h (revision 310844) @@ -0,0 +1,327 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2012 João Fernandes. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_PPC_H +#define CK_PR_PPC_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * This bounces the hardware thread from low to medium + * priority. I am unsure of the benefits of this approach + * but it is used by the Linux kernel. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("or 1, 1, 1;" + "or 2, 2, 2;" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +CK_PR_FENCE(atomic, "lwsync") +CK_PR_FENCE(atomic_store, "lwsync") +CK_PR_FENCE(atomic_load, "sync") +CK_PR_FENCE(store_atomic, "lwsync") +CK_PR_FENCE(load_atomic, "lwsync") +CK_PR_FENCE(store, "lwsync") +CK_PR_FENCE(store_load, "sync") +CK_PR_FENCE(load, "lwsync") +CK_PR_FENCE(load_store, "lwsync") +CK_PR_FENCE(memory, "sync") +CK_PR_FENCE(acquire, "lwsync") +CK_PR_FENCE(release, "lwsync") +CK_PR_FENCE(acqrel, "lwsync") +CK_PR_FENCE(lock, "lwsync") +CK_PR_FENCE(unlock, "lwsync") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I "%U1%X1 %0, %1" \ + : "=r" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint32_t, "lwz") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(32, uint32_t, "lwz") +CK_PR_LOAD_S(16, uint16_t, "lhz") +CK_PR_LOAD_S(8, uint8_t, "lbz") +CK_PR_LOAD_S(uint, unsigned int, "lwz") +CK_PR_LOAD_S(int, int, "lwz") +CK_PR_LOAD_S(short, short, "lhz") +CK_PR_LOAD_S(char, char, "lbz") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I "%U0%X0 %1, %0" \ + : "=m" (*(C *)target) \ + : "r" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint32_t, "stw") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(32, uint32_t, "stw") +CK_PR_STORE_S(16, uint16_t, "sth") +CK_PR_STORE_S(8, uint8_t, "stb") +CK_PR_STORE_S(uint, unsigned int, "stw") +CK_PR_STORE_S(int, int, "stw") +CK_PR_STORE_S(short, short, "sth") +CK_PR_STORE_S(char, char, "stb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +#define CK_PR_CAS(N, T, M) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(M *target, T compare, T set, M *value) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + *(T *)value = previous; \ + return (previous == compare); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(M *target, T compare, T set) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "lwarx %0, 0, %1;" \ + "cmpw 0, %0, %3;" \ + "bne- 2f;" \ + "stwcx. %2, 0, %1;" \ + "bne- 1b;" \ + "2:" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (set), \ + "r" (compare) \ + : "memory", "cc"); \ + return (previous == compare); \ + } + +CK_PR_CAS(ptr, void *, void) +#define CK_PR_CAS_S(a, b) CK_PR_CAS(a, b, b) +CK_PR_CAS_S(32, uint32_t) +CK_PR_CAS_S(uint, unsigned int) +CK_PR_CAS_S(int, int) + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +#define CK_PR_FAS(N, M, T, W) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(M *target, T v) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + "st" W "cx. %2, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (v) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAS(32, uint32_t, uint32_t, "w") +CK_PR_FAS(ptr, void, void *, "w") +CK_PR_FAS(int, int, int, "w") +CK_PR_FAS(uint, unsigned int, unsigned int, "w") + +#undef CK_PR_FAS + +#define CK_PR_UNARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I ";" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "w") +CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "w") +CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "w") +CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "w") + +#define CK_PR_UNARY_S(S, T, W) \ + CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \ + CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \ + CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \ + CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W) + +CK_PR_UNARY_S(32, uint32_t, "w") +CK_PR_UNARY_S(uint, unsigned int, "w") +CK_PR_UNARY_S(int, int, "w") + +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY + +#define CK_PR_BINARY(O, N, M, T, I, W) \ + CK_CC_INLINE static void \ + ck_pr_##O##_##N(M *target, T delta) \ + { \ + T previous; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %1;" \ + I " %0, %2, %0;" \ + "st" W "cx. %0, 0, %1;" \ + "bne- 1b;" \ + : "=&r" (previous) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return; \ + } + +CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "w") +CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "w") +CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "w") +CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "w") +CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "w") + +#define CK_PR_BINARY_S(S, T, W) \ + CK_PR_BINARY(and, S, T, T, "and", W) \ + CK_PR_BINARY(add, S, T, T, "add", W) \ + CK_PR_BINARY(or, S, T, T, "or", W) \ + CK_PR_BINARY(sub, S, T, T, "subf", W) \ + CK_PR_BINARY(xor, S, T, T, "xor", W) + +CK_PR_BINARY_S(32, uint32_t, "w") +CK_PR_BINARY_S(uint, unsigned int, "w") +CK_PR_BINARY_S(int, int, "w") + +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +CK_CC_INLINE static void * +ck_pr_faa_ptr(void *target, uintptr_t delta) +{ + uintptr_t previous, r; + + __asm__ __volatile__("1:" + "lwarx %0, 0, %2;" + "add %1, %3, %0;" + "stwcx. %1, 0, %2;" + "bne- 1b;" + : "=&r" (previous), + "=&r" (r) + : "r" (target), + "r" (delta) + : "memory", "cc"); + + return (void *)(previous); +} + +#define CK_PR_FAA(S, T, W) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(T *target, T delta) \ + { \ + T previous, r; \ + __asm__ __volatile__("1:" \ + "l" W "arx %0, 0, %2;" \ + "add %1, %3, %0;" \ + "st" W "cx. %1, 0, %2;" \ + "bne- 1b;" \ + : "=&r" (previous), \ + "=&r" (r) \ + : "r" (target), \ + "r" (delta) \ + : "memory", "cc"); \ + return (previous); \ + } + +CK_PR_FAA(32, uint32_t, "w") +CK_PR_FAA(uint, unsigned int, "w") +CK_PR_FAA(int, int, "w") + +#undef CK_PR_FAA + +#endif /* CK_PR_PPC_H */ + Property changes on: vendor-sys/ck/20161230/include/gcc/ppc/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/sparcv9/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/sparcv9/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/sparcv9/ck_f_pr.h (revision 310844) @@ -0,0 +1,26 @@ +#define CK_F_PR_CAS_64 +#define CK_F_PR_CAS_64_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FAS_INT +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_STORE_64 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_DOUBLE +#define CK_F_PR_STORE_UINT +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_LOAD_64 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_DOUBLE +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR + Property changes on: vendor-sys/ck/20161230/include/gcc/sparcv9/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/sparcv9/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/sparcv9/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/sparcv9/ck_pr.h (revision 310844) @@ -0,0 +1,228 @@ +/* + * Copyright 2009, 2010 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_SPARCV9_H +#define CK_PR_SPARCV9_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* + * Minimum interface requirement met. + */ +#define CK_F_PR + +/* + * Order loads at the least. + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + + __asm__ __volatile__("membar #LoadLoad" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +/* + * Atomic operations are treated as both load and store + * operations on SPARCv9. + */ +CK_PR_FENCE(atomic, "membar #StoreStore") +CK_PR_FENCE(atomic_store, "membar #StoreStore") +CK_PR_FENCE(atomic_load, "membar #StoreLoad") +CK_PR_FENCE(store_atomic, "membar #StoreStore") +CK_PR_FENCE(load_atomic, "membar #LoadStore") +CK_PR_FENCE(store, "membar #StoreStore") +CK_PR_FENCE(store_load, "membar #StoreLoad") +CK_PR_FENCE(load, "membar #LoadLoad") +CK_PR_FENCE(load_store, "membar #LoadStore") +CK_PR_FENCE(memory, "membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad") +CK_PR_FENCE(acquire, "membar #LoadLoad | #LoadStore") +CK_PR_FENCE(release, "membar #LoadStore | #StoreStore") +CK_PR_FENCE(acqrel, "membar #LoadLoad | #LoadStore | #StoreStore") +CK_PR_FENCE(lock, "membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad") +CK_PR_FENCE(unlock, "membar #LoadStore | #StoreStore") + +#undef CK_PR_FENCE + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " [%1], %0" \ + : "=&r" (r) \ + : "r" (target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, uint64_t, "ldx") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(64, uint64_t, "ldx") +CK_PR_LOAD_S(32, uint32_t, "lduw") +CK_PR_LOAD_S(uint, unsigned int, "lduw") +CK_PR_LOAD_S(double, double, "ldx") +CK_PR_LOAD_S(int, int, "ldsw") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, [%1]" \ + : \ + : "r" (v), \ + "r" (target) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, uint64_t, "stx") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(8, uint8_t, "stub") +CK_PR_STORE_S(64, uint64_t, "stx") +CK_PR_STORE_S(32, uint32_t, "stuw") +CK_PR_STORE_S(uint, unsigned int, "stuw") +CK_PR_STORE_S(double, double, "stx") +CK_PR_STORE_S(int, int, "stsw") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +CK_CC_INLINE static bool +ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value) +{ + + __asm__ __volatile__("casx [%1], %2, %0" + : "+&r" (set) + : "r" (target), + "r" (compare) + : "memory"); + + *value = set; + return (compare == set); +} + +CK_CC_INLINE static bool +ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set) +{ + + __asm__ __volatile__("casx [%1], %2, %0" + : "+&r" (set) + : "r" (target), + "r" (compare) + : "memory"); + + return (compare == set); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr(void *target, void *compare, void *set) +{ + + return ck_pr_cas_64(target, (uint64_t)compare, (uint64_t)set); +} + +CK_CC_INLINE static bool +ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *previous) +{ + + return ck_pr_cas_64_value(target, (uint64_t)compare, (uint64_t)set, previous); +} + +#define CK_PR_CAS(N, T) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \ + { \ + __asm__ __volatile__("cas [%1], %2, %0" \ + : "+&r" (set) \ + : "r" (target), \ + "r" (compare) \ + : "memory"); \ + *value = set; \ + return (compare == set); \ + } \ + CK_CC_INLINE static bool \ + ck_pr_cas_##N(T *target, T compare, T set) \ + { \ + __asm__ __volatile__("cas [%1], %2, %0" \ + : "+&r" (set) \ + : "r" (target), \ + "r" (compare) \ + : "memory"); \ + return (compare == set); \ + } + +CK_PR_CAS(32, uint32_t) +CK_PR_CAS(uint, unsigned int) +CK_PR_CAS(int, int) + +#undef CK_PR_CAS + +#define CK_PR_FAS(N, T) \ + CK_CC_INLINE static T \ + ck_pr_fas_##N(T *target, T update) \ + { \ + \ + __asm__ __volatile__("swap [%1], %0" \ + : "+&r" (update) \ + : "r" (target) \ + : "memory"); \ + return (update); \ + } + +CK_PR_FAS(int, int) +CK_PR_FAS(uint, unsigned int) +CK_PR_FAS(32, uint32_t) + +#undef CK_PR_FAS + +#endif /* CK_PR_SPARCV9_H */ + Property changes on: vendor-sys/ck/20161230/include/gcc/sparcv9/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/x86/ck_f_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/x86/ck_f_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/x86/ck_f_pr.h (revision 310844) @@ -0,0 +1,152 @@ +/* DO NOT EDIT. This is auto-generated from feature.sh */ +#define CK_F_PR_ADD_16 +#define CK_F_PR_ADD_32 +#define CK_F_PR_ADD_8 +#define CK_F_PR_ADD_CHAR +#define CK_F_PR_ADD_INT +#define CK_F_PR_ADD_PTR +#define CK_F_PR_ADD_UINT +#define CK_F_PR_AND_16 +#define CK_F_PR_AND_32 +#define CK_F_PR_AND_8 +#define CK_F_PR_AND_CHAR +#define CK_F_PR_AND_INT +#define CK_F_PR_AND_PTR +#define CK_F_PR_AND_UINT +#define CK_F_PR_BTC_16 +#define CK_F_PR_BTC_32 +#define CK_F_PR_BTC_INT +#define CK_F_PR_BTC_PTR +#define CK_F_PR_BTC_UINT +#define CK_F_PR_BTR_16 +#define CK_F_PR_BTR_32 +#define CK_F_PR_BTR_INT +#define CK_F_PR_BTR_PTR +#define CK_F_PR_BTR_UINT +#define CK_F_PR_BTS_16 +#define CK_F_PR_BTS_32 +#define CK_F_PR_BTS_INT +#define CK_F_PR_BTS_PTR +#define CK_F_PR_BTS_UINT +#define CK_F_PR_CAS_16 +#define CK_F_PR_CAS_16_VALUE +#define CK_F_PR_CAS_32 +#define CK_F_PR_CAS_32_VALUE +#define CK_F_PR_CAS_8 +#define CK_F_PR_CAS_8_VALUE +#define CK_F_PR_CAS_CHAR +#define CK_F_PR_CAS_CHAR_VALUE +#define CK_F_PR_CAS_INT +#define CK_F_PR_CAS_INT_VALUE +#define CK_F_PR_CAS_PTR +#define CK_F_PR_CAS_PTR_VALUE +#define CK_F_PR_CAS_UINT +#define CK_F_PR_CAS_UINT_VALUE +#define CK_F_PR_DEC_16 +#define CK_F_PR_DEC_16_ZERO +#define CK_F_PR_DEC_32 +#define CK_F_PR_DEC_32_ZERO +#define CK_F_PR_DEC_8 +#define CK_F_PR_DEC_8_ZERO +#define CK_F_PR_DEC_CHAR +#define CK_F_PR_DEC_CHAR_ZERO +#define CK_F_PR_DEC_INT +#define CK_F_PR_DEC_INT_ZERO +#define CK_F_PR_DEC_PTR +#define CK_F_PR_DEC_PTR_ZERO +#define CK_F_PR_DEC_UINT +#define CK_F_PR_DEC_UINT_ZERO +#define CK_F_PR_FAA_16 +#define CK_F_PR_FAA_32 +#define CK_F_PR_FAA_8 +#define CK_F_PR_FAA_CHAR +#define CK_F_PR_FAA_INT +#define CK_F_PR_FAA_PTR +#define CK_F_PR_FAA_UINT +#define CK_F_PR_FAS_16 +#define CK_F_PR_FAS_32 +#define CK_F_PR_FAS_8 +#define CK_F_PR_FAS_CHAR +#define CK_F_PR_FAS_INT +#define CK_F_PR_FAS_PTR +#define CK_F_PR_FAS_UINT +#define CK_F_PR_FENCE_LOAD +#define CK_F_PR_FENCE_LOAD_DEPENDS +#define CK_F_PR_FENCE_MEMORY +#define CK_F_PR_FENCE_STORE +#define CK_F_PR_FENCE_STRICT_LOAD +#define CK_F_PR_FENCE_STRICT_LOAD_DEPENDS +#define CK_F_PR_FENCE_STRICT_MEMORY +#define CK_F_PR_FENCE_STRICT_STORE +#define CK_F_PR_INC_16 +#define CK_F_PR_INC_16_ZERO +#define CK_F_PR_INC_32 +#define CK_F_PR_INC_32_ZERO +#define CK_F_PR_INC_8 +#define CK_F_PR_INC_8_ZERO +#define CK_F_PR_INC_CHAR +#define CK_F_PR_INC_CHAR_ZERO +#define CK_F_PR_INC_INT +#define CK_F_PR_INC_INT_ZERO +#define CK_F_PR_INC_PTR +#define CK_F_PR_INC_PTR_ZERO +#define CK_F_PR_INC_UINT +#define CK_F_PR_INC_UINT_ZERO +#define CK_F_PR_LOAD_16 +#define CK_F_PR_LOAD_32 +#define CK_F_PR_LOAD_8 +#define CK_F_PR_LOAD_CHAR +#define CK_F_PR_LOAD_INT +#define CK_F_PR_LOAD_PTR +#define CK_F_PR_LOAD_UINT +#define CK_F_PR_NEG_16 +#define CK_F_PR_NEG_16_ZERO +#define CK_F_PR_NEG_32 +#define CK_F_PR_NEG_32_ZERO +#define CK_F_PR_NEG_8 +#define CK_F_PR_NEG_8_ZERO +#define CK_F_PR_NEG_CHAR +#define CK_F_PR_NEG_CHAR_ZERO +#define CK_F_PR_NEG_INT +#define CK_F_PR_NEG_INT_ZERO +#define CK_F_PR_NEG_PTR +#define CK_F_PR_NEG_PTR_ZERO +#define CK_F_PR_NEG_UINT +#define CK_F_PR_NEG_UINT_ZERO +#define CK_F_PR_NOT_16 +#define CK_F_PR_NOT_32 +#define CK_F_PR_NOT_8 +#define CK_F_PR_NOT_CHAR +#define CK_F_PR_NOT_INT +#define CK_F_PR_NOT_PTR +#define CK_F_PR_NOT_UINT +#define CK_F_PR_OR_16 +#define CK_F_PR_OR_32 +#define CK_F_PR_OR_8 +#define CK_F_PR_OR_CHAR +#define CK_F_PR_OR_INT +#define CK_F_PR_OR_PTR +#define CK_F_PR_OR_UINT +#define CK_F_PR_STALL +#define CK_F_PR_STORE_16 +#define CK_F_PR_STORE_32 +#define CK_F_PR_STORE_8 +#define CK_F_PR_STORE_CHAR +#define CK_F_PR_STORE_INT +#define CK_F_PR_STORE_PTR +#define CK_F_PR_STORE_UINT +#define CK_F_PR_SUB_16 +#define CK_F_PR_SUB_32 +#define CK_F_PR_SUB_8 +#define CK_F_PR_SUB_CHAR +#define CK_F_PR_SUB_INT +#define CK_F_PR_SUB_PTR +#define CK_F_PR_SUB_UINT +#define CK_F_PR_XOR_16 +#define CK_F_PR_XOR_32 +#define CK_F_PR_XOR_8 +#define CK_F_PR_XOR_CHAR +#define CK_F_PR_XOR_INT +#define CK_F_PR_XOR_PTR +#define CK_F_PR_XOR_UINT + Property changes on: vendor-sys/ck/20161230/include/gcc/x86/ck_f_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/gcc/x86/ck_pr.h =================================================================== --- vendor-sys/ck/20161230/include/gcc/x86/ck_pr.h (nonexistent) +++ vendor-sys/ck/20161230/include/gcc/x86/ck_pr.h (revision 310844) @@ -0,0 +1,390 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2011 Devon H. O'Dell + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PR_X86_H +#define CK_PR_X86_H + +#ifndef CK_PR_H +#error Do not include this file directly, use ck_pr.h +#endif + +#include +#include +#include + +/* + * The following represent supported atomic operations. + * These operations may be emulated. + */ +#include "ck_f_pr.h" + +/* Minimum requirements for the CK_PR interface are met. */ +#define CK_F_PR + +#ifdef CK_MD_UMP +#define CK_PR_LOCK_PREFIX +#else +#define CK_PR_LOCK_PREFIX "lock " +#endif + +/* + * Prevent speculative execution in busy-wait loops (P4 <=) + * or "predefined delay". + */ +CK_CC_INLINE static void +ck_pr_stall(void) +{ + __asm__ __volatile__("pause" ::: "memory"); + return; +} + +#define CK_PR_FENCE(T, I) \ + CK_CC_INLINE static void \ + ck_pr_fence_strict_##T(void) \ + { \ + __asm__ __volatile__(I ::: "memory"); \ + } + +CK_PR_FENCE(atomic, "sfence") +CK_PR_FENCE(atomic_store, "sfence") +CK_PR_FENCE(atomic_load, "mfence") +CK_PR_FENCE(store_atomic, "sfence") +CK_PR_FENCE(load_atomic, "mfence") +CK_PR_FENCE(load, "lfence") +CK_PR_FENCE(load_store, "mfence") +CK_PR_FENCE(store, "sfence") +CK_PR_FENCE(store_load, "mfence") +CK_PR_FENCE(memory, "mfence") +CK_PR_FENCE(release, "mfence") +CK_PR_FENCE(acquire, "mfence") +CK_PR_FENCE(acqrel, "mfence") +CK_PR_FENCE(lock, "mfence") +CK_PR_FENCE(unlock, "mfence") + +#undef CK_PR_FENCE + +/* + * Atomic fetch-and-store operations. + */ +#define CK_PR_FAS(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_fas_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %0, %1" \ + : "+m" (*(C *)target), \ + "+q" (v) \ + : \ + : "memory"); \ + return v; \ + } + +CK_PR_FAS(ptr, void, void *, char, "xchgl") + +#define CK_PR_FAS_S(S, T, I) CK_PR_FAS(S, T, T, T, I) + +CK_PR_FAS_S(char, char, "xchgb") +CK_PR_FAS_S(uint, unsigned int, "xchgl") +CK_PR_FAS_S(int, int, "xchgl") +CK_PR_FAS_S(32, uint32_t, "xchgl") +CK_PR_FAS_S(16, uint16_t, "xchgw") +CK_PR_FAS_S(8, uint8_t, "xchgb") + +#undef CK_PR_FAS_S +#undef CK_PR_FAS + +#define CK_PR_LOAD(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_md_load_##S(const M *target) \ + { \ + T r; \ + __asm__ __volatile__(I " %1, %0" \ + : "=q" (r) \ + : "m" (*(const C *)target) \ + : "memory"); \ + return (r); \ + } + +CK_PR_LOAD(ptr, void, void *, char, "movl") + +#define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I) + +CK_PR_LOAD_S(char, char, "movb") +CK_PR_LOAD_S(uint, unsigned int, "movl") +CK_PR_LOAD_S(int, int, "movl") +CK_PR_LOAD_S(32, uint32_t, "movl") +CK_PR_LOAD_S(16, uint16_t, "movw") +CK_PR_LOAD_S(8, uint8_t, "movb") + +#undef CK_PR_LOAD_S +#undef CK_PR_LOAD + +#define CK_PR_STORE(S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_md_store_##S(M *target, T v) \ + { \ + __asm__ __volatile__(I " %1, %0" \ + : "=m" (*(C *)target) \ + : CK_CC_IMM "q" (v) \ + : "memory"); \ + return; \ + } + +CK_PR_STORE(ptr, void, const void *, char, "movl") + +#define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I) + +CK_PR_STORE_S(char, char, "movb") +CK_PR_STORE_S(uint, unsigned int, "movl") +CK_PR_STORE_S(int, int, "movl") +CK_PR_STORE_S(32, uint32_t, "movl") +CK_PR_STORE_S(16, uint16_t, "movw") +CK_PR_STORE_S(8, uint8_t, "movb") + +#undef CK_PR_STORE_S +#undef CK_PR_STORE + +/* + * Atomic fetch-and-add operations. + */ +#define CK_PR_FAA(S, M, T, C, I) \ + CK_CC_INLINE static T \ + ck_pr_faa_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target), \ + "+q" (d) \ + : \ + : "memory", "cc"); \ + return (d); \ + } + +CK_PR_FAA(ptr, void, uintptr_t, char, "xaddl") + +#define CK_PR_FAA_S(S, T, I) CK_PR_FAA(S, T, T, T, I) + +CK_PR_FAA_S(char, char, "xaddb") +CK_PR_FAA_S(uint, unsigned int, "xaddl") +CK_PR_FAA_S(int, int, "xaddl") +CK_PR_FAA_S(32, uint32_t, "xaddl") +CK_PR_FAA_S(16, uint16_t, "xaddw") +CK_PR_FAA_S(8, uint8_t, "xaddb") + +#undef CK_PR_FAA_S +#undef CK_PR_FAA + +/* + * Atomic store-only unary operations. + */ +#define CK_PR_UNARY(K, S, T, C, I) \ + CK_PR_UNARY_R(K, S, T, C, I) \ + CK_PR_UNARY_V(K, S, T, C, I) + +#define CK_PR_UNARY_R(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(T *target) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0" \ + : "+m" (*(C *)target) \ + : \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_UNARY_V(K, S, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S##_zero(T *target, bool *r) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \ + : "+m" (*(C *)target), \ + "=m" (*r) \ + : \ + : "memory", "cc"); \ + return; \ + } + + +#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_UNARY(K, ptr, void, char, #K "l") \ + CK_PR_UNARY_S(K, char, char, #K "b") \ + CK_PR_UNARY_S(K, int, int, #K "l") \ + CK_PR_UNARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_UNARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_UNARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_UNARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(inc) +CK_PR_GENERATE(dec) +CK_PR_GENERATE(neg) + +/* not does not affect condition flags. */ +#undef CK_PR_UNARY_V +#define CK_PR_UNARY_V(a, b, c, d, e) +CK_PR_GENERATE(not) + +#undef CK_PR_GENERATE +#undef CK_PR_UNARY_S +#undef CK_PR_UNARY_V +#undef CK_PR_UNARY_R +#undef CK_PR_UNARY + +/* + * Atomic store-only binary operations. + */ +#define CK_PR_BINARY(K, S, M, T, C, I) \ + CK_CC_INLINE static void \ + ck_pr_##K##_##S(M *target, T d) \ + { \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %1, %0" \ + : "+m" (*(C *)target) \ + : CK_CC_IMM "q" (d) \ + : "memory", "cc"); \ + return; \ + } + +#define CK_PR_BINARY_S(K, S, T, I) CK_PR_BINARY(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BINARY(K, ptr, void, uintptr_t, char, #K "l") \ + CK_PR_BINARY_S(K, char, char, #K "b") \ + CK_PR_BINARY_S(K, int, int, #K "l") \ + CK_PR_BINARY_S(K, uint, unsigned int, #K "l") \ + CK_PR_BINARY_S(K, 32, uint32_t, #K "l") \ + CK_PR_BINARY_S(K, 16, uint16_t, #K "w") \ + CK_PR_BINARY_S(K, 8, uint8_t, #K "b") + +CK_PR_GENERATE(add) +CK_PR_GENERATE(sub) +CK_PR_GENERATE(and) +CK_PR_GENERATE(or) +CK_PR_GENERATE(xor) + +#undef CK_PR_GENERATE +#undef CK_PR_BINARY_S +#undef CK_PR_BINARY + +/* + * Atomic compare and swap. + */ +#define CK_PR_CAS(S, M, T, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S(M *target, T compare, T set) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I " %2, %0; setz %1" \ + : "+m" (*(C *)target), \ + "=a" (z) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return z; \ + } + +CK_PR_CAS(ptr, void, void *, char, "cmpxchgl") + +#define CK_PR_CAS_S(S, T, I) CK_PR_CAS(S, T, T, T, I) + +CK_PR_CAS_S(char, char, "cmpxchgb") +CK_PR_CAS_S(int, int, "cmpxchgl") +CK_PR_CAS_S(uint, unsigned int, "cmpxchgl") +CK_PR_CAS_S(32, uint32_t, "cmpxchgl") +CK_PR_CAS_S(16, uint16_t, "cmpxchgw") +CK_PR_CAS_S(8, uint8_t, "cmpxchgb") + +#undef CK_PR_CAS_S +#undef CK_PR_CAS + +/* + * Compare and swap, set *v to old value of target. + */ +#define CK_PR_CAS_O(S, M, T, C, I, R) \ + CK_CC_INLINE static bool \ + ck_pr_cas_##S##_value(M *target, T compare, T set, M *v) \ + { \ + bool z; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX "cmpxchg" I " %3, %0;" \ + "mov %% " R ", %2;" \ + "setz %1;" \ + : "+m" (*(C *)target), \ + "=a" (z), \ + "=m" (*(C *)v) \ + : "q" (set), \ + "a" (compare) \ + : "memory", "cc"); \ + return (bool)z; \ + } + +CK_PR_CAS_O(ptr, void, void *, char, "l", "eax") + +#define CK_PR_CAS_O_S(S, T, I, R) \ + CK_PR_CAS_O(S, T, T, T, I, R) + +CK_PR_CAS_O_S(char, char, "b", "al") +CK_PR_CAS_O_S(int, int, "l", "eax") +CK_PR_CAS_O_S(uint, unsigned int, "l", "eax") +CK_PR_CAS_O_S(32, uint32_t, "l", "eax") +CK_PR_CAS_O_S(16, uint16_t, "w", "ax") +CK_PR_CAS_O_S(8, uint8_t, "b", "al") + +#undef CK_PR_CAS_O_S +#undef CK_PR_CAS_O + +/* + * Atomic bit test operations. + */ +#define CK_PR_BT(K, S, T, P, C, I) \ + CK_CC_INLINE static bool \ + ck_pr_##K##_##S(T *target, unsigned int b) \ + { \ + bool c; \ + __asm__ __volatile__(CK_PR_LOCK_PREFIX I "; setc %1" \ + : "+m" (*(C *)target), \ + "=q" (c) \ + : "q" ((P)b) \ + : "memory", "cc"); \ + return (bool)c; \ + } + +#define CK_PR_BT_S(K, S, T, I) CK_PR_BT(K, S, T, T, T, I) + +#define CK_PR_GENERATE(K) \ + CK_PR_BT(K, ptr, void, uint32_t, char, #K "l %2, %0") \ + CK_PR_BT_S(K, uint, unsigned int, #K "l %2, %0") \ + CK_PR_BT_S(K, int, int, #K "l %2, %0") \ + CK_PR_BT_S(K, 32, uint32_t, #K "l %2, %0") \ + CK_PR_BT_S(K, 16, uint16_t, #K "w %w2, %0") + +CK_PR_GENERATE(btc) +CK_PR_GENERATE(bts) +CK_PR_GENERATE(btr) + +#undef CK_PR_GENERATE +#undef CK_PR_BT + +#endif /* CK_PR_X86_H */ + Property changes on: vendor-sys/ck/20161230/include/gcc/x86/ck_pr.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_array.h =================================================================== --- vendor-sys/ck/20161230/include/ck_array.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_array.h (revision 310844) @@ -0,0 +1,100 @@ +/* + * Copyright 2013-2015 Samy Al Bahra + * Copyright 2013-2014 AppNexus, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_ARRAY_H +#define CK_ARRAY_H + +#include +#include +#include +#include +#include + +struct _ck_array { + unsigned int n_committed; + unsigned int length; + void *values[]; +}; + +struct ck_array { + struct ck_malloc *allocator; + struct _ck_array *active; + unsigned int n_entries; + struct _ck_array *transaction; +}; +typedef struct ck_array ck_array_t; + +struct ck_array_iterator { + struct _ck_array *snapshot; +}; +typedef struct ck_array_iterator ck_array_iterator_t; + +#define CK_ARRAY_MODE_SPMC 0U +#define CK_ARRAY_MODE_MPMC (void) /* Unsupported. */ + +bool ck_array_init(ck_array_t *, unsigned int, struct ck_malloc *, unsigned int); +bool ck_array_commit(ck_array_t *); +bool ck_array_put(ck_array_t *, void *); +int ck_array_put_unique(ck_array_t *, void *); +bool ck_array_remove(ck_array_t *, void *); +void ck_array_deinit(ck_array_t *, bool); + +CK_CC_INLINE static unsigned int +ck_array_length(struct ck_array *array) +{ + struct _ck_array *a = ck_pr_load_ptr(&array->active); + + ck_pr_fence_load(); + return ck_pr_load_uint(&a->n_committed); +} + +CK_CC_INLINE static void * +ck_array_buffer(struct ck_array *array, unsigned int *length) +{ + struct _ck_array *a = ck_pr_load_ptr(&array->active); + + ck_pr_fence_load(); + *length = ck_pr_load_uint(&a->n_committed); + return a->values; +} + +CK_CC_INLINE static bool +ck_array_initialized(struct ck_array *array) +{ + + return ck_pr_load_ptr(&array->active) != NULL; +} + +#define CK_ARRAY_FOREACH(a, i, b) \ + (i)->snapshot = ck_pr_load_ptr(&(a)->active); \ + ck_pr_fence_load(); \ + for (unsigned int _ck_i = 0; \ + _ck_i < (a)->active->n_committed && \ + ((*b) = (a)->active->values[_ck_i], 1); \ + _ck_i++) + +#endif /* CK_ARRAY_H */ Property changes on: vendor-sys/ck/20161230/include/ck_array.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_backoff.h =================================================================== --- vendor-sys/ck/20161230/include/ck_backoff.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_backoff.h (revision 310844) @@ -0,0 +1,57 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BACKOFF_H +#define CK_BACKOFF_H + +#include +#include + +#ifndef CK_BACKOFF_CEILING +#define CK_BACKOFF_CEILING ((1 << 20) - 1) +#endif + +#define CK_BACKOFF_INITIALIZER (1 << 9) + +typedef unsigned int ck_backoff_t; + +/* + * This is a exponential back-off implementation. + */ +CK_CC_INLINE static void +ck_backoff_eb(unsigned int *c) +{ + unsigned int i, ceiling; + + ceiling = *c; + for (i = 0; i < ceiling; i++) + ck_pr_barrier(); + + *c = ceiling <<= ceiling < CK_BACKOFF_CEILING; + return; +} + +#endif /* CK_BACKOFF_H */ Property changes on: vendor-sys/ck/20161230/include/ck_backoff.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_barrier.h =================================================================== --- vendor-sys/ck/20161230/include/ck_barrier.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_barrier.h (revision 310844) @@ -0,0 +1,164 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BARRIER_H +#define CK_BARRIER_H + +#include + +struct ck_barrier_centralized { + unsigned int value; + unsigned int sense; +}; +typedef struct ck_barrier_centralized ck_barrier_centralized_t; + +struct ck_barrier_centralized_state { + unsigned int sense; +}; +typedef struct ck_barrier_centralized_state ck_barrier_centralized_state_t; + +#define CK_BARRIER_CENTRALIZED_INITIALIZER {0, 0} +#define CK_BARRIER_CENTRALIZED_STATE_INITIALIZER {0} + +void ck_barrier_centralized(ck_barrier_centralized_t *, + ck_barrier_centralized_state_t *, unsigned int); + +struct ck_barrier_combining_group { + unsigned int k; + unsigned int count; + unsigned int sense; + struct ck_barrier_combining_group *parent; + struct ck_barrier_combining_group *left; + struct ck_barrier_combining_group *right; + struct ck_barrier_combining_group *next; +} CK_CC_CACHELINE; +typedef struct ck_barrier_combining_group ck_barrier_combining_group_t; + +struct ck_barrier_combining_state { + unsigned int sense; +}; +typedef struct ck_barrier_combining_state ck_barrier_combining_state_t; + +#define CK_BARRIER_COMBINING_STATE_INITIALIZER {~0} + +struct ck_barrier_combining { + struct ck_barrier_combining_group *root; + ck_spinlock_fas_t mutex; +}; +typedef struct ck_barrier_combining ck_barrier_combining_t; + +void ck_barrier_combining_init(ck_barrier_combining_t *, ck_barrier_combining_group_t *); + +void ck_barrier_combining_group_init(ck_barrier_combining_t *, + ck_barrier_combining_group_t *, unsigned int); + +void ck_barrier_combining(ck_barrier_combining_t *, + ck_barrier_combining_group_t *, + ck_barrier_combining_state_t *); + +struct ck_barrier_dissemination_flag { + unsigned int tflag; + unsigned int *pflag; +}; +typedef struct ck_barrier_dissemination_flag ck_barrier_dissemination_flag_t; + +struct ck_barrier_dissemination { + unsigned int nthr; + unsigned int size; + unsigned int tid; + struct ck_barrier_dissemination_flag *flags[2]; +}; +typedef struct ck_barrier_dissemination ck_barrier_dissemination_t; + +struct ck_barrier_dissemination_state { + int parity; + unsigned int sense; + unsigned int tid; +}; +typedef struct ck_barrier_dissemination_state ck_barrier_dissemination_state_t; + +void ck_barrier_dissemination_init(ck_barrier_dissemination_t *, + ck_barrier_dissemination_flag_t **, unsigned int); + +void ck_barrier_dissemination_subscribe(ck_barrier_dissemination_t *, + ck_barrier_dissemination_state_t *); + +unsigned int ck_barrier_dissemination_size(unsigned int); + +void ck_barrier_dissemination(ck_barrier_dissemination_t *, + ck_barrier_dissemination_state_t *); + +struct ck_barrier_tournament_round { + int role; + unsigned int *opponent; + unsigned int flag; +}; +typedef struct ck_barrier_tournament_round ck_barrier_tournament_round_t; + +struct ck_barrier_tournament { + unsigned int tid; + unsigned int size; + struct ck_barrier_tournament_round **rounds; +}; +typedef struct ck_barrier_tournament ck_barrier_tournament_t; + +struct ck_barrier_tournament_state { + unsigned int sense; + unsigned int vpid; +}; +typedef struct ck_barrier_tournament_state ck_barrier_tournament_state_t; + +void ck_barrier_tournament_subscribe(ck_barrier_tournament_t *, + ck_barrier_tournament_state_t *); +void ck_barrier_tournament_init(ck_barrier_tournament_t *, + ck_barrier_tournament_round_t **, + unsigned int); +unsigned int ck_barrier_tournament_size(unsigned int); +void ck_barrier_tournament(ck_barrier_tournament_t *, ck_barrier_tournament_state_t *); + +struct ck_barrier_mcs { + unsigned int tid; + unsigned int *children[2]; + unsigned int childnotready[4]; + unsigned int dummy; + unsigned int havechild[4]; + unsigned int *parent; + unsigned int parentsense; +}; +typedef struct ck_barrier_mcs ck_barrier_mcs_t; + +struct ck_barrier_mcs_state { + unsigned int sense; + unsigned int vpid; +}; +typedef struct ck_barrier_mcs_state ck_barrier_mcs_state_t; + +void ck_barrier_mcs_init(ck_barrier_mcs_t *, unsigned int); +void ck_barrier_mcs_subscribe(ck_barrier_mcs_t *, ck_barrier_mcs_state_t *); +void ck_barrier_mcs(ck_barrier_mcs_t *, ck_barrier_mcs_state_t *); + +#endif /* CK_BARRIER_H */ Property changes on: vendor-sys/ck/20161230/include/ck_barrier.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_bitmap.h =================================================================== --- vendor-sys/ck/20161230/include/ck_bitmap.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_bitmap.h (revision 310844) @@ -0,0 +1,515 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * Copyright 2012-2014 AppNexus, Inc. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BITMAP_H +#define CK_BITMAP_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(CK_F_PR_LOAD_UINT) || !defined(CK_F_PR_STORE_UINT) || \ + !defined(CK_F_PR_AND_UINT) || !defined(CK_F_PR_OR_UINT) || \ + !defined(CK_F_CC_CTZ) +#error "ck_bitmap is not supported on your platform." +#endif + +#define CK_BITMAP_BLOCK (sizeof(unsigned int) * CHAR_BIT) +#define CK_BITMAP_OFFSET(i) ((i) % CK_BITMAP_BLOCK) +#define CK_BITMAP_BIT(i) (1U << CK_BITMAP_OFFSET(i)) +#define CK_BITMAP_PTR(x, i) ((x) + ((i) / CK_BITMAP_BLOCK)) +#define CK_BITMAP_BLOCKS(n) (((n) + CK_BITMAP_BLOCK - 1) / CK_BITMAP_BLOCK) + +#define CK_BITMAP_INSTANCE(n_entries) \ + union { \ + struct { \ + unsigned int n_bits; \ + unsigned int map[CK_BITMAP_BLOCKS(n_entries)]; \ + } content; \ + struct ck_bitmap bitmap; \ + } + +#define CK_BITMAP_ITERATOR_INIT(a, b) \ + ck_bitmap_iterator_init((a), &(b)->bitmap) + +#define CK_BITMAP_INIT(a, b, c) \ + ck_bitmap_init(&(a)->bitmap, (b), (c)) + +#define CK_BITMAP_NEXT(a, b, c) \ + ck_bitmap_next(&(a)->bitmap, (b), (c)) + +#define CK_BITMAP_SET(a, b) \ + ck_bitmap_set(&(a)->bitmap, (b)) + +#define CK_BITMAP_BTS(a, b) \ + ck_bitmap_bts(&(a)->bitmap, (b)) + +#define CK_BITMAP_RESET(a, b) \ + ck_bitmap_reset(&(a)->bitmap, (b)) + +#define CK_BITMAP_TEST(a, b) \ + ck_bitmap_test(&(a)->bitmap, (b)) + +#define CK_BITMAP_UNION(a, b) \ + ck_bitmap_union(&(a)->bitmap, &(b)->bitmap) + +#define CK_BITMAP_INTERSECTION(a, b) \ + ck_bitmap_intersection(&(a)->bitmap, &(b)->bitmap) + +#define CK_BITMAP_INTERSECTION_NEGATE(a, b) \ + ck_bitmap_intersection_negate(&(a)->bitmap, &(b)->bitmap) + +#define CK_BITMAP_CLEAR(a) \ + ck_bitmap_clear(&(a)->bitmap) + +#define CK_BITMAP_EMPTY(a, b) \ + ck_bitmap_empty(&(a)->bitmap, b) + +#define CK_BITMAP_FULL(a, b) \ + ck_bitmap_full(&(a)->bitmap, b) + +#define CK_BITMAP_COUNT(a, b) \ + ck_bitmap_count(&(a)->bitmap, b) + +#define CK_BITMAP_COUNT_INTERSECT(a, b, c) \ + ck_bitmap_count_intersect(&(a)->bitmap, b, c) + +#define CK_BITMAP_BITS(a) \ + ck_bitmap_bits(&(a)->bitmap) + +#define CK_BITMAP_BUFFER(a) \ + ck_bitmap_buffer(&(a)->bitmap) + +#define CK_BITMAP(a) \ + (&(a)->bitmap) + +struct ck_bitmap { + unsigned int n_bits; + unsigned int map[]; +}; +typedef struct ck_bitmap ck_bitmap_t; + +struct ck_bitmap_iterator { + unsigned int cache; + unsigned int n_block; + unsigned int n_limit; +}; +typedef struct ck_bitmap_iterator ck_bitmap_iterator_t; + +CK_CC_INLINE static unsigned int +ck_bitmap_base(unsigned int n_bits) +{ + + return CK_BITMAP_BLOCKS(n_bits) * sizeof(unsigned int); +} + +/* + * Returns the required number of bytes for a ck_bitmap_t object supporting the + * specified number of bits. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_size(unsigned int n_bits) +{ + + return ck_bitmap_base(n_bits) + sizeof(struct ck_bitmap); +} + +/* + * Returns total number of bits in specified bitmap. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_bits(const struct ck_bitmap *bitmap) +{ + + return bitmap->n_bits; +} + +/* + * Returns a pointer to the bit buffer associated + * with the specified bitmap. + */ +CK_CC_INLINE static void * +ck_bitmap_buffer(struct ck_bitmap *bitmap) +{ + + return bitmap->map; +} + +/* + * Sets the bit at the offset specified in the second argument. + */ +CK_CC_INLINE static void +ck_bitmap_set(struct ck_bitmap *bitmap, unsigned int n) +{ + + ck_pr_or_uint(CK_BITMAP_PTR(bitmap->map, n), CK_BITMAP_BIT(n)); + return; +} + +/* + * Performs a test-and-set operation at the offset specified in the + * second argument. + * Returns true if the bit at the specified offset was already set, + * false otherwise. + */ +CK_CC_INLINE static bool +ck_bitmap_bts(struct ck_bitmap *bitmap, unsigned int n) +{ + + return ck_pr_bts_uint(CK_BITMAP_PTR(bitmap->map, n), + CK_BITMAP_OFFSET(n)); +} + +/* + * Resets the bit at the offset specified in the second argument. + */ +CK_CC_INLINE static void +ck_bitmap_reset(struct ck_bitmap *bitmap, unsigned int n) +{ + + ck_pr_and_uint(CK_BITMAP_PTR(bitmap->map, n), ~CK_BITMAP_BIT(n)); + return; +} + +/* + * Determines whether the bit at offset specified in the + * second argument is set. + */ +CK_CC_INLINE static bool +ck_bitmap_test(const struct ck_bitmap *bitmap, unsigned int n) +{ + unsigned int block; + + block = ck_pr_load_uint(CK_BITMAP_PTR(bitmap->map, n)); + return block & CK_BITMAP_BIT(n); +} + +/* + * Combines bits from second bitmap into the first bitmap. This is not a + * linearized operation with respect to the complete bitmap. + */ +CK_CC_INLINE static void +ck_bitmap_union(struct ck_bitmap *dst, const struct ck_bitmap *src) +{ + unsigned int n; + unsigned int n_buckets = dst->n_bits; + + if (src->n_bits < dst->n_bits) + n_buckets = src->n_bits; + + n_buckets = CK_BITMAP_BLOCKS(n_buckets); + for (n = 0; n < n_buckets; n++) { + ck_pr_or_uint(&dst->map[n], + ck_pr_load_uint(&src->map[n])); + } + + return; +} + +/* + * Intersects bits from second bitmap into the first bitmap. This is + * not a linearized operation with respect to the complete bitmap. + * Any trailing bit in dst is cleared. + */ +CK_CC_INLINE static void +ck_bitmap_intersection(struct ck_bitmap *dst, const struct ck_bitmap *src) +{ + unsigned int n; + unsigned int n_buckets = dst->n_bits; + unsigned int n_intersect = n_buckets; + + if (src->n_bits < n_intersect) + n_intersect = src->n_bits; + + n_buckets = CK_BITMAP_BLOCKS(n_buckets); + n_intersect = CK_BITMAP_BLOCKS(n_intersect); + for (n = 0; n < n_intersect; n++) { + ck_pr_and_uint(&dst->map[n], + ck_pr_load_uint(&src->map[n])); + } + + for (; n < n_buckets; n++) + ck_pr_store_uint(&dst->map[n], 0); + + return; +} + +/* + * Intersects the complement of bits from second bitmap into the first + * bitmap. This is not a linearized operation with respect to the + * complete bitmap. Any trailing bit in dst is left as is. + */ +CK_CC_INLINE static void +ck_bitmap_intersection_negate(struct ck_bitmap *dst, + const struct ck_bitmap *src) +{ + unsigned int n; + unsigned int n_intersect = dst->n_bits; + + if (src->n_bits < n_intersect) + n_intersect = src->n_bits; + + n_intersect = CK_BITMAP_BLOCKS(n_intersect); + for (n = 0; n < n_intersect; n++) { + ck_pr_and_uint(&dst->map[n], + (~ck_pr_load_uint(&src->map[n]))); + } + + return; +} + +/* + * Resets all bits in the provided bitmap. This is not a linearized + * operation in ck_bitmap. + */ +CK_CC_INLINE static void +ck_bitmap_clear(struct ck_bitmap *bitmap) +{ + unsigned int i; + unsigned int n_buckets = ck_bitmap_base(bitmap->n_bits) / + sizeof(unsigned int); + + for (i = 0; i < n_buckets; i++) + ck_pr_store_uint(&bitmap->map[i], 0); + + return; +} + +/* + * Returns true if the first limit bits in bitmap are cleared. If + * limit is greater than the bitmap size, limit is truncated to that + * size. + */ +CK_CC_INLINE static bool +ck_bitmap_empty(const ck_bitmap_t *bitmap, unsigned int limit) +{ + unsigned int i, words, slop; + + if (limit > bitmap->n_bits) + limit = bitmap->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0; i < words; i++) { + if (ck_pr_load_uint(&bitmap->map[i]) != 0) { + return false; + } + } + + if (slop > 0) { + unsigned int word; + + word = ck_pr_load_uint(&bitmap->map[i]); + if ((word & ((1U << slop) - 1)) != 0) + return false; + } + + return true; +} + +/* + * Returns true if the first limit bits in bitmap are set. If limit + * is greater than the bitmap size, limit is truncated to that size. + */ +CK_CC_UNUSED static bool +ck_bitmap_full(const ck_bitmap_t *bitmap, unsigned int limit) +{ + unsigned int i, slop, words; + + if (limit > bitmap->n_bits) { + limit = bitmap->n_bits; + } + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0; i < words; i++) { + if (ck_pr_load_uint(&bitmap->map[i]) != -1U) + return false; + } + + if (slop > 0) { + unsigned int word; + + word = ~ck_pr_load_uint(&bitmap->map[i]); + if ((word & ((1U << slop) - 1)) != 0) + return false; + } + return true; +} + +/* + * Returns the number of set bit in bitmap, upto (and excluding) + * limit. If limit is greater than the bitmap size, it is truncated + * to that size. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_count(const ck_bitmap_t *bitmap, unsigned int limit) +{ + unsigned int count, i, slop, words; + + if (limit > bitmap->n_bits) + limit = bitmap->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0, count = 0; i < words; i++) + count += ck_cc_popcount(ck_pr_load_uint(&bitmap->map[i])); + + if (slop > 0) { + unsigned int word; + + word = ck_pr_load_uint(&bitmap->map[i]); + count += ck_cc_popcount(word & ((1U << slop) - 1)); + } + return count; +} + +/* + * Returns the number of set bit in the intersection of two bitmaps, + * upto (and excluding) limit. If limit is greater than either bitmap + * size, it is truncated to the smallest. + */ +CK_CC_INLINE static unsigned int +ck_bitmap_count_intersect(const ck_bitmap_t *x, const ck_bitmap_t *y, + unsigned int limit) +{ + unsigned int count, i, slop, words; + + if (limit > x->n_bits) + limit = x->n_bits; + + if (limit > y->n_bits) + limit = y->n_bits; + + words = limit / CK_BITMAP_BLOCK; + slop = limit % CK_BITMAP_BLOCK; + for (i = 0, count = 0; i < words; i++) { + unsigned int xi, yi; + + xi = ck_pr_load_uint(&x->map[i]); + yi = ck_pr_load_uint(&y->map[i]); + count += ck_cc_popcount(xi & yi); + } + + if (slop > 0) { + unsigned int word, xi, yi; + + xi = ck_pr_load_uint(&x->map[i]); + yi = ck_pr_load_uint(&y->map[i]); + word = xi & yi; + count += ck_cc_popcount(word & ((1U << slop) - 1)); + } + return count; +} + +/* + * Initializes a ck_bitmap pointing to a region of memory with + * ck_bitmap_size(n_bits) bytes. Third argument determines whether + * default bit value is 1 (true) or 0 (false). + */ +CK_CC_INLINE static void +ck_bitmap_init(struct ck_bitmap *bitmap, + unsigned int n_bits, + bool set) +{ + unsigned int base = ck_bitmap_base(n_bits); + + bitmap->n_bits = n_bits; + memset(bitmap->map, -(int)set, base); + + if (set == true) { + unsigned int b = n_bits % CK_BITMAP_BLOCK; + + if (b == 0) + return; + + *CK_BITMAP_PTR(bitmap->map, n_bits - 1) &= (1U << b) - 1U; + } + + return; +} + +/* + * Initialize iterator for use with provided bitmap. + */ +CK_CC_INLINE static void +ck_bitmap_iterator_init(struct ck_bitmap_iterator *i, + const struct ck_bitmap *bitmap) +{ + + i->n_block = 0; + i->n_limit = CK_BITMAP_BLOCKS(bitmap->n_bits); + if (i->n_limit > 0) { + i->cache = ck_pr_load_uint(&bitmap->map[0]); + } else { + i->cache = 0; + } + return; +} + +/* + * Iterate to next bit. + */ +CK_CC_INLINE static bool +ck_bitmap_next(const struct ck_bitmap *bitmap, + struct ck_bitmap_iterator *i, + unsigned int *bit) +{ + unsigned int cache = i->cache; + unsigned int n_block = i->n_block; + unsigned int n_limit = i->n_limit; + + if (cache == 0) { + if (n_block >= n_limit) + return false; + + for (n_block++; n_block < n_limit; n_block++) { + cache = ck_pr_load_uint(&bitmap->map[n_block]); + if (cache != 0) + goto non_zero; + } + + i->cache = 0; + i->n_block = n_block; + return false; + } + +non_zero: + *bit = CK_BITMAP_BLOCK * n_block + ck_cc_ctz(cache); + i->cache = cache & (cache - 1); + i->n_block = n_block; + return true; +} + +#endif /* CK_BITMAP_H */ Property changes on: vendor-sys/ck/20161230/include/ck_bitmap.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_brlock.h =================================================================== --- vendor-sys/ck/20161230/include/ck_brlock.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_brlock.h (revision 310844) @@ -0,0 +1,279 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BRLOCK_H +#define CK_BRLOCK_H + +/* + * Big reader spinlocks provide cache-local contention-free read + * lock acquisition in the absence of writers. This comes at the + * cost of O(n) write lock acquisition. They were first implemented + * in the Linux kernel by Ingo Molnar and David S. Miller around the + * year 2000. + * + * This implementation is thread-agnostic which comes at the cost + * of larger reader objects due to necessary linkage overhead. In + * order to cut down on TLB pressure, it is recommended to allocate + * these objects on the same page. + */ + +#include +#include +#include + +struct ck_brlock_reader { + unsigned int n_readers; + struct ck_brlock_reader *previous; + struct ck_brlock_reader *next; +}; +typedef struct ck_brlock_reader ck_brlock_reader_t; + +#define CK_BRLOCK_READER_INITIALIZER {0} + +struct ck_brlock { + struct ck_brlock_reader *readers; + unsigned int writer; +}; +typedef struct ck_brlock ck_brlock_t; + +#define CK_BRLOCK_INITIALIZER {NULL, false} + +CK_CC_INLINE static void +ck_brlock_init(struct ck_brlock *br) +{ + + br->readers = NULL; + br->writer = false; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_brlock_write_lock(struct ck_brlock *br) +{ + struct ck_brlock_reader *cursor; + + /* + * As the frequency of write acquisitions should be low, + * there is no point to more advanced contention avoidance. + */ + while (ck_pr_fas_uint(&br->writer, true) == true) + ck_pr_stall(); + + ck_pr_fence_atomic_load(); + + /* The reader list is protected under the writer br. */ + for (cursor = br->readers; cursor != NULL; cursor = cursor->next) { + while (ck_pr_load_uint(&cursor->n_readers) != 0) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_brlock_write_unlock(struct ck_brlock *br) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&br->writer, false); + return; +} + +CK_CC_INLINE static bool +ck_brlock_write_trylock(struct ck_brlock *br, unsigned int factor) +{ + struct ck_brlock_reader *cursor; + unsigned int steps = 0; + + while (ck_pr_fas_uint(&br->writer, true) == true) { + if (++steps >= factor) + return false; + + ck_pr_stall(); + } + + /* + * We do not require a strict fence here as atomic RMW operations + * are serializing. + */ + ck_pr_fence_atomic_load(); + + for (cursor = br->readers; cursor != NULL; cursor = cursor->next) { + while (ck_pr_load_uint(&cursor->n_readers) != 0) { + if (++steps >= factor) { + ck_brlock_write_unlock(br); + return false; + } + + ck_pr_stall(); + } + } + + ck_pr_fence_lock(); + return true; +} + +CK_CC_INLINE static void +ck_brlock_read_register(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + reader->n_readers = 0; + reader->previous = NULL; + + /* Implicit compiler barrier. */ + ck_brlock_write_lock(br); + + reader->next = ck_pr_load_ptr(&br->readers); + if (reader->next != NULL) + reader->next->previous = reader; + ck_pr_store_ptr(&br->readers, reader); + + ck_brlock_write_unlock(br); + return; +} + +CK_CC_INLINE static void +ck_brlock_read_unregister(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + ck_brlock_write_lock(br); + + if (reader->next != NULL) + reader->next->previous = reader->previous; + + if (reader->previous != NULL) + reader->previous->next = reader->next; + else + br->readers = reader->next; + + ck_brlock_write_unlock(br); + return; +} + +CK_CC_INLINE static void +ck_brlock_read_lock(struct ck_brlock *br, struct ck_brlock_reader *reader) +{ + + if (reader->n_readers >= 1) { + ck_pr_store_uint(&reader->n_readers, reader->n_readers + 1); + return; + } + + for (;;) { + while (ck_pr_load_uint(&br->writer) == true) + ck_pr_stall(); + +#if defined(__x86__) || defined(__x86_64__) + ck_pr_fas_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_atomic_load(); +#else + ck_pr_store_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_store_load(); +#endif + + if (ck_pr_load_uint(&br->writer) == false) + break; + + ck_pr_store_uint(&reader->n_readers, 0); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static bool +ck_brlock_read_trylock(struct ck_brlock *br, + struct ck_brlock_reader *reader, + unsigned int factor) +{ + unsigned int steps = 0; + + if (reader->n_readers >= 1) { + ck_pr_store_uint(&reader->n_readers, reader->n_readers + 1); + return true; + } + + for (;;) { + while (ck_pr_load_uint(&br->writer) == true) { + if (++steps >= factor) + return false; + + ck_pr_stall(); + } + +#if defined(__x86__) || defined(__x86_64__) + ck_pr_fas_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_atomic_load(); +#else + ck_pr_store_uint(&reader->n_readers, 1); + + /* + * Serialize reader counter update with respect to load of + * writer. + */ + ck_pr_fence_store_load(); +#endif + + if (ck_pr_load_uint(&br->writer) == false) + break; + + ck_pr_store_uint(&reader->n_readers, 0); + + if (++steps >= factor) + return false; + } + + ck_pr_fence_lock(); + return true; +} + +CK_CC_INLINE static void +ck_brlock_read_unlock(struct ck_brlock_reader *reader) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&reader->n_readers, reader->n_readers - 1); + return; +} + +#endif /* CK_BRLOCK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_brlock.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_bytelock.h =================================================================== --- vendor-sys/ck/20161230/include/ck_bytelock.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_bytelock.h (revision 310844) @@ -0,0 +1,196 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_BYTELOCK_H +#define CK_BYTELOCK_H + +/* + * The implementations here are derived from the work described in: + * Dice, D. and Shavit, N. 2010. TLRW: return of the read-write lock. + * In Proceedings of the 22nd ACM Symposium on Parallelism in Algorithms + * and Architectures (Thira, Santorini, Greece, June 13 - 15, 2010). + * SPAA '10. ACM, New York, NY, 284-293. + */ + +#include +#include +#include +#include +#include +#include + +struct ck_bytelock { + unsigned int owner; + unsigned int n_readers; + uint8_t readers[CK_MD_CACHELINE - sizeof(unsigned int) * 2] CK_CC_ALIGN(8); +}; +typedef struct ck_bytelock ck_bytelock_t; + +#define CK_BYTELOCK_INITIALIZER { 0, 0, {0} } +#define CK_BYTELOCK_UNSLOTTED UINT_MAX + +CK_CC_INLINE static void +ck_bytelock_init(struct ck_bytelock *bytelock) +{ + unsigned int i; + + bytelock->owner = 0; + bytelock->n_readers = 0; + for (i = 0; i < sizeof bytelock->readers; i++) + bytelock->readers[i] = false; + + ck_pr_barrier(); + return; +} + +#ifdef CK_F_PR_LOAD_64 +#define CK_BYTELOCK_LENGTH sizeof(uint64_t) +#define CK_BYTELOCK_LOAD ck_pr_load_64 +#define CK_BYTELOCK_TYPE uint64_t +#elif defined(CK_F_PR_LOAD_32) +#define CK_BYTELOCK_LENGTH sizeof(uint32_t) +#define CK_BYTELOCK_LOAD ck_pr_load_32 +#define CK_BYTELOCK_TYPE uint32_t +#else +#error Unsupported platform. +#endif + +CK_CC_INLINE static void +ck_bytelock_write_lock(struct ck_bytelock *bytelock, unsigned int slot) +{ + CK_BYTELOCK_TYPE *readers = (void *)bytelock->readers; + unsigned int i; + + /* Announce upcoming writer acquisition. */ + while (ck_pr_cas_uint(&bytelock->owner, 0, slot) == false) + ck_pr_stall(); + + /* If we are slotted, we might be upgrading from a read lock. */ + if (slot <= sizeof bytelock->readers) + ck_pr_store_8(&bytelock->readers[slot - 1], false); + + /* + * Wait for slotted readers to drain out. This also provides the + * lock acquire semantics. + */ + ck_pr_fence_atomic_load(); + + for (i = 0; i < sizeof(bytelock->readers) / CK_BYTELOCK_LENGTH; i++) { + while (CK_BYTELOCK_LOAD(&readers[i]) != false) + ck_pr_stall(); + } + + /* Wait for unslotted readers to drain out. */ + while (ck_pr_load_uint(&bytelock->n_readers) != 0) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +#undef CK_BYTELOCK_LENGTH +#undef CK_BYTELOCK_LOAD +#undef CK_BYTELOCK_TYPE + +CK_CC_INLINE static void +ck_bytelock_write_unlock(struct ck_bytelock *bytelock) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&bytelock->owner, 0); + return; +} + +CK_CC_INLINE static void +ck_bytelock_read_lock(struct ck_bytelock *bytelock, unsigned int slot) +{ + + if (ck_pr_load_uint(&bytelock->owner) == slot) { + ck_pr_store_8(&bytelock->readers[slot - 1], true); + ck_pr_fence_strict_store(); + ck_pr_store_uint(&bytelock->owner, 0); + return; + } + + /* Unslotted threads will have to use the readers counter. */ + if (slot > sizeof bytelock->readers) { + for (;;) { + ck_pr_inc_uint(&bytelock->n_readers); + ck_pr_fence_atomic_load(); + if (ck_pr_load_uint(&bytelock->owner) == 0) + break; + ck_pr_dec_uint(&bytelock->n_readers); + + while (ck_pr_load_uint(&bytelock->owner) != 0) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; + } + + slot -= 1; + for (;;) { +#ifdef CK_F_PR_FAA_8 + ck_pr_fas_8(&bytelock->readers[slot], true); + ck_pr_fence_atomic_load(); +#else + ck_pr_store_8(&bytelock->readers[slot], true); + ck_pr_fence_store_load(); +#endif + + /* + * If there is no owner at this point, our slot has + * already been published and it is guaranteed no + * write acquisition will succeed until we drain out. + */ + if (ck_pr_load_uint(&bytelock->owner) == 0) + break; + + ck_pr_store_8(&bytelock->readers[slot], false); + while (ck_pr_load_uint(&bytelock->owner) != 0) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_bytelock_read_unlock(struct ck_bytelock *bytelock, unsigned int slot) +{ + + ck_pr_fence_unlock(); + + if (slot > sizeof bytelock->readers) + ck_pr_dec_uint(&bytelock->n_readers); + else + ck_pr_store_8(&bytelock->readers[slot - 1], false); + + return; +} + +#endif /* CK_BYTELOCK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_bytelock.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_cc.h =================================================================== --- vendor-sys/ck/20161230/include/ck_cc.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_cc.h (revision 310844) @@ -0,0 +1,180 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * Copyright 2014 Paul Khuong. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_CC_H +#define CK_CC_H + +#if defined(__GNUC__) || defined(__SUNPRO_C) +#include "gcc/ck_cc.h" +#endif + +#ifndef CK_CC_RESTRICT +#define CK_CC_RESTRICT +#endif + +#ifndef CK_CC_INLINE +#define CK_CC_INLINE inline +#endif + +#ifndef CK_CC_FORCE_INLINE +#define CK_CC_FORCE_INLINE inline +#endif + +#define CK_CC_DECONST_PTR(X) ((void *)(uintptr_t)(X)) + +/* + * Container function. + * This relies on (compiler) implementation-defined behavior. + */ +#define CK_CC_CONTAINER(F, T, M, N) \ + CK_CC_INLINE static T * \ + N(F *p) \ + { \ + F *n = p; \ + return (T *)(void *)(((char *)n) - ((size_t)&((T *)0)->M)); \ + } + +#define CK_CC_PAD(x) union { char pad[x]; } + +#ifndef CK_CC_ALIASED +#define CK_CC_ALIASED +#endif + +#ifndef CK_CC_UNUSED +#define CK_CC_UNUSED +#endif + +#ifndef CK_CC_USED +#define CK_CC_USED +#endif + +#ifndef CK_CC_IMM +#define CK_CC_IMM +#endif + +#ifndef CK_CC_PACKED +#define CK_CC_PACKED +#endif + +#ifndef CK_CC_WEAKREF +#define CK_CC_WEAKREF +#endif + +#ifndef CK_CC_ALIGN +#define CK_CC_ALIGN(X) +#endif + +#ifndef CK_CC_CACHELINE +#define CK_CC_CACHELINE +#endif + +#ifndef CK_CC_LIKELY +#define CK_CC_LIKELY(x) x +#endif + +#ifndef CK_CC_UNLIKELY +#define CK_CC_UNLIKELY(x) x +#endif + +#ifndef CK_CC_TYPEOF +#define CK_CC_TYPEOF(X, DEFAULT) (DEFAULT) +#endif + +#ifndef CK_F_CC_FFS +#define CK_F_CC_FFS +CK_CC_INLINE static int +ck_cc_ffs(unsigned int x) +{ + unsigned int i; + + if (x == 0) + return 0; + + for (i = 1; (x & 1) == 0; i++, x >>= 1); + + return i; +} +#endif + +#ifndef CK_F_CC_CLZ +#define CK_F_CC_CLZ +#include + +CK_CC_INLINE static int +ck_cc_clz(unsigned int x) +{ + unsigned int count, i; + + for (count = 0, i = sizeof(unsigned int) * CHAR_BIT; i > 0; count++) { + unsigned int bit = 1U << --i; + + if (x & bit) + break; + } + + return count; +} +#endif + +#ifndef CK_F_CC_CTZ +#define CK_F_CC_CTZ +CK_CC_INLINE static int +ck_cc_ctz(unsigned int x) +{ + unsigned int i; + + if (x == 0) + return 0; + + for (i = 0; (x & 1) == 0; i++, x >>= 1); + + return i; +} +#endif + +#ifndef CK_F_CC_POPCOUNT +#define CK_F_CC_POPCOUNT +CK_CC_INLINE static int +ck_cc_popcount(unsigned int x) +{ + unsigned int acc; + + for (acc = 0; x != 0; x >>= 1) + acc += x & 1; + + return acc; +} +#endif + + +#ifdef __cplusplus +#define CK_CPP_CAST(type, arg) static_cast(arg) +#else +#define CK_CPP_CAST(type, arg) arg +#endif + +#endif /* CK_CC_H */ Property changes on: vendor-sys/ck/20161230/include/ck_cc.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_cohort.h =================================================================== --- vendor-sys/ck/20161230/include/ck_cohort.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_cohort.h (revision 310844) @@ -0,0 +1,161 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * Copyright 2013 Brendon Scheinman. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_COHORT_H +#define CK_COHORT_H + +/* + * This is an implementation of lock cohorts as described in: + * Dice, D.; Marathe, V.; and Shavit, N. 2012. + * Lock Cohorting: A General Technique for Designing NUMA Locks + */ + +#include +#include +#include + +enum ck_cohort_state { + CK_COHORT_STATE_GLOBAL = 0, + CK_COHORT_STATE_LOCAL = 1 +}; + +#define CK_COHORT_DEFAULT_LOCAL_PASS_LIMIT 10 + +#define CK_COHORT_NAME(N) ck_cohort_##N +#define CK_COHORT_INSTANCE(N) struct CK_COHORT_NAME(N) +#define CK_COHORT_INIT(N, C, GL, LL, P) ck_cohort_##N##_init(C, GL, LL, P) +#define CK_COHORT_LOCK(N, C, GC, LC) ck_cohort_##N##_lock(C, GC, LC) +#define CK_COHORT_UNLOCK(N, C, GC, LC) ck_cohort_##N##_unlock(C, GC, LC) +#define CK_COHORT_TRYLOCK(N, C, GLC, LLC, LUC) ck_cohort_##N##_trylock(C, GLC, LLC, LUC) +#define CK_COHORT_LOCKED(N, C, GC, LC) ck_cohort_##N##_locked(C, GC, LC) + +#define CK_COHORT_PROTOTYPE(N, GL, GU, GI, LL, LU, LI) \ + CK_COHORT_INSTANCE(N) { \ + void *global_lock; \ + void *local_lock; \ + enum ck_cohort_state release_state; \ + unsigned int waiting_threads; \ + unsigned int acquire_count; \ + unsigned int local_pass_limit; \ + }; \ + \ + CK_CC_INLINE static void \ + ck_cohort_##N##_init(struct ck_cohort_##N *cohort, \ + void *global_lock, void *local_lock, unsigned int pass_limit) \ + { \ + cohort->global_lock = global_lock; \ + cohort->local_lock = local_lock; \ + cohort->release_state = CK_COHORT_STATE_GLOBAL; \ + cohort->waiting_threads = 0; \ + cohort->acquire_count = 0; \ + cohort->local_pass_limit = pass_limit; \ + ck_pr_barrier(); \ + return; \ + } \ + \ + CK_CC_INLINE static void \ + ck_cohort_##N##_lock(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context) \ + { \ + \ + ck_pr_inc_uint(&cohort->waiting_threads); \ + LL(cohort->local_lock, local_context); \ + ck_pr_dec_uint(&cohort->waiting_threads); \ + \ + if (cohort->release_state == CK_COHORT_STATE_GLOBAL) { \ + GL(cohort->global_lock, global_context); \ + } \ + \ + ++cohort->acquire_count; \ + return; \ + } \ + \ + CK_CC_INLINE static void \ + ck_cohort_##N##_unlock(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context) \ + { \ + \ + if (ck_pr_load_uint(&cohort->waiting_threads) > 0 \ + && cohort->acquire_count < cohort->local_pass_limit) { \ + cohort->release_state = CK_COHORT_STATE_LOCAL; \ + } else { \ + GU(cohort->global_lock, global_context); \ + cohort->release_state = CK_COHORT_STATE_GLOBAL; \ + cohort->acquire_count = 0; \ + } \ + \ + ck_pr_fence_release(); \ + LU(cohort->local_lock, local_context); \ + \ + return; \ + } \ + \ + CK_CC_INLINE static bool \ + ck_cohort_##N##_locked(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context) \ + { \ + return GI(cohort->local_lock, local_context) || \ + LI(cohort->global_lock, global_context); \ + } + +#define CK_COHORT_TRYLOCK_PROTOTYPE(N, GL, GU, GI, GTL, LL, LU, LI, LTL) \ + CK_COHORT_PROTOTYPE(N, GL, GU, GI, LL, LU, LI) \ + CK_CC_INLINE static bool \ + ck_cohort_##N##_trylock(CK_COHORT_INSTANCE(N) *cohort, \ + void *global_context, void *local_context, \ + void *local_unlock_context) \ + { \ + \ + bool trylock_result; \ + \ + ck_pr_inc_uint(&cohort->waiting_threads); \ + trylock_result = LTL(cohort->local_lock, local_context); \ + ck_pr_dec_uint(&cohort->waiting_threads); \ + if (trylock_result == false) { \ + return false; \ + } \ + \ + if (cohort->release_state == CK_COHORT_STATE_GLOBAL && \ + GTL(cohort->global_lock, global_context) == false) { \ + LU(cohort->local_lock, local_unlock_context); \ + return false; \ + } \ + \ + ++cohort->acquire_count; \ + return true; \ + } + +#define CK_COHORT_INITIALIZER { \ + .global_lock = NULL, \ + .local_lock = NULL, \ + .release_state = CK_COHORT_STATE_GLOBAL, \ + .waiting_threads = 0, \ + .acquire_count = 0, \ + .local_pass_limit = CK_COHORT_DEFAULT_LOCAL_PASS_LIMIT \ +} + +#endif /* CK_COHORT_H */ Property changes on: vendor-sys/ck/20161230/include/ck_cohort.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_elide.h =================================================================== --- vendor-sys/ck/20161230/include/ck_elide.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_elide.h (revision 310844) @@ -0,0 +1,321 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_ELIDE_H +#define CK_ELIDE_H + +/* + * As RTM is currently only supported on TSO x86 architectures, + * fences have been omitted. They will be necessary for other + * non-TSO architectures with TM support. + */ + +#include +#include +#include + +/* + * skip_-prefixed counters represent the number of consecutive + * elisions to forfeit. retry_-prefixed counters represent the + * number of elision retries to attempt before forfeit. + * + * _busy: Lock was busy + * _other: Unknown explicit abort + * _conflict: Data conflict in elision section + */ +struct ck_elide_config { + unsigned short skip_busy; + short retry_busy; + unsigned short skip_other; + short retry_other; + unsigned short skip_conflict; + short retry_conflict; +}; + +#define CK_ELIDE_CONFIG_DEFAULT_INITIALIZER { \ + .skip_busy = 5, \ + .retry_busy = 256, \ + .skip_other = 3, \ + .retry_other = 3, \ + .skip_conflict = 2, \ + .retry_conflict = 5 \ +} + +struct ck_elide_stat { + unsigned int n_fallback; + unsigned int n_elide; + unsigned short skip; +}; +typedef struct ck_elide_stat ck_elide_stat_t; + +#define CK_ELIDE_STAT_INITIALIZER { 0, 0, 0 } + +CK_CC_INLINE static void +ck_elide_stat_init(ck_elide_stat_t *st) +{ + + memset(st, 0, sizeof(*st)); + return; +} + +#ifdef CK_F_PR_RTM +enum _ck_elide_hint { + CK_ELIDE_HINT_RETRY = 0, + CK_ELIDE_HINT_SPIN, + CK_ELIDE_HINT_STOP +}; + +#define CK_ELIDE_LOCK_BUSY 0xFF + +static enum _ck_elide_hint +_ck_elide_fallback(int *retry, + struct ck_elide_stat *st, + struct ck_elide_config *c, + unsigned int status) +{ + + st->n_fallback++; + if (*retry > 0) + return CK_ELIDE_HINT_RETRY; + + if (st->skip != 0) + return CK_ELIDE_HINT_STOP; + + if (status & CK_PR_RTM_EXPLICIT) { + if (CK_PR_RTM_CODE(status) == CK_ELIDE_LOCK_BUSY) { + st->skip = c->skip_busy; + *retry = c->retry_busy; + return CK_ELIDE_HINT_SPIN; + } + + st->skip = c->skip_other; + return CK_ELIDE_HINT_STOP; + } + + if ((status & CK_PR_RTM_RETRY) && + (status & CK_PR_RTM_CONFLICT)) { + st->skip = c->skip_conflict; + *retry = c->retry_conflict; + return CK_ELIDE_HINT_RETRY; + } + + /* + * Capacity, debug and nesting abortions are likely to be + * invariant conditions for the acquisition, execute regular + * path instead. If retry bit is not set, then take the hint. + */ + st->skip = USHRT_MAX; + return CK_ELIDE_HINT_STOP; +} + +/* + * Defines an elision implementation according to the following variables: + * N - Namespace of elision implementation. + * T - Typename of mutex. + * L_P - Lock predicate, returns false if resource is available. + * L - Function to call if resource is unavailable of transaction aborts. + * U_P - Unlock predicate, returns false if elision failed. + * U - Function to call if transaction failed. + */ +#define CK_ELIDE_PROTOTYPE(N, T, L_P, L, U_P, U) \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock_adaptive(T *lock, \ + struct ck_elide_stat *st, \ + struct ck_elide_config *c) \ + { \ + enum _ck_elide_hint hint; \ + int retry; \ + \ + if (CK_CC_UNLIKELY(st->skip != 0)) { \ + st->skip--; \ + goto acquire; \ + } \ + \ + retry = c->retry_conflict; \ + do { \ + unsigned int status = ck_pr_rtm_begin(); \ + if (status == CK_PR_RTM_STARTED) { \ + if (L_P(lock) == true) \ + ck_pr_rtm_abort(CK_ELIDE_LOCK_BUSY); \ + \ + return; \ + } \ + \ + hint = _ck_elide_fallback(&retry, st, c, status); \ + if (hint == CK_ELIDE_HINT_RETRY) \ + continue; \ + \ + if (hint == CK_ELIDE_HINT_SPIN) { \ + while (--retry != 0) { \ + if (L_P(lock) == false) \ + break; \ + \ + ck_pr_stall(); \ + } \ + \ + continue; \ + } \ + \ + if (hint == CK_ELIDE_HINT_STOP) \ + break; \ + } while (CK_CC_LIKELY(--retry > 0)); \ + \ + acquire: \ + L(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock_adaptive(struct ck_elide_stat *st, T *lock) \ + { \ + \ + if (U_P(lock) == false) { \ + ck_pr_rtm_end(); \ + st->skip = 0; \ + st->n_elide++; \ + } else { \ + U(lock); \ + } \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock(T *lock) \ + { \ + \ + if (ck_pr_rtm_begin() != CK_PR_RTM_STARTED) { \ + L(lock); \ + return; \ + } \ + \ + if (L_P(lock) == true) \ + ck_pr_rtm_abort(CK_ELIDE_LOCK_BUSY); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock(T *lock) \ + { \ + \ + if (U_P(lock) == false) { \ + ck_pr_rtm_end(); \ + } else { \ + U(lock); \ + } \ + \ + return; \ + } + +#define CK_ELIDE_TRYLOCK_PROTOTYPE(N, T, TL_P, TL) \ + CK_CC_INLINE static bool \ + ck_elide_##N##_trylock(T *lock) \ + { \ + \ + if (ck_pr_rtm_begin() != CK_PR_RTM_STARTED) \ + return false; \ + \ + if (TL_P(lock) == true) \ + ck_pr_rtm_abort(CK_ELIDE_LOCK_BUSY); \ + \ + return true; \ + } +#else +/* + * If RTM is not enabled on the target platform (CK_F_PR_RTM) then these + * elision wrappers directly calls into the user-specified lock operations. + * Unfortunately, the storage cost of both ck_elide_config and ck_elide_stat + * are paid (typically a storage cost that is a function of lock objects and + * thread count). + */ +#define CK_ELIDE_PROTOTYPE(N, T, L_P, L, U_P, U) \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock_adaptive(T *lock, \ + struct ck_elide_stat *st, \ + struct ck_elide_config *c) \ + { \ + \ + (void)st; \ + (void)c; \ + L(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock_adaptive(struct ck_elide_stat *st, \ + T *lock) \ + { \ + \ + (void)st; \ + U(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_lock(T *lock) \ + { \ + \ + L(lock); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_elide_##N##_unlock(T *lock) \ + { \ + \ + U(lock); \ + return; \ + } + +#define CK_ELIDE_TRYLOCK_PROTOTYPE(N, T, TL_P, TL) \ + CK_CC_INLINE static bool \ + ck_elide_##N##_trylock(T *lock) \ + { \ + \ + return TL_P(lock); \ + } +#endif /* !CK_F_PR_RTM */ + +/* + * Best-effort elision lock operations. First argument is name (N) + * associated with implementation and the second is a pointer to + * the type specified above (T). + * + * Unlike the adaptive variant, this interface does not have any retry + * semantics. In environments where jitter is low, this may yield a tighter + * fast path. + */ +#define CK_ELIDE_LOCK(NAME, LOCK) ck_elide_##NAME##_lock(LOCK) +#define CK_ELIDE_UNLOCK(NAME, LOCK) ck_elide_##NAME##_unlock(LOCK) +#define CK_ELIDE_TRYLOCK(NAME, LOCK) ck_elide_##NAME##_trylock(LOCK) + +/* + * Adaptive elision lock operations. In addition to name and pointer + * to the lock, you must pass in a pointer to an initialized + * ck_elide_config structure along with a per-thread stat structure. + */ +#define CK_ELIDE_LOCK_ADAPTIVE(NAME, STAT, CONFIG, LOCK) \ + ck_elide_##NAME##_lock_adaptive(LOCK, STAT, CONFIG) + +#define CK_ELIDE_UNLOCK_ADAPTIVE(NAME, STAT, LOCK) \ + ck_elide_##NAME##_unlock_adaptive(STAT, LOCK) + +#endif /* CK_ELIDE_H */ Property changes on: vendor-sys/ck/20161230/include/ck_elide.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_epoch.h =================================================================== --- vendor-sys/ck/20161230/include/ck_epoch.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_epoch.h (revision 310844) @@ -0,0 +1,207 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_EPOCH_H +#define CK_EPOCH_H + +/* + * The implementation here is inspired from the work described in: + * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University + * of Cambridge Computing Laboratory. + */ + +#include +#include +#include +#include +#include + +#ifndef CK_EPOCH_LENGTH +#define CK_EPOCH_LENGTH 4 +#endif + +/* + * This is used for sense detection with-respect to concurrent + * epoch sections. + */ +#define CK_EPOCH_SENSE (2) + +struct ck_epoch_entry; +typedef struct ck_epoch_entry ck_epoch_entry_t; +typedef void ck_epoch_cb_t(ck_epoch_entry_t *); + +/* + * This should be embedded into objects you wish to be the target of + * ck_epoch_cb_t functions (with ck_epoch_call). + */ +struct ck_epoch_entry { + ck_epoch_cb_t *function; + ck_stack_entry_t stack_entry; +}; + +/* + * A section object may be passed to every begin-end pair to allow for + * forward progress guarantees with-in prolonged active sections. + */ +struct ck_epoch_section { + unsigned int bucket; +}; +typedef struct ck_epoch_section ck_epoch_section_t; + +/* + * Return pointer to ck_epoch_entry container object. + */ +#define CK_EPOCH_CONTAINER(T, M, N) \ + CK_CC_CONTAINER(struct ck_epoch_entry, T, M, N) + +struct ck_epoch_ref { + unsigned int epoch; + unsigned int count; +}; + +struct ck_epoch_record { + struct ck_epoch *global; + unsigned int state; + unsigned int epoch; + unsigned int active; + struct { + struct ck_epoch_ref bucket[CK_EPOCH_SENSE]; + } local CK_CC_CACHELINE; + unsigned int n_pending; + unsigned int n_peak; + unsigned long n_dispatch; + ck_stack_t pending[CK_EPOCH_LENGTH]; + ck_stack_entry_t record_next; +} CK_CC_CACHELINE; +typedef struct ck_epoch_record ck_epoch_record_t; + +struct ck_epoch { + unsigned int epoch; + char pad[CK_MD_CACHELINE - sizeof(unsigned int)]; + ck_stack_t records; + unsigned int n_free; +}; +typedef struct ck_epoch ck_epoch_t; + +/* + * Internal functions. + */ +void _ck_epoch_addref(ck_epoch_record_t *, ck_epoch_section_t *); +void _ck_epoch_delref(ck_epoch_record_t *, ck_epoch_section_t *); + +/* + * Marks the beginning of an epoch-protected section. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_begin(ck_epoch_record_t *record, ck_epoch_section_t *section) +{ + struct ck_epoch *epoch = record->global; + + /* + * Only observe new epoch if thread is not recursing into a read + * section. + */ + if (record->active == 0) { + unsigned int g_epoch; + + /* + * It is possible for loads to be re-ordered before the store + * is committed into the caller's epoch and active fields. + * For this reason, store to load serialization is necessary. + */ +#if defined(CK_MD_TSO) + ck_pr_fas_uint(&record->active, 1); + ck_pr_fence_atomic_load(); +#else + ck_pr_store_uint(&record->active, 1); + ck_pr_fence_memory(); +#endif + + /* + * This load is allowed to be re-ordered prior to setting + * active flag due to monotonic nature of the global epoch. + * However, stale values lead to measurable performance + * degradation in some torture tests so we disallow early load + * of global epoch. + */ + g_epoch = ck_pr_load_uint(&epoch->epoch); + ck_pr_store_uint(&record->epoch, g_epoch); + } else { + ck_pr_store_uint(&record->active, record->active + 1); + } + + if (section != NULL) + _ck_epoch_addref(record, section); + + return; +} + +/* + * Marks the end of an epoch-protected section. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_end(ck_epoch_record_t *record, ck_epoch_section_t *section) +{ + + ck_pr_fence_release(); + ck_pr_store_uint(&record->active, record->active - 1); + + if (section != NULL) + _ck_epoch_delref(record, section); + + return; +} + +/* + * Defers the execution of the function pointed to by the "cb" + * argument until an epoch counter loop. This allows for a + * non-blocking deferral. + */ +CK_CC_FORCE_INLINE static void +ck_epoch_call(ck_epoch_record_t *record, + ck_epoch_entry_t *entry, + ck_epoch_cb_t *function) +{ + struct ck_epoch *epoch = record->global; + unsigned int e = ck_pr_load_uint(&epoch->epoch); + unsigned int offset = e & (CK_EPOCH_LENGTH - 1); + + record->n_pending++; + entry->function = function; + ck_stack_push_spnc(&record->pending[offset], &entry->stack_entry); + return; +} + +void ck_epoch_init(ck_epoch_t *); +ck_epoch_record_t *ck_epoch_recycle(ck_epoch_t *); +void ck_epoch_register(ck_epoch_t *, ck_epoch_record_t *); +void ck_epoch_unregister(ck_epoch_record_t *); +bool ck_epoch_poll(ck_epoch_record_t *); +void ck_epoch_synchronize(ck_epoch_record_t *); +void ck_epoch_barrier(ck_epoch_record_t *); +void ck_epoch_reclaim(ck_epoch_record_t *); + +#endif /* CK_EPOCH_H */ Property changes on: vendor-sys/ck/20161230/include/ck_epoch.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_fifo.h =================================================================== --- vendor-sys/ck/20161230/include/ck_fifo.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_fifo.h (revision 310844) @@ -0,0 +1,478 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_FIFO_H +#define CK_FIFO_H + +#include +#include +#include +#include +#include + +#ifndef CK_F_FIFO_SPSC +#define CK_F_FIFO_SPSC +struct ck_fifo_spsc_entry { + void *value; + struct ck_fifo_spsc_entry *next; +}; +typedef struct ck_fifo_spsc_entry ck_fifo_spsc_entry_t; + +struct ck_fifo_spsc { + ck_spinlock_t m_head; + struct ck_fifo_spsc_entry *head; + char pad[CK_MD_CACHELINE - sizeof(struct ck_fifo_spsc_entry *) - sizeof(ck_spinlock_t)]; + ck_spinlock_t m_tail; + struct ck_fifo_spsc_entry *tail; + struct ck_fifo_spsc_entry *head_snapshot; + struct ck_fifo_spsc_entry *garbage; +}; +typedef struct ck_fifo_spsc ck_fifo_spsc_t; + +CK_CC_INLINE static bool +ck_fifo_spsc_enqueue_trylock(struct ck_fifo_spsc *fifo) +{ + + return ck_spinlock_trylock(&fifo->m_tail); +} + +CK_CC_INLINE static void +ck_fifo_spsc_enqueue_lock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_lock(&fifo->m_tail); + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_enqueue_unlock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_unlock(&fifo->m_tail); + return; +} + +CK_CC_INLINE static bool +ck_fifo_spsc_dequeue_trylock(struct ck_fifo_spsc *fifo) +{ + + return ck_spinlock_trylock(&fifo->m_head); +} + +CK_CC_INLINE static void +ck_fifo_spsc_dequeue_lock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_lock(&fifo->m_head); + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_dequeue_unlock(struct ck_fifo_spsc *fifo) +{ + + ck_spinlock_unlock(&fifo->m_head); + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_init(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry *stub) +{ + + ck_spinlock_init(&fifo->m_head); + ck_spinlock_init(&fifo->m_tail); + + stub->next = NULL; + fifo->head = fifo->tail = fifo->head_snapshot = fifo->garbage = stub; + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_deinit(struct ck_fifo_spsc *fifo, struct ck_fifo_spsc_entry **garbage) +{ + + *garbage = fifo->head; + fifo->head = fifo->tail = NULL; + return; +} + +CK_CC_INLINE static void +ck_fifo_spsc_enqueue(struct ck_fifo_spsc *fifo, + struct ck_fifo_spsc_entry *entry, + void *value) +{ + + entry->value = value; + entry->next = NULL; + + /* If stub->next is visible, guarantee that entry is consistent. */ + ck_pr_fence_store(); + ck_pr_store_ptr(&fifo->tail->next, entry); + fifo->tail = entry; + return; +} + +CK_CC_INLINE static bool +ck_fifo_spsc_dequeue(struct ck_fifo_spsc *fifo, void *value) +{ + struct ck_fifo_spsc_entry *entry; + + /* + * The head pointer is guaranteed to always point to a stub entry. + * If the stub entry does not point to an entry, then the queue is + * empty. + */ + entry = ck_pr_load_ptr(&fifo->head->next); + if (entry == NULL) + return false; + + /* If entry is visible, guarantee store to value is visible. */ + ck_pr_store_ptr_unsafe(value, entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr(&fifo->head, entry); + return true; +} + +/* + * Recycle a node. This technique for recycling nodes is based on + * Dmitriy Vyukov's work. + */ +CK_CC_INLINE static struct ck_fifo_spsc_entry * +ck_fifo_spsc_recycle(struct ck_fifo_spsc *fifo) +{ + struct ck_fifo_spsc_entry *garbage; + + if (fifo->head_snapshot == fifo->garbage) { + fifo->head_snapshot = ck_pr_load_ptr(&fifo->head); + if (fifo->head_snapshot == fifo->garbage) + return NULL; + } + + garbage = fifo->garbage; + fifo->garbage = garbage->next; + return garbage; +} + +CK_CC_INLINE static bool +ck_fifo_spsc_isempty(struct ck_fifo_spsc *fifo) +{ + struct ck_fifo_spsc_entry *head = ck_pr_load_ptr(&fifo->head); + return ck_pr_load_ptr(&head->next) == NULL; +} + +#define CK_FIFO_SPSC_ISEMPTY(f) ((f)->head->next == NULL) +#define CK_FIFO_SPSC_FIRST(f) ((f)->head->next) +#define CK_FIFO_SPSC_NEXT(m) ((m)->next) +#define CK_FIFO_SPSC_SPARE(f) ((f)->head) +#define CK_FIFO_SPSC_FOREACH(fifo, entry) \ + for ((entry) = CK_FIFO_SPSC_FIRST(fifo); \ + (entry) != NULL; \ + (entry) = CK_FIFO_SPSC_NEXT(entry)) +#define CK_FIFO_SPSC_FOREACH_SAFE(fifo, entry, T) \ + for ((entry) = CK_FIFO_SPSC_FIRST(fifo); \ + (entry) != NULL && ((T) = (entry)->next, 1); \ + (entry) = (T)) + +#endif /* CK_F_FIFO_SPSC */ + +#ifdef CK_F_PR_CAS_PTR_2 +#ifndef CK_F_FIFO_MPMC +#define CK_F_FIFO_MPMC +struct ck_fifo_mpmc_entry; +struct ck_fifo_mpmc_pointer { + struct ck_fifo_mpmc_entry *pointer; + char *generation CK_CC_PACKED; +} CK_CC_ALIGN(16); + +struct ck_fifo_mpmc_entry { + void *value; + struct ck_fifo_mpmc_pointer next; +}; +typedef struct ck_fifo_mpmc_entry ck_fifo_mpmc_entry_t; + +struct ck_fifo_mpmc { + struct ck_fifo_mpmc_pointer head; + char pad[CK_MD_CACHELINE - sizeof(struct ck_fifo_mpmc_pointer)]; + struct ck_fifo_mpmc_pointer tail; +}; +typedef struct ck_fifo_mpmc ck_fifo_mpmc_t; + +CK_CC_INLINE static void +ck_fifo_mpmc_init(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry *stub) +{ + + stub->next.pointer = NULL; + stub->next.generation = NULL; + fifo->head.pointer = fifo->tail.pointer = stub; + fifo->head.generation = fifo->tail.generation = NULL; + return; +} + +CK_CC_INLINE static void +ck_fifo_mpmc_deinit(struct ck_fifo_mpmc *fifo, struct ck_fifo_mpmc_entry **garbage) +{ + + *garbage = fifo->head.pointer; + fifo->head.pointer = fifo->tail.pointer = NULL; + return; +} + +CK_CC_INLINE static void +ck_fifo_mpmc_enqueue(struct ck_fifo_mpmc *fifo, + struct ck_fifo_mpmc_entry *entry, + void *value) +{ + struct ck_fifo_mpmc_pointer tail, next, update; + + /* + * Prepare the upcoming node and make sure to commit the updates + * before publishing. + */ + entry->value = value; + entry->next.pointer = NULL; + entry->next.generation = 0; + ck_pr_fence_store_atomic(); + + for (;;) { + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + next.generation = ck_pr_load_ptr(&tail.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer); + + if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation) + continue; + + if (next.pointer != NULL) { + /* + * If the tail pointer has an entry following it then + * it needs to be forwarded to the next entry. This + * helps us guarantee we are always operating on the + * last entry. + */ + update.pointer = next.pointer; + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + } else { + /* + * Attempt to commit new entry to the end of the + * current tail. + */ + update.pointer = entry; + update.generation = next.generation + 1; + if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == true) + break; + } + } + + ck_pr_fence_atomic(); + + /* After a successful insert, forward the tail to the new entry. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return; +} + +CK_CC_INLINE static bool +ck_fifo_mpmc_tryenqueue(struct ck_fifo_mpmc *fifo, + struct ck_fifo_mpmc_entry *entry, + void *value) +{ + struct ck_fifo_mpmc_pointer tail, next, update; + + entry->value = value; + entry->next.pointer = NULL; + entry->next.generation = 0; + + ck_pr_fence_store_atomic(); + + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + next.generation = ck_pr_load_ptr(&tail.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&tail.pointer->next.pointer); + + if (ck_pr_load_ptr(&fifo->tail.generation) != tail.generation) + return false; + + if (next.pointer != NULL) { + /* + * If the tail pointer has an entry following it then + * it needs to be forwarded to the next entry. This + * helps us guarantee we are always operating on the + * last entry. + */ + update.pointer = next.pointer; + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return false; + } else { + /* + * Attempt to commit new entry to the end of the + * current tail. + */ + update.pointer = entry; + update.generation = next.generation + 1; + if (ck_pr_cas_ptr_2(&tail.pointer->next, &next, &update) == false) + return false; + } + + ck_pr_fence_atomic(); + + /* After a successful insert, forward the tail to the new entry. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return true; +} + +CK_CC_INLINE static bool +ck_fifo_mpmc_dequeue(struct ck_fifo_mpmc *fifo, + void *value, + struct ck_fifo_mpmc_entry **garbage) +{ + struct ck_fifo_mpmc_pointer head, tail, next, update; + + for (;;) { + head.generation = ck_pr_load_ptr(&fifo->head.generation); + ck_pr_fence_load(); + head.pointer = ck_pr_load_ptr(&fifo->head.pointer); + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + + next.generation = ck_pr_load_ptr(&head.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&head.pointer->next.pointer); + + update.pointer = next.pointer; + if (head.pointer == tail.pointer) { + /* + * The head is guaranteed to always point at a stub + * entry. If the stub entry has no references then the + * queue is empty. + */ + if (next.pointer == NULL) + return false; + + /* Forward the tail pointer if necessary. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + } else { + /* + * It is possible for head snapshot to have been + * re-used. Avoid deferencing during enqueue + * re-use. + */ + if (next.pointer == NULL) + continue; + + /* Save value before commit. */ + *(void **)value = ck_pr_load_ptr(&next.pointer->value); + + /* Forward the head pointer to the next entry. */ + update.generation = head.generation + 1; + if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == true) + break; + } + } + + *garbage = head.pointer; + return true; +} + +CK_CC_INLINE static bool +ck_fifo_mpmc_trydequeue(struct ck_fifo_mpmc *fifo, + void *value, + struct ck_fifo_mpmc_entry **garbage) +{ + struct ck_fifo_mpmc_pointer head, tail, next, update; + + head.generation = ck_pr_load_ptr(&fifo->head.generation); + ck_pr_fence_load(); + head.pointer = ck_pr_load_ptr(&fifo->head.pointer); + + tail.generation = ck_pr_load_ptr(&fifo->tail.generation); + ck_pr_fence_load(); + tail.pointer = ck_pr_load_ptr(&fifo->tail.pointer); + + next.generation = ck_pr_load_ptr(&head.pointer->next.generation); + ck_pr_fence_load(); + next.pointer = ck_pr_load_ptr(&head.pointer->next.pointer); + + update.pointer = next.pointer; + if (head.pointer == tail.pointer) { + /* + * The head is guaranteed to always point at a stub + * entry. If the stub entry has no references then the + * queue is empty. + */ + if (next.pointer == NULL) + return false; + + /* Forward the tail pointer if necessary. */ + update.generation = tail.generation + 1; + ck_pr_cas_ptr_2(&fifo->tail, &tail, &update); + return false; + } else { + /* + * It is possible for head snapshot to have been + * re-used. Avoid deferencing during enqueue. + */ + if (next.pointer == NULL) + return false; + + /* Save value before commit. */ + *(void **)value = ck_pr_load_ptr(&next.pointer->value); + + /* Forward the head pointer to the next entry. */ + update.generation = head.generation + 1; + if (ck_pr_cas_ptr_2(&fifo->head, &head, &update) == false) + return false; + } + + *garbage = head.pointer; + return true; +} + +#define CK_FIFO_MPMC_ISEMPTY(f) ((f)->head.pointer->next.pointer == NULL) +#define CK_FIFO_MPMC_FIRST(f) ((f)->head.pointer->next.pointer) +#define CK_FIFO_MPMC_NEXT(m) ((m)->next.pointer) +#define CK_FIFO_MPMC_FOREACH(fifo, entry) \ + for ((entry) = CK_FIFO_MPMC_FIRST(fifo); \ + (entry) != NULL; \ + (entry) = CK_FIFO_MPMC_NEXT(entry)) +#define CK_FIFO_MPMC_FOREACH_SAFE(fifo, entry, T) \ + for ((entry) = CK_FIFO_MPMC_FIRST(fifo); \ + (entry) != NULL && ((T) = (entry)->next.pointer, 1); \ + (entry) = (T)) + +#endif /* CK_F_FIFO_MPMC */ +#endif /* CK_F_PR_CAS_PTR_2 */ + +#endif /* CK_FIFO_H */ Property changes on: vendor-sys/ck/20161230/include/ck_fifo.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_hp.h =================================================================== --- vendor-sys/ck/20161230/include/ck_hp.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_hp.h (revision 310844) @@ -0,0 +1,121 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HP_H +#define CK_HP_H + +#include +#include +#include +#include + +#ifndef CK_HP_CACHE +#define CK_HP_CACHE 512 +#endif + +struct ck_hp_hazard; +typedef void (*ck_hp_destructor_t)(void *); + +struct ck_hp { + ck_stack_t subscribers; + unsigned int n_subscribers; + unsigned int n_free; + unsigned int threshold; + unsigned int degree; + ck_hp_destructor_t destroy; +}; +typedef struct ck_hp ck_hp_t; + +struct ck_hp_hazard { + void *pointer; + void *data; + ck_stack_entry_t pending_entry; +}; +typedef struct ck_hp_hazard ck_hp_hazard_t; + +enum { + CK_HP_USED = 0, + CK_HP_FREE = 1 +}; + +struct ck_hp_record { + int state; + void **pointers; + void *cache[CK_HP_CACHE]; + struct ck_hp *global; + ck_stack_t pending; + unsigned int n_pending; + ck_stack_entry_t global_entry; + unsigned int n_peak; + uint64_t n_reclamations; +} CK_CC_CACHELINE; +typedef struct ck_hp_record ck_hp_record_t; + +CK_CC_INLINE static void +ck_hp_set(struct ck_hp_record *record, unsigned int i, void *pointer) +{ + + ck_pr_store_ptr(&record->pointers[i], pointer); + return; +} + +CK_CC_INLINE static void +ck_hp_set_fence(struct ck_hp_record *record, unsigned int i, void *pointer) +{ + +#ifdef CK_MD_TSO + ck_pr_fas_ptr(&record->pointers[i], pointer); +#else + ck_pr_store_ptr(&record->pointers[i], pointer); + ck_pr_fence_memory(); +#endif + + return; +} + +CK_CC_INLINE static void +ck_hp_clear(struct ck_hp_record *record) +{ + void **pointers = record->pointers; + unsigned int i; + + for (i = 0; i < record->global->degree; i++) + *pointers++ = NULL; + + return; +} + +void ck_hp_init(ck_hp_t *, unsigned int, unsigned int, ck_hp_destructor_t); +void ck_hp_set_threshold(ck_hp_t *, unsigned int); +void ck_hp_register(ck_hp_t *, ck_hp_record_t *, void **); +void ck_hp_unregister(ck_hp_record_t *); +ck_hp_record_t *ck_hp_recycle(ck_hp_t *); +void ck_hp_reclaim(ck_hp_record_t *); +void ck_hp_free(ck_hp_record_t *, ck_hp_hazard_t *, void *, void *); +void ck_hp_retire(ck_hp_record_t *, ck_hp_hazard_t *, void *, void *); +void ck_hp_purge(ck_hp_record_t *); + +#endif /* CK_HP_H */ Property changes on: vendor-sys/ck/20161230/include/ck_hp.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_hp_fifo.h =================================================================== --- vendor-sys/ck/20161230/include/ck_hp_fifo.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_hp_fifo.h (revision 310844) @@ -0,0 +1,215 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HP_FIFO_H +#define CK_HP_FIFO_H + +#include +#include +#include +#include + +#define CK_HP_FIFO_SLOTS_COUNT (2) +#define CK_HP_FIFO_SLOTS_SIZE (sizeof(void *) * CK_HP_FIFO_SLOTS_COUNT) + +/* + * Though it is possible to embed the data structure, measurements need + * to be made for the cost of this. If we were to embed the hazard pointer + * state into the data structure, this means every deferred reclamation + * will also include a cache invalidation when linking into the hazard pointer + * pending queue. This may lead to terrible cache line bouncing. + */ +struct ck_hp_fifo_entry { + void *value; + ck_hp_hazard_t hazard; + struct ck_hp_fifo_entry *next; +}; +typedef struct ck_hp_fifo_entry ck_hp_fifo_entry_t; + +struct ck_hp_fifo { + struct ck_hp_fifo_entry *head; + struct ck_hp_fifo_entry *tail; +}; +typedef struct ck_hp_fifo ck_hp_fifo_t; + +CK_CC_INLINE static void +ck_hp_fifo_init(struct ck_hp_fifo *fifo, struct ck_hp_fifo_entry *stub) +{ + + fifo->head = fifo->tail = stub; + stub->next = NULL; + return; +} + +CK_CC_INLINE static void +ck_hp_fifo_deinit(struct ck_hp_fifo *fifo, struct ck_hp_fifo_entry **stub) +{ + + *stub = fifo->head; + fifo->head = fifo->tail = NULL; + return; +} + +CK_CC_INLINE static void +ck_hp_fifo_enqueue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + struct ck_hp_fifo_entry *entry, + void *value) +{ + struct ck_hp_fifo_entry *tail, *next; + + entry->value = value; + entry->next = NULL; + ck_pr_fence_store_atomic(); + + for (;;) { + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, tail); + if (tail != ck_pr_load_ptr(&fifo->tail)) + continue; + + next = ck_pr_load_ptr(&tail->next); + if (next != NULL) { + ck_pr_cas_ptr(&fifo->tail, tail, next); + continue; + } else if (ck_pr_cas_ptr(&fifo->tail->next, next, entry) == true) + break; + } + + ck_pr_fence_atomic(); + ck_pr_cas_ptr(&fifo->tail, tail, entry); + return; +} + +CK_CC_INLINE static bool +ck_hp_fifo_tryenqueue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + struct ck_hp_fifo_entry *entry, + void *value) +{ + struct ck_hp_fifo_entry *tail, *next; + + entry->value = value; + entry->next = NULL; + ck_pr_fence_store_atomic(); + + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, tail); + if (tail != ck_pr_load_ptr(&fifo->tail)) + return false; + + next = ck_pr_load_ptr(&tail->next); + if (next != NULL) { + ck_pr_cas_ptr(&fifo->tail, tail, next); + return false; + } else if (ck_pr_cas_ptr(&fifo->tail->next, next, entry) == false) + return false; + + ck_pr_fence_atomic(); + ck_pr_cas_ptr(&fifo->tail, tail, entry); + return true; +} + +CK_CC_INLINE static struct ck_hp_fifo_entry * +ck_hp_fifo_dequeue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + void *value) +{ + struct ck_hp_fifo_entry *head, *tail, *next; + + for (;;) { + head = ck_pr_load_ptr(&fifo->head); + ck_pr_fence_load(); + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, head); + if (head != ck_pr_load_ptr(&fifo->head)) + continue; + + next = ck_pr_load_ptr(&head->next); + ck_hp_set_fence(record, 1, next); + if (head != ck_pr_load_ptr(&fifo->head)) + continue; + + if (head == tail) { + if (next == NULL) + return NULL; + + ck_pr_cas_ptr(&fifo->tail, tail, next); + continue; + } else if (ck_pr_cas_ptr(&fifo->head, head, next) == true) + break; + } + + ck_pr_store_ptr_unsafe(value, next->value); + return head; +} + +CK_CC_INLINE static struct ck_hp_fifo_entry * +ck_hp_fifo_trydequeue_mpmc(ck_hp_record_t *record, + struct ck_hp_fifo *fifo, + void *value) +{ + struct ck_hp_fifo_entry *head, *tail, *next; + + head = ck_pr_load_ptr(&fifo->head); + ck_pr_fence_load(); + tail = ck_pr_load_ptr(&fifo->tail); + ck_hp_set_fence(record, 0, head); + if (head != ck_pr_load_ptr(&fifo->head)) + return NULL; + + next = ck_pr_load_ptr(&head->next); + ck_hp_set_fence(record, 1, next); + if (head != ck_pr_load_ptr(&fifo->head)) + return NULL; + + if (head == tail) { + if (next == NULL) + return NULL; + + ck_pr_cas_ptr(&fifo->tail, tail, next); + return NULL; + } else if (ck_pr_cas_ptr(&fifo->head, head, next) == false) + return NULL; + + ck_pr_store_ptr_unsafe(value, next->value); + return head; +} + +#define CK_HP_FIFO_ISEMPTY(f) ((f)->head->next == NULL) +#define CK_HP_FIFO_FIRST(f) ((f)->head->next) +#define CK_HP_FIFO_NEXT(m) ((m)->next) +#define CK_HP_FIFO_FOREACH(fifo, entry) \ + for ((entry) = CK_HP_FIFO_FIRST(fifo); \ + (entry) != NULL; \ + (entry) = CK_HP_FIFO_NEXT(entry)) +#define CK_HP_FIFO_FOREACH_SAFE(fifo, entry, T) \ + for ((entry) = CK_HP_FIFO_FIRST(fifo); \ + (entry) != NULL && ((T) = (entry)->next, 1); \ + (entry) = (T)) + +#endif /* CK_HP_FIFO_H */ Property changes on: vendor-sys/ck/20161230/include/ck_hp_fifo.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_hp_stack.h =================================================================== --- vendor-sys/ck/20161230/include/ck_hp_stack.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_hp_stack.h (revision 310844) @@ -0,0 +1,110 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HP_STACK_H +#define CK_HP_STACK_H + +#include +#include +#include +#include +#include + +#define CK_HP_STACK_SLOTS_COUNT 1 +#define CK_HP_STACK_SLOTS_SIZE sizeof(void *) + +CK_CC_INLINE static void +ck_hp_stack_push_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + ck_stack_push_upmc(target, entry); + return; +} + +CK_CC_INLINE static bool +ck_hp_stack_trypush_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + return ck_stack_trypush_upmc(target, entry); +} + +CK_CC_INLINE static struct ck_stack_entry * +ck_hp_stack_pop_mpmc(ck_hp_record_t *record, struct ck_stack *target) +{ + struct ck_stack_entry *entry, *update; + + do { + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return NULL; + + ck_hp_set_fence(record, 0, entry); + } while (entry != ck_pr_load_ptr(&target->head)); + + while (ck_pr_cas_ptr_value(&target->head, entry, entry->next, &entry) == false) { + if (entry == NULL) + return NULL; + + ck_hp_set_fence(record, 0, entry); + + update = ck_pr_load_ptr(&target->head); + while (entry != update) { + ck_hp_set_fence(record, 0, update); + entry = update; + update = ck_pr_load_ptr(&target->head); + if (update == NULL) + return NULL; + } + } + + return entry; +} + +CK_CC_INLINE static bool +ck_hp_stack_trypop_mpmc(ck_hp_record_t *record, struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return false; + + ck_hp_set_fence(record, 0, entry); + if (entry != ck_pr_load_ptr(&target->head)) + goto leave; + + if (ck_pr_cas_ptr_value(&target->head, entry, entry->next, &entry) == false) + goto leave; + + *r = entry; + return true; + +leave: + ck_hp_set(record, 0, NULL); + return false; +} + +#endif /* CK_HP_STACK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_hp_stack.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_hs.h =================================================================== --- vendor-sys/ck/20161230/include/ck_hs.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_hs.h (revision 310844) @@ -0,0 +1,134 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HS_H +#define CK_HS_H + +#include +#include +#include +#include +#include +#include +#include + +/* + * Indicates a single-writer many-reader workload. Mutually + * exclusive with CK_HS_MODE_MPMC + */ +#define CK_HS_MODE_SPMC 1 + +/* + * Indicates that values to be stored are not pointers but + * values. Allows for full precision. Mutually exclusive + * with CK_HS_MODE_OBJECT. + */ +#define CK_HS_MODE_DIRECT 2 + +/* + * Indicates that the values to be stored are pointers. + * Allows for space optimizations in the presence of pointer + * packing. Mutually exclusive with CK_HS_MODE_DIRECT. + */ +#define CK_HS_MODE_OBJECT 8 + +/* + * Indicates a delete-heavy workload. This will reduce the + * need for garbage collection at the cost of approximately + * 12% to 20% increased memory usage. + */ +#define CK_HS_MODE_DELETE 16 + +/* Currently unsupported. */ +#define CK_HS_MODE_MPMC (void) + +/* + * Hash callback function. + */ +typedef unsigned long ck_hs_hash_cb_t(const void *, unsigned long); + +/* + * Returns pointer to object if objects are equivalent. + */ +typedef bool ck_hs_compare_cb_t(const void *, const void *); + +#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS) +#define CK_HS_PP +#define CK_HS_KEY_MASK ((1U << ((sizeof(void *) * 8) - CK_MD_VMA_BITS)) - 1) +#endif + +struct ck_hs_map; +struct ck_hs { + struct ck_malloc *m; + struct ck_hs_map *map; + unsigned int mode; + unsigned long seed; + ck_hs_hash_cb_t *hf; + ck_hs_compare_cb_t *compare; +}; +typedef struct ck_hs ck_hs_t; + +struct ck_hs_stat { + unsigned long tombstones; + unsigned long n_entries; + unsigned int probe_maximum; +}; + +struct ck_hs_iterator { + void **cursor; + unsigned long offset; +}; +typedef struct ck_hs_iterator ck_hs_iterator_t; + +#define CK_HS_ITERATOR_INITIALIZER { NULL, 0 } + +/* Convenience wrapper to table hash function. */ +#define CK_HS_HASH(T, F, K) F((K), (T)->seed) + +typedef void *ck_hs_apply_fn_t(void *, void *); +bool ck_hs_apply(ck_hs_t *, unsigned long, const void *, ck_hs_apply_fn_t *, void *); +void ck_hs_iterator_init(ck_hs_iterator_t *); +bool ck_hs_next(ck_hs_t *, ck_hs_iterator_t *, void **); +bool ck_hs_move(ck_hs_t *, ck_hs_t *, ck_hs_hash_cb_t *, + ck_hs_compare_cb_t *, struct ck_malloc *); +bool ck_hs_init(ck_hs_t *, unsigned int, ck_hs_hash_cb_t *, + ck_hs_compare_cb_t *, struct ck_malloc *, unsigned long, unsigned long); +void ck_hs_destroy(ck_hs_t *); +void *ck_hs_get(ck_hs_t *, unsigned long, const void *); +bool ck_hs_put(ck_hs_t *, unsigned long, const void *); +bool ck_hs_put_unique(ck_hs_t *, unsigned long, const void *); +bool ck_hs_set(ck_hs_t *, unsigned long, const void *, void **); +bool ck_hs_fas(ck_hs_t *, unsigned long, const void *, void **); +void *ck_hs_remove(ck_hs_t *, unsigned long, const void *); +bool ck_hs_grow(ck_hs_t *, unsigned long); +bool ck_hs_rebuild(ck_hs_t *); +bool ck_hs_gc(ck_hs_t *, unsigned long, unsigned long); +unsigned long ck_hs_count(ck_hs_t *); +bool ck_hs_reset(ck_hs_t *); +bool ck_hs_reset_size(ck_hs_t *, unsigned long); +void ck_hs_stat(ck_hs_t *, struct ck_hs_stat *); + +#endif /* CK_HS_H */ Property changes on: vendor-sys/ck/20161230/include/ck_hs.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_ht.h =================================================================== --- vendor-sys/ck/20161230/include/ck_ht.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_ht.h (revision 310844) @@ -0,0 +1,271 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HT_H +#define CK_HT_H + +#include + +#define CK_F_HT +#if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_STORE_64) +#define CK_HT_TYPE uint64_t +#define CK_HT_TYPE_LOAD ck_pr_load_64 +#define CK_HT_TYPE_STORE ck_pr_store_64 +#define CK_HT_TYPE_MAX UINT64_MAX +#else +#define CK_HT_TYPE uint32_t +#define CK_HT_TYPE_LOAD ck_pr_load_32 +#define CK_HT_TYPE_STORE ck_pr_store_32 +#define CK_HT_TYPE_MAX UINT32_MAX +#endif + + +#include +#include +#include +#include +#include +#include + +struct ck_ht_hash { + uint64_t value; +}; +typedef struct ck_ht_hash ck_ht_hash_t; + +#define CK_HT_MODE_DIRECT 1U +#define CK_HT_MODE_BYTESTRING 2U +#define CK_HT_WORKLOAD_DELETE 4U + +#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS) +#define CK_HT_PP +#define CK_HT_KEY_LENGTH ((sizeof(void *) * 8) - CK_MD_VMA_BITS) +#define CK_HT_KEY_MASK ((1U << CK_HT_KEY_LENGTH) - 1) +#else +#define CK_HT_KEY_LENGTH 65535U +#endif + +struct ck_ht_entry { +#ifdef CK_HT_PP + uintptr_t key; + uintptr_t value CK_CC_PACKED; +} CK_CC_ALIGN(16); +#else + uintptr_t key; + uintptr_t value; + CK_HT_TYPE key_length; + CK_HT_TYPE hash; +} CK_CC_ALIGN(32); +#endif +typedef struct ck_ht_entry ck_ht_entry_t; + +/* + * The user is free to define their own stub values. + */ +#ifndef CK_HT_KEY_EMPTY +#define CK_HT_KEY_EMPTY ((uintptr_t)0) +#endif + +#ifndef CK_HT_KEY_TOMBSTONE +#define CK_HT_KEY_TOMBSTONE (~CK_HT_KEY_EMPTY) +#endif + +/* + * Hash callback function. First argument is updated to contain a hash value, + * second argument is the key, third argument is key length and final argument + * is the hash table seed value. + */ +typedef void ck_ht_hash_cb_t(ck_ht_hash_t *, const void *, size_t, uint64_t); + +struct ck_ht_map; +struct ck_ht { + struct ck_malloc *m; + struct ck_ht_map *map; + unsigned int mode; + uint64_t seed; + ck_ht_hash_cb_t *h; +}; +typedef struct ck_ht ck_ht_t; + +struct ck_ht_stat { + uint64_t probe_maximum; + uint64_t n_entries; +}; + +struct ck_ht_iterator { + struct ck_ht_entry *current; + uint64_t offset; +}; +typedef struct ck_ht_iterator ck_ht_iterator_t; + +#define CK_HT_ITERATOR_INITIALIZER { NULL, 0 } + +CK_CC_INLINE static void +ck_ht_iterator_init(struct ck_ht_iterator *iterator) +{ + + iterator->current = NULL; + iterator->offset = 0; + return; +} + +CK_CC_INLINE static bool +ck_ht_entry_empty(ck_ht_entry_t *entry) +{ + + return entry->key == CK_HT_KEY_EMPTY; +} + +CK_CC_INLINE static void +ck_ht_entry_key_set_direct(ck_ht_entry_t *entry, uintptr_t key) +{ + + entry->key = key; + return; +} + +CK_CC_INLINE static void +ck_ht_entry_key_set(ck_ht_entry_t *entry, const void *key, uint16_t key_length) +{ + +#ifdef CK_HT_PP + entry->key = (uintptr_t)key | ((uintptr_t)key_length << CK_MD_VMA_BITS); +#else + entry->key = (uintptr_t)key; + entry->key_length = key_length; +#endif + + return; +} + +CK_CC_INLINE static void * +ck_ht_entry_key(ck_ht_entry_t *entry) +{ + +#ifdef CK_HT_PP + return (void *)(entry->key & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1)); +#else + return (void *)entry->key; +#endif +} + +CK_CC_INLINE static uint16_t +ck_ht_entry_key_length(ck_ht_entry_t *entry) +{ + +#ifdef CK_HT_PP + return entry->key >> CK_MD_VMA_BITS; +#else + return entry->key_length; +#endif +} + +CK_CC_INLINE static void * +ck_ht_entry_value(ck_ht_entry_t *entry) +{ + +#ifdef CK_HT_PP + return (void *)(entry->value & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1)); +#else + return (void *)entry->value; +#endif +} + +CK_CC_INLINE static void +ck_ht_entry_set(struct ck_ht_entry *entry, + ck_ht_hash_t h, + const void *key, + uint16_t key_length, + const void *value) +{ + +#ifdef CK_HT_PP + entry->key = (uintptr_t)key | ((uintptr_t)key_length << CK_MD_VMA_BITS); + entry->value = (uintptr_t)value | ((uintptr_t)(h.value >> 32) << CK_MD_VMA_BITS); +#else + entry->key = (uintptr_t)key; + entry->value = (uintptr_t)value; + entry->key_length = key_length; + entry->hash = h.value; +#endif + + return; +} + +CK_CC_INLINE static void +ck_ht_entry_set_direct(struct ck_ht_entry *entry, + ck_ht_hash_t h, + uintptr_t key, + uintptr_t value) +{ + + entry->key = key; + entry->value = value; + +#ifndef CK_HT_PP + entry->hash = h.value; +#else + (void)h; +#endif + return; +} + +CK_CC_INLINE static uintptr_t +ck_ht_entry_key_direct(ck_ht_entry_t *entry) +{ + + return entry->key; +} + +CK_CC_INLINE static uintptr_t +ck_ht_entry_value_direct(ck_ht_entry_t *entry) +{ + + return entry->value; +} + +/* + * Iteration must occur without any concurrent mutations on + * the hash table. + */ +bool ck_ht_next(ck_ht_t *, ck_ht_iterator_t *, ck_ht_entry_t **entry); + +void ck_ht_stat(ck_ht_t *, struct ck_ht_stat *); +void ck_ht_hash(ck_ht_hash_t *, ck_ht_t *, const void *, uint16_t); +void ck_ht_hash_direct(ck_ht_hash_t *, ck_ht_t *, uintptr_t); +bool ck_ht_init(ck_ht_t *, unsigned int, ck_ht_hash_cb_t *, + struct ck_malloc *, CK_HT_TYPE, uint64_t); +void ck_ht_destroy(ck_ht_t *); +bool ck_ht_set_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_put_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_get_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_gc(struct ck_ht *, unsigned long, unsigned long); +bool ck_ht_grow_spmc(ck_ht_t *, CK_HT_TYPE); +bool ck_ht_remove_spmc(ck_ht_t *, ck_ht_hash_t, ck_ht_entry_t *); +bool ck_ht_reset_spmc(ck_ht_t *); +bool ck_ht_reset_size_spmc(ck_ht_t *, CK_HT_TYPE); +CK_HT_TYPE ck_ht_count(ck_ht_t *); + +#endif /* CK_HT_H */ Property changes on: vendor-sys/ck/20161230/include/ck_ht.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_limits.h =================================================================== --- vendor-sys/ck/20161230/include/ck_limits.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_limits.h (revision 310844) @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__linux__) && defined(__KERNEL__) +#include + +#ifndef UINT8_MAX +#define UINT8_MAX ((u8)(~0U)) +#endif +#ifndef UINT16_MAX +#define UINT16_MAX USHRT_MAX +#endif +#ifndef UINT32_MAX +#define UINT32_MAX UINT_MAX +#endif +#ifndef UINT64_MAX +#define UINT64_MAX ULLONG_MAX +#endif + +#elif defined(__FreeBSD__) && defined(_KERNEL) +#include +#include +#else +#include +#endif /* __linux__ && __KERNEL__ */ Property changes on: vendor-sys/ck/20161230/include/ck_limits.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_malloc.h =================================================================== --- vendor-sys/ck/20161230/include/ck_malloc.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_malloc.h (revision 310844) @@ -0,0 +1,39 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_MALLOC_H +#define CK_MALLOC_H + +#include +#include + +struct ck_malloc { + void *(*malloc)(size_t); + void *(*realloc)(void *, size_t, size_t, bool); + void (*free)(void *, size_t, bool); +}; + +#endif /* CK_MALLOC_H */ Property changes on: vendor-sys/ck/20161230/include/ck_malloc.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_pflock.h =================================================================== --- vendor-sys/ck/20161230/include/ck_pflock.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_pflock.h (revision 310844) @@ -0,0 +1,142 @@ +/* + * Copyright 2013 John Wittrock. + * Copyright 2013-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_PFLOCK_H +#define CK_PFLOCK_H + +/* + * This is an implementation of phase-fair locks derived from the work + * described in: + * Brandenburg, B. and Anderson, J. 2010. Spin-Based + * Reader-Writer Synchronization for Multiprocessor Real-Time Systems + */ + +#include +#include + +struct ck_pflock { + uint32_t rin; + uint32_t rout; + uint32_t win; + uint32_t wout; +}; +typedef struct ck_pflock ck_pflock_t; + +#define CK_PFLOCK_LSB 0xFFFFFFF0 +#define CK_PFLOCK_RINC 0x100 /* Reader increment value. */ +#define CK_PFLOCK_WBITS 0x3 /* Writer bits in reader. */ +#define CK_PFLOCK_PRES 0x2 /* Writer present bit. */ +#define CK_PFLOCK_PHID 0x1 /* Phase ID bit. */ + +#define CK_PFLOCK_INITIALIZER {0, 0, 0, 0} + +CK_CC_INLINE static void +ck_pflock_init(struct ck_pflock *pf) +{ + + pf->rin = 0; + pf->rout = 0; + pf->win = 0; + pf->wout = 0; + ck_pr_barrier(); + + return; +} + +CK_CC_INLINE static void +ck_pflock_write_unlock(ck_pflock_t *pf) +{ + + ck_pr_fence_unlock(); + + /* Migrate from write phase to read phase. */ + ck_pr_and_32(&pf->rin, CK_PFLOCK_LSB); + + /* Allow other writers to continue. */ + ck_pr_faa_32(&pf->wout, 1); + return; +} + +CK_CC_INLINE static void +ck_pflock_write_lock(ck_pflock_t *pf) +{ + uint32_t ticket; + + /* Acquire ownership of write-phase. */ + ticket = ck_pr_faa_32(&pf->win, 1); + while (ck_pr_load_32(&pf->wout) != ticket) + ck_pr_stall(); + + /* + * Acquire ticket on read-side in order to allow them + * to flush. Indicates to any incoming reader that a + * write-phase is pending. + */ + ticket = ck_pr_faa_32(&pf->rin, + (ticket & CK_PFLOCK_PHID) | CK_PFLOCK_PRES); + + /* Wait for any pending readers to flush. */ + while (ck_pr_load_32(&pf->rout) != ticket) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_pflock_read_unlock(ck_pflock_t *pf) +{ + + ck_pr_fence_unlock(); + ck_pr_faa_32(&pf->rout, CK_PFLOCK_RINC); + return; +} + +CK_CC_INLINE static void +ck_pflock_read_lock(ck_pflock_t *pf) +{ + uint32_t w; + + /* + * If no writer is present, then the operation has completed + * successfully. + */ + w = ck_pr_faa_32(&pf->rin, CK_PFLOCK_RINC) & CK_PFLOCK_WBITS; + if (w == 0) + goto leave; + + /* Wait for current write phase to complete. */ + while ((ck_pr_load_32(&pf->rin) & CK_PFLOCK_WBITS) == w) + ck_pr_stall(); + +leave: + /* Acquire semantics with respect to readers. */ + ck_pr_fence_lock(); + return; +} + +#endif /* CK_PFLOCK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_pflock.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_queue.h =================================================================== --- vendor-sys/ck/20161230/include/ck_queue.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_queue.h (revision 310844) @@ -0,0 +1,428 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/*- + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + * $FreeBSD$ + */ + +#ifndef CK_QUEUE_H +#define CK_QUEUE_H + +#include + +/* + * This file defines three types of data structures: singly-linked lists, + * singly-linked tail queues and lists. + * + * A singly-linked list is headed by a single forward pointer. The elements + * are singly linked for minimum space and pointer manipulation overhead at + * the expense of O(n) removal for arbitrary elements. New elements can be + * added to the list after an existing element or at the head of the list. + * Elements being removed from the head of the list should use the explicit + * macro for this purpose for optimum efficiency. A singly-linked list may + * only be traversed in the forward direction. Singly-linked lists are ideal + * for applications with large datasets and few or no removals or for + * implementing a LIFO queue. + * + * A singly-linked tail queue is headed by a pair of pointers, one to the + * head of the list and the other to the tail of the list. The elements are + * singly linked for minimum space and pointer manipulation overhead at the + * expense of O(n) removal for arbitrary elements. New elements can be added + * to the list after an existing element, at the head of the list, or at the + * end of the list. Elements being removed from the head of the tail queue + * should use the explicit macro for this purpose for optimum efficiency. + * A singly-linked tail queue may only be traversed in the forward direction. + * Singly-linked tail queues are ideal for applications with large datasets + * and few or no removals or for implementing a FIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may only be traversed in the forward direction. + * + * It is safe to use _FOREACH/_FOREACH_SAFE in the presence of concurrent + * modifications to the list. Writers to these lists must, on the other hand, + * implement writer-side synchronization. The _SWAP operations are not atomic. + * This facility is currently unsupported on architectures such as the Alpha + * which require load-depend memory fences. + * + * CK_SLIST CK_LIST CK_STAILQ + * _HEAD + + + + * _HEAD_INITIALIZER + + + + * _ENTRY + + + + * _INIT + + + + * _EMPTY + + + + * _FIRST + + + + * _NEXT + + + + * _FOREACH + + + + * _FOREACH_SAFE + + + + * _INSERT_HEAD + + + + * _INSERT_BEFORE - + - + * _INSERT_AFTER + + + + * _INSERT_TAIL - - + + * _REMOVE_AFTER + - + + * _REMOVE_HEAD + - + + * _REMOVE + + + + * _SWAP + + + + * _MOVE + + + + */ + +/* + * Singly-linked List declarations. + */ +#define CK_SLIST_HEAD(name, type) \ +struct name { \ + struct type *slh_first; /* first element */ \ +} + +#define CK_SLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define CK_SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +/* + * Singly-linked List functions. + */ +#define CK_SLIST_EMPTY(head) \ + (ck_pr_load_ptr(&(head)->slh_first) == NULL) + +#define CK_SLIST_FIRST(head) \ + (ck_pr_load_ptr(&(head)->slh_first)) + +#define CK_SLIST_NEXT(elm, field) \ + ck_pr_load_ptr(&((elm)->field.sle_next)) + +#define CK_SLIST_FOREACH(var, head, field) \ + for ((var) = CK_SLIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_SLIST_NEXT((var), field)) + +#define CK_SLIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_SLIST_FIRST(head); \ + (var) && (ck_pr_fence_load(), (tvar) = CK_SLIST_NEXT(var, field), 1);\ + (var) = (tvar)) + +#define CK_SLIST_FOREACH_PREVPTR(var, varp, head, field) \ + for ((varp) = &(head)->slh_first; \ + ((var) = ck_pr_load_ptr(varp)) != NULL && (ck_pr_fence_load(), 1); \ + (varp) = &(var)->field.sle_next) + +#define CK_SLIST_INIT(head) do { \ + ck_pr_store_ptr(&(head)->slh_first, NULL); \ + ck_pr_fence_store(); \ +} while (0) + +#define CK_SLIST_INSERT_AFTER(a, b, field) do { \ + (b)->field.sle_next = (a)->field.sle_next; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(a)->field.sle_next, b); \ +} while (0) + +#define CK_SLIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.sle_next = (head)->slh_first; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(head)->slh_first, elm); \ +} while (0) + +#define CK_SLIST_REMOVE_AFTER(elm, field) do { \ + ck_pr_store_ptr(&(elm)->field.sle_next, \ + (elm)->field.sle_next->field.sle_next); \ +} while (0) + +#define CK_SLIST_REMOVE(head, elm, type, field) do { \ + if ((head)->slh_first == (elm)) { \ + CK_SLIST_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->slh_first; \ + while (curelm->field.sle_next != (elm)) \ + curelm = curelm->field.sle_next; \ + CK_SLIST_REMOVE_AFTER(curelm, field); \ + } \ +} while (0) + +#define CK_SLIST_REMOVE_HEAD(head, field) do { \ + ck_pr_store_ptr(&(head)->slh_first, \ + (head)->slh_first->field.sle_next); \ +} while (0) + +#define CK_SLIST_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->slh_first, (head2)->slh_first); \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_SLIST_SWAP(a, b, type) do { \ + struct type *swap_first = (a)->slh_first; \ + (a)->slh_first = (b)->slh_first; \ + (b)->slh_first = swap_first; \ +} while (0) + +/* + * Singly-linked Tail queue declarations. + */ +#define CK_STAILQ_HEAD(name, type) \ +struct name { \ + struct type *stqh_first;/* first element */ \ + struct type **stqh_last;/* addr of last next element */ \ +} + +#define CK_STAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).stqh_first } + +#define CK_STAILQ_ENTRY(type) \ +struct { \ + struct type *stqe_next; /* next element */ \ +} + +/* + * Singly-linked Tail queue functions. + */ +#define CK_STAILQ_CONCAT(head1, head2) do { \ + if ((head2)->stqh_first == NULL) { \ + ck_pr_store_ptr((head1)->stqh_last, (head2)->stqh_first); \ + ck_pr_fence_store(); \ + (head1)->stqh_last = (head2)->stqh_last; \ + CK_STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define CK_STAILQ_EMPTY(head) (ck_pr_load_ptr(&(head)->stqh_first) == NULL) + +#define CK_STAILQ_FIRST(head) (ck_pr_load_ptr(&(head)->stqh_first)) + +#define CK_STAILQ_FOREACH(var, head, field) \ + for((var) = CK_STAILQ_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_STAILQ_NEXT((var), field)) + +#define CK_STAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_STAILQ_FIRST((head)); \ + (var) && (ck_pr_fence_load(), (tvar) = \ + CK_STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define CK_STAILQ_INIT(head) do { \ + ck_pr_store_ptr(&(head)->stqh_first, NULL); \ + ck_pr_fence_store(); \ + (head)->stqh_last = &(head)->stqh_first; \ +} while (0) + +#define CK_STAILQ_INSERT_AFTER(head, tqelm, elm, field) do { \ + (elm)->field.stqe_next = (tqelm)->field.stqe_next; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(tqelm)->field.stqe_next, elm); \ + if ((elm)->field.stqe_next == NULL) \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.stqe_next = (head)->stqh_first; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr(&(head)->stqh_first, elm); \ + if ((elm)->field.stqe_next == NULL) \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_INSERT_TAIL(head, elm, field) do { \ + (elm)->field.stqe_next = NULL; \ + ck_pr_fence_store(); \ + ck_pr_store_ptr((head)->stqh_last, (elm)); \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_NEXT(elm, field) \ + (ck_pr_load_ptr(&(elm)->field.stqe_next)) + +#define CK_STAILQ_REMOVE(head, elm, type, field) do { \ + if ((head)->stqh_first == (elm)) { \ + CK_STAILQ_REMOVE_HEAD((head), field); \ + } else { \ + struct type *curelm = (head)->stqh_first; \ + while (curelm->field.stqe_next != (elm)) \ + curelm = curelm->field.stqe_next; \ + CK_STAILQ_REMOVE_AFTER(head, curelm, field); \ + } \ +} while (0) + +#define CK_STAILQ_REMOVE_AFTER(head, elm, field) do { \ + ck_pr_store_ptr(&(elm)->field.stqe_next, \ + (elm)->field.stqe_next->field.stqe_next); \ + if ((elm)->field.stqe_next == NULL) \ + (head)->stqh_last = &(elm)->field.stqe_next; \ +} while (0) + +#define CK_STAILQ_REMOVE_HEAD(head, field) do { \ + ck_pr_store_ptr(&(head)->stqh_first, \ + (head)->stqh_first->field.stqe_next); \ + if ((head)->stqh_first == NULL) \ + (head)->stqh_last = &(head)->stqh_first; \ +} while (0) + +#define CK_STAILQ_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->stqh_first, (head2)->stqh_first); \ + (head1)->stqh_last = (head2)->stqh_last; \ + if ((head2)->stqh_last == &(head2)->stqh_first) \ + (head1)->stqh_last = &(head1)->stqh_first; \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_STAILQ_SWAP(head1, head2, type) do { \ + struct type *swap_first = CK_STAILQ_FIRST(head1); \ + struct type **swap_last = (head1)->stqh_last; \ + CK_STAILQ_FIRST(head1) = CK_STAILQ_FIRST(head2); \ + (head1)->stqh_last = (head2)->stqh_last; \ + CK_STAILQ_FIRST(head2) = swap_first; \ + (head2)->stqh_last = swap_last; \ + if (CK_STAILQ_EMPTY(head1)) \ + (head1)->stqh_last = &(head1)->stqh_first; \ + if (CK_STAILQ_EMPTY(head2)) \ + (head2)->stqh_last = &(head2)->stqh_first; \ +} while (0) + +/* + * List declarations. + */ +#define CK_LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define CK_LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define CK_LIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +#define CK_LIST_FIRST(head) ck_pr_load_ptr(&(head)->lh_first) +#define CK_LIST_EMPTY(head) (CK_LIST_FIRST(head) == NULL) +#define CK_LIST_NEXT(elm, field) ck_pr_load_ptr(&(elm)->field.le_next) + +#define CK_LIST_FOREACH(var, head, field) \ + for ((var) = CK_LIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), 1); \ + (var) = CK_LIST_NEXT((var), field)) + +#define CK_LIST_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = CK_LIST_FIRST((head)); \ + (var) && (ck_pr_fence_load(), (tvar) = CK_LIST_NEXT((var), field), 1);\ + (var) = (tvar)) + +#define CK_LIST_INIT(head) do { \ + ck_pr_store_ptr(&(head)->lh_first, NULL); \ + ck_pr_fence_store(); \ +} while (0) + +#define CK_LIST_INSERT_AFTER(listelm, elm, field) do { \ + (elm)->field.le_next = (listelm)->field.le_next; \ + (elm)->field.le_prev = &(listelm)->field.le_next; \ + ck_pr_fence_store(); \ + if ((listelm)->field.le_next != NULL) \ + (listelm)->field.le_next->field.le_prev = &(elm)->field.le_next;\ + ck_pr_store_ptr(&(listelm)->field.le_next, elm); \ +} while (0) + +#define CK_LIST_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + (elm)->field.le_next = (listelm); \ + ck_pr_fence_store(); \ + ck_pr_store_ptr((listelm)->field.le_prev, (elm)); \ + (listelm)->field.le_prev = &(elm)->field.le_next; \ +} while (0) + +#define CK_LIST_INSERT_HEAD(head, elm, field) do { \ + (elm)->field.le_next = (head)->lh_first; \ + ck_pr_fence_store(); \ + if ((elm)->field.le_next != NULL) \ + (head)->lh_first->field.le_prev = &(elm)->field.le_next; \ + ck_pr_store_ptr(&(head)->lh_first, elm); \ + (elm)->field.le_prev = &(head)->lh_first; \ +} while (0) + +#define CK_LIST_REMOVE(elm, field) do { \ + ck_pr_store_ptr((elm)->field.le_prev, (elm)->field.le_next); \ + if ((elm)->field.le_next != NULL) \ + (elm)->field.le_next->field.le_prev = (elm)->field.le_prev; \ +} while (0) + +#define CK_LIST_MOVE(head1, head2, field) do { \ + ck_pr_store_ptr(&(head1)->lh_first, (head2)->lh_first); \ + if ((head1)->lh_first != NULL) \ + (head1)->lh_first->field.le_prev = &(head1)->lh_first; \ +} while (0) + +/* + * This operation is not applied atomically. + */ +#define CK_LIST_SWAP(head1, head2, type, field) do { \ + struct type *swap_tmp = (head1)->lh_first; \ + (head1)->lh_first = (head2)->lh_first; \ + (head2)->lh_first = swap_tmp; \ + if ((swap_tmp = (head1)->lh_first) != NULL) \ + swap_tmp->field.le_prev = &(head1)->lh_first; \ + if ((swap_tmp = (head2)->lh_first) != NULL) \ + swap_tmp->field.le_prev = &(head2)->lh_first; \ +} while (0) + +#endif /* CK_QUEUE_H */ Property changes on: vendor-sys/ck/20161230/include/ck_queue.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_rhs.h =================================================================== --- vendor-sys/ck/20161230/include/ck_rhs.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_rhs.h (revision 310844) @@ -0,0 +1,134 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RHS_H +#define CK_RHS_H + +#include +#include +#include +#include +#include +#include +#include + +/* + * Indicates a single-writer many-reader workload. Mutually + * exclusive with CK_RHS_MODE_MPMC + */ +#define CK_RHS_MODE_SPMC 1 + +/* + * Indicates that values to be stored are not pointers but + * values. Allows for full precision. Mutually exclusive + * with CK_RHS_MODE_OBJECT. + */ +#define CK_RHS_MODE_DIRECT 2 + +/* + * Indicates that the values to be stored are pointers. + * Allows for space optimizations in the presence of pointer + * packing. Mutually exclusive with CK_RHS_MODE_DIRECT. + */ +#define CK_RHS_MODE_OBJECT 8 + +/* + * Indicated that the load is read-mostly, so get should be optimized + * over put and delete + */ +#define CK_RHS_MODE_READ_MOSTLY 16 + +/* Currently unsupported. */ +#define CK_RHS_MODE_MPMC (void) + +/* + * Hash callback function. + */ +typedef unsigned long ck_rhs_hash_cb_t(const void *, unsigned long); + +/* + * Returns pointer to object if objects are equivalent. + */ +typedef bool ck_rhs_compare_cb_t(const void *, const void *); + +#if defined(CK_MD_POINTER_PACK_ENABLE) && defined(CK_MD_VMA_BITS) +#define CK_RHS_PP +#define CK_RHS_KEY_MASK ((1U << ((sizeof(void *) * 8) - CK_MD_VMA_BITS)) - 1) +#endif + +struct ck_rhs_map; +struct ck_rhs { + struct ck_malloc *m; + struct ck_rhs_map *map; + unsigned int mode; + unsigned int load_factor; + unsigned long seed; + ck_rhs_hash_cb_t *hf; + ck_rhs_compare_cb_t *compare; +}; +typedef struct ck_rhs ck_rhs_t; + +struct ck_rhs_stat { + unsigned long n_entries; + unsigned int probe_maximum; +}; + +struct ck_rhs_iterator { + void **cursor; + unsigned long offset; +}; +typedef struct ck_rhs_iterator ck_rhs_iterator_t; + +#define CK_RHS_ITERATOR_INITIALIZER { NULL, 0 } + +/* Convenience wrapper to table hash function. */ +#define CK_RHS_HASH(T, F, K) F((K), (T)->seed) + +typedef void *ck_rhs_apply_fn_t(void *, void *); +bool ck_rhs_apply(ck_rhs_t *, unsigned long, const void *, ck_rhs_apply_fn_t *, void *); +void ck_rhs_iterator_init(ck_rhs_iterator_t *); +bool ck_rhs_next(ck_rhs_t *, ck_rhs_iterator_t *, void **); +bool ck_rhs_move(ck_rhs_t *, ck_rhs_t *, ck_rhs_hash_cb_t *, + ck_rhs_compare_cb_t *, struct ck_malloc *); +bool ck_rhs_init(ck_rhs_t *, unsigned int, ck_rhs_hash_cb_t *, + ck_rhs_compare_cb_t *, struct ck_malloc *, unsigned long, unsigned long); +void ck_rhs_destroy(ck_rhs_t *); +void *ck_rhs_get(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_put(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_put_unique(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_set(ck_rhs_t *, unsigned long, const void *, void **); +bool ck_rhs_fas(ck_rhs_t *, unsigned long, const void *, void **); +void *ck_rhs_remove(ck_rhs_t *, unsigned long, const void *); +bool ck_rhs_grow(ck_rhs_t *, unsigned long); +bool ck_rhs_rebuild(ck_rhs_t *); +bool ck_rhs_gc(ck_rhs_t *); +unsigned long ck_rhs_count(ck_rhs_t *); +bool ck_rhs_reset(ck_rhs_t *); +bool ck_rhs_reset_size(ck_rhs_t *, unsigned long); +void ck_rhs_stat(ck_rhs_t *, struct ck_rhs_stat *); +bool ck_rhs_set_load_factor(ck_rhs_t *, unsigned int); + +#endif /* CK_RHS_H */ Property changes on: vendor-sys/ck/20161230/include/ck_rhs.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_ring.h =================================================================== --- vendor-sys/ck/20161230/include/ck_ring.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_ring.h (revision 310844) @@ -0,0 +1,656 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RING_H +#define CK_RING_H + +#include +#include +#include +#include +#include + +/* + * Concurrent ring buffer. + */ + +struct ck_ring { + unsigned int c_head; + char pad[CK_MD_CACHELINE - sizeof(unsigned int)]; + unsigned int p_tail; + unsigned int p_head; + char _pad[CK_MD_CACHELINE - sizeof(unsigned int) * 2]; + unsigned int size; + unsigned int mask; +}; +typedef struct ck_ring ck_ring_t; + +struct ck_ring_buffer { + void *value; +}; +typedef struct ck_ring_buffer ck_ring_buffer_t; + +CK_CC_INLINE static unsigned int +ck_ring_size(const struct ck_ring *ring) +{ + unsigned int c, p; + + c = ck_pr_load_uint(&ring->c_head); + p = ck_pr_load_uint(&ring->p_tail); + return (p - c) & ring->mask; +} + +CK_CC_INLINE static unsigned int +ck_ring_capacity(const struct ck_ring *ring) +{ + return ring->size; +} + +CK_CC_INLINE static void +ck_ring_init(struct ck_ring *ring, unsigned int size) +{ + + ring->size = size; + ring->mask = size - 1; + ring->p_tail = 0; + ring->p_head = 0; + ring->c_head = 0; + return; +} + +/* + * The _ck_ring_* namespace is internal only and must not used externally. + */ +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_sp(struct ck_ring *ring, + void *CK_CC_RESTRICT buffer, + const void *CK_CC_RESTRICT entry, + unsigned int ts, + unsigned int *size) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer, delta; + + consumer = ck_pr_load_uint(&ring->c_head); + producer = ring->p_tail; + delta = producer + 1; + if (size != NULL) + *size = (producer - consumer) & mask; + + if (CK_CC_UNLIKELY((delta & mask) == (consumer & mask))) + return false; + + buffer = (char *)buffer + ts * (producer & mask); + memcpy(buffer, entry, ts); + + /* + * Make sure to update slot value before indicating + * that the slot is available for consumption. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&ring->p_tail, delta); + return true; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_sp_size(struct ck_ring *ring, + void *CK_CC_RESTRICT buffer, + const void *CK_CC_RESTRICT entry, + unsigned int ts, + unsigned int *size) +{ + unsigned int sz; + bool r; + + r = _ck_ring_enqueue_sp(ring, buffer, entry, ts, &sz); + *size = sz; + return r; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_dequeue_sc(struct ck_ring *ring, + const void *CK_CC_RESTRICT buffer, + void *CK_CC_RESTRICT target, + unsigned int size) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer; + + consumer = ring->c_head; + producer = ck_pr_load_uint(&ring->p_tail); + + if (CK_CC_UNLIKELY(consumer == producer)) + return false; + + /* + * Make sure to serialize with respect to our snapshot + * of the producer counter. + */ + ck_pr_fence_load(); + + buffer = (const char *)buffer + size * (consumer & mask); + memcpy(target, buffer, size); + + /* + * Make sure copy is completed with respect to consumer + * update. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&ring->c_head, consumer + 1); + return true; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_mp(struct ck_ring *ring, + void *buffer, + const void *entry, + unsigned int ts, + unsigned int *size) +{ + const unsigned int mask = ring->mask; + unsigned int producer, consumer, delta; + bool r = true; + + producer = ck_pr_load_uint(&ring->p_head); + + do { + /* + * The snapshot of producer must be up to date with + * respect to consumer. + */ + ck_pr_fence_load(); + consumer = ck_pr_load_uint(&ring->c_head); + + delta = producer + 1; + if (CK_CC_UNLIKELY((delta & mask) == (consumer & mask))) { + r = false; + goto leave; + } + } while (ck_pr_cas_uint_value(&ring->p_head, + producer, + delta, + &producer) == false); + + buffer = (char *)buffer + ts * (producer & mask); + memcpy(buffer, entry, ts); + + /* + * Wait until all concurrent producers have completed writing + * their data into the ring buffer. + */ + while (ck_pr_load_uint(&ring->p_tail) != producer) + ck_pr_stall(); + + /* + * Ensure that copy is completed before updating shared producer + * counter. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&ring->p_tail, delta); + +leave: + if (size != NULL) + *size = (producer - consumer) & mask; + + return r; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_enqueue_mp_size(struct ck_ring *ring, + void *buffer, + const void *entry, + unsigned int ts, + unsigned int *size) +{ + unsigned int sz; + bool r; + + r = _ck_ring_enqueue_mp(ring, buffer, entry, ts, &sz); + *size = sz; + return r; +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_trydequeue_mc(struct ck_ring *ring, + const void *buffer, + void *data, + unsigned int size) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer; + + consumer = ck_pr_load_uint(&ring->c_head); + ck_pr_fence_load(); + producer = ck_pr_load_uint(&ring->p_tail); + + if (CK_CC_UNLIKELY(consumer == producer)) + return false; + + ck_pr_fence_load(); + + buffer = (const char *)buffer + size * (consumer & mask); + memcpy(data, buffer, size); + + ck_pr_fence_store_atomic(); + return ck_pr_cas_uint(&ring->c_head, consumer, consumer + 1); +} + +CK_CC_FORCE_INLINE static bool +_ck_ring_dequeue_mc(struct ck_ring *ring, + const void *buffer, + void *data, + unsigned int ts) +{ + const unsigned int mask = ring->mask; + unsigned int consumer, producer; + + consumer = ck_pr_load_uint(&ring->c_head); + + do { + const char *target; + + /* + * Producer counter must represent state relative to + * our latest consumer snapshot. + */ + ck_pr_fence_load(); + producer = ck_pr_load_uint(&ring->p_tail); + + if (CK_CC_UNLIKELY(consumer == producer)) + return false; + + ck_pr_fence_load(); + + target = (const char *)buffer + ts * (consumer & mask); + memcpy(data, target, ts); + + /* Serialize load with respect to head update. */ + ck_pr_fence_store_atomic(); + } while (ck_pr_cas_uint_value(&ring->c_head, + consumer, + consumer + 1, + &consumer) == false); + + return true; +} + +/* + * The ck_ring_*_spsc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is only provided if there is up + * to one concurrent consumer and up to one concurrent producer. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_spsc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_sp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_spsc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_sp(ring, buffer, + &entry, sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_spsc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_sc(ring, buffer, + (void **)data, sizeof(void *)); +} + +/* + * The ck_ring_*_mpmc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is provided for any number of + * producers and consumers. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_mpmc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_mp(ring, buffer, &entry, + sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_mpmc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_mp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_trydequeue_mpmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_trydequeue_mc(ring, + buffer, (void **)data, sizeof(void *)); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_mpmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_mc(ring, buffer, (void **)data, + sizeof(void *)); +} + +/* + * The ck_ring_*_spmc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is provided for any number of + * consumers with up to one concurrent producer. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_spmc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_sp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_spmc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_sp(ring, buffer, &entry, + sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_trydequeue_spmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_trydequeue_mc(ring, buffer, (void **)data, sizeof(void *)); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_spmc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_mc(ring, buffer, (void **)data, sizeof(void *)); +} + +/* + * The ck_ring_*_mpsc namespace is the public interface for interacting with a + * ring buffer containing pointers. Correctness is provided for any number of + * producers with up to one concurrent consumers. + */ +CK_CC_INLINE static bool +ck_ring_enqueue_mpsc(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry) +{ + + return _ck_ring_enqueue_mp(ring, buffer, &entry, + sizeof(entry), NULL); +} + +CK_CC_INLINE static bool +ck_ring_enqueue_mpsc_size(struct ck_ring *ring, + struct ck_ring_buffer *buffer, + const void *entry, + unsigned int *size) +{ + + return _ck_ring_enqueue_mp_size(ring, buffer, &entry, + sizeof(entry), size); +} + +CK_CC_INLINE static bool +ck_ring_dequeue_mpsc(struct ck_ring *ring, + const struct ck_ring_buffer *buffer, + void *data) +{ + + return _ck_ring_dequeue_sc(ring, buffer, (void **)data, + sizeof(void *)); +} + +/* + * CK_RING_PROTOTYPE is used to define a type-safe interface for inlining + * values of a particular type in the ring the buffer. + */ +#define CK_RING_PROTOTYPE(name, type) \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spsc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_sp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_sp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_spsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_sc(a, b, c, \ + sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spmc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_sp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_spmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_sp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_trydequeue_spmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_trydequeue_mc(a, \ + b, c, sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_spmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_mc(a, b, c, \ + sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_mp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpsc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_mp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_mpsc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_sc(a, b, c, \ + sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpmc_size_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c, \ + unsigned int *d) \ +{ \ + \ + return _ck_ring_enqueue_mp_size(a, b, c, \ + sizeof(struct type), d); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_enqueue_mpmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_enqueue_mp(a, b, c, \ + sizeof(struct type), NULL); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_trydequeue_mpmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_trydequeue_mc(a, \ + b, c, sizeof(struct type)); \ +} \ + \ +CK_CC_INLINE static bool \ +ck_ring_dequeue_mpmc_##name(struct ck_ring *a, \ + struct type *b, \ + struct type *c) \ +{ \ + \ + return _ck_ring_dequeue_mc(a, b, c, \ + sizeof(struct type)); \ +} + +/* + * A single producer with one concurrent consumer. + */ +#define CK_RING_ENQUEUE_SPSC(name, a, b, c) \ + ck_ring_enqueue_spsc_##name(a, b, c) +#define CK_RING_ENQUEUE_SPSC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_spsc_size_##name(a, b, c, d) +#define CK_RING_DEQUEUE_SPSC(name, a, b, c) \ + ck_ring_dequeue_spsc_##name(a, b, c) + +/* + * A single producer with any number of concurrent consumers. + */ +#define CK_RING_ENQUEUE_SPMC(name, a, b, c) \ + ck_ring_enqueue_spmc_##name(a, b, c) +#define CK_RING_ENQUEUE_SPMC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_spmc_size_##name(a, b, c, d) +#define CK_RING_TRYDEQUEUE_SPMC(name, a, b, c) \ + ck_ring_trydequeue_spmc_##name(a, b, c) +#define CK_RING_DEQUEUE_SPMC(name, a, b, c) \ + ck_ring_dequeue_spmc_##name(a, b, c) + +/* + * Any number of concurrent producers with up to one + * concurrent consumer. + */ +#define CK_RING_ENQUEUE_MPSC(name, a, b, c) \ + ck_ring_enqueue_mpsc_##name(a, b, c) +#define CK_RING_ENQUEUE_MPSC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_mpsc_size_##name(a, b, c, d) +#define CK_RING_DEQUEUE_MPSC(name, a, b, c) \ + ck_ring_dequeue_mpsc_##name(a, b, c) + +/* + * Any number of concurrent producers and consumers. + */ +#define CK_RING_ENQUEUE_MPMC(name, a, b, c) \ + ck_ring_enqueue_mpmc_##name(a, b, c) +#define CK_RING_ENQUEUE_MPMC_SIZE(name, a, b, c, d) \ + ck_ring_enqueue_mpmc_size_##name(a, b, c, d) +#define CK_RING_TRYDEQUEUE_MPMC(name, a, b, c) \ + ck_ring_trydequeue_mpmc_##name(a, b, c) +#define CK_RING_DEQUEUE_MPMC(name, a, b, c) \ + ck_ring_dequeue_mpmc_##name(a, b, c) + +#endif /* CK_RING_H */ Property changes on: vendor-sys/ck/20161230/include/ck_ring.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_rwcohort.h =================================================================== --- vendor-sys/ck/20161230/include/ck_rwcohort.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_rwcohort.h (revision 310844) @@ -0,0 +1,317 @@ +/* + * Copyright 2013-2015 Samy Al Bahra. + * Copyright 2013 Brendon Scheinman. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RWCOHORT_H +#define CK_RWCOHORT_H + +/* + * This is an implementation of NUMA-aware reader-writer locks as described in: + * Calciu, I.; Dice, D.; Lev, Y.; Luchangco, V.; Marathe, V.; and Shavit, N. 2014. + * NUMA-Aware Reader-Writer Locks + */ + +#include +#include +#include +#include + +#define CK_RWCOHORT_WP_NAME(N) ck_rwcohort_wp_##N +#define CK_RWCOHORT_WP_INSTANCE(N) struct CK_RWCOHORT_WP_NAME(N) +#define CK_RWCOHORT_WP_INIT(N, RW, WL) ck_rwcohort_wp_##N##_init(RW, WL) +#define CK_RWCOHORT_WP_READ_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_read_lock(RW, C, GC, LC) +#define CK_RWCOHORT_WP_READ_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_read_unlock(RW) +#define CK_RWCOHORT_WP_WRITE_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_write_lock(RW, C, GC, LC) +#define CK_RWCOHORT_WP_WRITE_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_wp_##N##_write_unlock(RW, C, GC, LC) +#define CK_RWCOHORT_WP_DEFAULT_WAIT_LIMIT 1000 + +#define CK_RWCOHORT_WP_PROTOTYPE(N) \ + CK_RWCOHORT_WP_INSTANCE(N) { \ + unsigned int read_counter; \ + unsigned int write_barrier; \ + unsigned int wait_limit; \ + }; \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_init(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + unsigned int wait_limit) \ + { \ + \ + rw_cohort->read_counter = 0; \ + rw_cohort->write_barrier = 0; \ + rw_cohort->wait_limit = wait_limit; \ + ck_pr_barrier(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_write_lock(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + while (ck_pr_load_uint(&rw_cohort->write_barrier) > 0) \ + ck_pr_stall(); \ + \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + \ + while (ck_pr_load_uint(&rw_cohort->read_counter) > 0) \ + ck_pr_stall(); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_write_unlock(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + (void)rw_cohort; \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_read_lock(CK_RWCOHORT_WP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + unsigned int wait_count = 0; \ + bool raised = false; \ + \ + for (;;) { \ + ck_pr_inc_uint(&rw_cohort->read_counter); \ + ck_pr_fence_atomic_load(); \ + if (CK_COHORT_LOCKED(N, cohort, global_context, \ + local_context) == false) \ + break; \ + \ + ck_pr_dec_uint(&rw_cohort->read_counter); \ + while (CK_COHORT_LOCKED(N, cohort, global_context, \ + local_context) == true) { \ + ck_pr_stall(); \ + if (++wait_count > rw_cohort->wait_limit && \ + raised == false) { \ + ck_pr_inc_uint(&rw_cohort->write_barrier); \ + raised = true; \ + } \ + } \ + } \ + \ + if (raised == true) \ + ck_pr_dec_uint(&rw_cohort->write_barrier); \ + \ + ck_pr_fence_load(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_wp_##N##_read_unlock(CK_RWCOHORT_WP_INSTANCE(N) *cohort) \ + { \ + \ + ck_pr_fence_load_atomic(); \ + ck_pr_dec_uint(&cohort->read_counter); \ + return; \ + } + +#define CK_RWCOHORT_WP_INITIALIZER { \ + .read_counter = 0, \ + .write_barrier = 0, \ + .wait_limit = 0 \ +} + +#define CK_RWCOHORT_RP_NAME(N) ck_rwcohort_rp_##N +#define CK_RWCOHORT_RP_INSTANCE(N) struct CK_RWCOHORT_RP_NAME(N) +#define CK_RWCOHORT_RP_INIT(N, RW, WL) ck_rwcohort_rp_##N##_init(RW, WL) +#define CK_RWCOHORT_RP_READ_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_read_lock(RW, C, GC, LC) +#define CK_RWCOHORT_RP_READ_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_read_unlock(RW) +#define CK_RWCOHORT_RP_WRITE_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_write_lock(RW, C, GC, LC) +#define CK_RWCOHORT_RP_WRITE_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_rp_##N##_write_unlock(RW, C, GC, LC) +#define CK_RWCOHORT_RP_DEFAULT_WAIT_LIMIT 1000 + +#define CK_RWCOHORT_RP_PROTOTYPE(N) \ + CK_RWCOHORT_RP_INSTANCE(N) { \ + unsigned int read_counter; \ + unsigned int read_barrier; \ + unsigned int wait_limit; \ + }; \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_init(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + unsigned int wait_limit) \ + { \ + \ + rw_cohort->read_counter = 0; \ + rw_cohort->read_barrier = 0; \ + rw_cohort->wait_limit = wait_limit; \ + ck_pr_barrier(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_write_lock(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + unsigned int wait_count = 0; \ + bool raised = false; \ + \ + for (;;) { \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + if (ck_pr_load_uint(&rw_cohort->read_counter) == 0) \ + break; \ + \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + while (ck_pr_load_uint(&rw_cohort->read_counter) > 0) { \ + ck_pr_stall(); \ + if (++wait_count > rw_cohort->wait_limit && \ + raised == false) { \ + ck_pr_inc_uint(&rw_cohort->read_barrier); \ + raised = true; \ + } \ + } \ + } \ + \ + if (raised == true) \ + ck_pr_dec_uint(&rw_cohort->read_barrier); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_write_unlock(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, void *local_context) \ + { \ + \ + (void)rw_cohort; \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_read_lock(CK_RWCOHORT_RP_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + while (ck_pr_load_uint(&rw_cohort->read_barrier) > 0) \ + ck_pr_stall(); \ + \ + ck_pr_inc_uint(&rw_cohort->read_counter); \ + ck_pr_fence_atomic_load(); \ + \ + while (CK_COHORT_LOCKED(N, cohort, global_context, \ + local_context) == true) \ + ck_pr_stall(); \ + \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_rp_##N##_read_unlock(CK_RWCOHORT_RP_INSTANCE(N) *cohort) \ + { \ + \ + ck_pr_fence_load_atomic(); \ + ck_pr_dec_uint(&cohort->read_counter); \ + return; \ + } + +#define CK_RWCOHORT_RP_INITIALIZER { \ + .read_counter = 0, \ + .read_barrier = 0, \ + .wait_limit = 0 \ +} + +#define CK_RWCOHORT_NEUTRAL_NAME(N) ck_rwcohort_neutral_##N +#define CK_RWCOHORT_NEUTRAL_INSTANCE(N) struct CK_RWCOHORT_NEUTRAL_NAME(N) +#define CK_RWCOHORT_NEUTRAL_INIT(N, RW) ck_rwcohort_neutral_##N##_init(RW) +#define CK_RWCOHORT_NEUTRAL_READ_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_read_lock(RW, C, GC, LC) +#define CK_RWCOHORT_NEUTRAL_READ_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_read_unlock(RW) +#define CK_RWCOHORT_NEUTRAL_WRITE_LOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_write_lock(RW, C, GC, LC) +#define CK_RWCOHORT_NEUTRAL_WRITE_UNLOCK(N, RW, C, GC, LC) \ + ck_rwcohort_neutral_##N##_write_unlock(RW, C, GC, LC) +#define CK_RWCOHORT_NEUTRAL_DEFAULT_WAIT_LIMIT 1000 + +#define CK_RWCOHORT_NEUTRAL_PROTOTYPE(N) \ + CK_RWCOHORT_NEUTRAL_INSTANCE(N) { \ + unsigned int read_counter; \ + }; \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_init(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort) \ + { \ + \ + rw_cohort->read_counter = 0; \ + ck_pr_barrier(); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_write_lock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort,\ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + while (ck_pr_load_uint(&rw_cohort->read_counter) > 0) { \ + ck_pr_stall(); \ + } \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_write_unlock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort,\ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, void *local_context) \ + { \ + \ + (void)rw_cohort; \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_read_lock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *rw_cohort, \ + CK_COHORT_INSTANCE(N) *cohort, void *global_context, \ + void *local_context) \ + { \ + \ + CK_COHORT_LOCK(N, cohort, global_context, local_context); \ + ck_pr_inc_uint(&rw_cohort->read_counter); \ + CK_COHORT_UNLOCK(N, cohort, global_context, local_context); \ + return; \ + } \ + CK_CC_INLINE static void \ + ck_rwcohort_neutral_##N##_read_unlock(CK_RWCOHORT_NEUTRAL_INSTANCE(N) *cohort) \ + { \ + \ + ck_pr_fence_load_atomic(); \ + ck_pr_dec_uint(&cohort->read_counter); \ + return; \ + } + +#define CK_RWCOHORT_NEUTRAL_INITIALIZER { \ + .read_counter = 0, \ +} + +#endif /* CK_RWCOHORT_H */ Property changes on: vendor-sys/ck/20161230/include/ck_rwcohort.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_rwlock.h =================================================================== --- vendor-sys/ck/20161230/include/ck_rwlock.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_rwlock.h (revision 310844) @@ -0,0 +1,302 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_RWLOCK_H +#define CK_RWLOCK_H + +#include +#include +#include +#include + +struct ck_rwlock { + unsigned int writer; + unsigned int n_readers; +}; +typedef struct ck_rwlock ck_rwlock_t; + +#define CK_RWLOCK_INITIALIZER {0, 0} + +CK_CC_INLINE static void +ck_rwlock_init(struct ck_rwlock *rw) +{ + + rw->writer = 0; + rw->n_readers = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_rwlock_write_unlock(ck_rwlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&rw->writer, 0); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_locked_writer(ck_rwlock_t *rw) +{ + bool r; + + r = ck_pr_load_uint(&rw->writer); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_rwlock_write_downgrade(ck_rwlock_t *rw) +{ + + ck_pr_inc_uint(&rw->n_readers); + ck_rwlock_write_unlock(rw); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_locked(ck_rwlock_t *rw) +{ + bool l; + + l = ck_pr_load_uint(&rw->n_readers) | + ck_pr_load_uint(&rw->writer); + ck_pr_fence_acquire(); + return l; +} + +CK_CC_INLINE static bool +ck_rwlock_write_trylock(ck_rwlock_t *rw) +{ + + if (ck_pr_fas_uint(&rw->writer, 1) != 0) + return false; + + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->n_readers) != 0) { + ck_rwlock_write_unlock(rw); + return false; + } + + ck_pr_fence_lock(); + return true; +} + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_rwlock_write, ck_rwlock_t, + ck_rwlock_locked, ck_rwlock_write_trylock) + +CK_CC_INLINE static void +ck_rwlock_write_lock(ck_rwlock_t *rw) +{ + + while (ck_pr_fas_uint(&rw->writer, 1) != 0) + ck_pr_stall(); + + ck_pr_fence_atomic_load(); + + while (ck_pr_load_uint(&rw->n_readers) != 0) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_ELIDE_PROTOTYPE(ck_rwlock_write, ck_rwlock_t, + ck_rwlock_locked, ck_rwlock_write_lock, + ck_rwlock_locked_writer, ck_rwlock_write_unlock) + +CK_CC_INLINE static bool +ck_rwlock_read_trylock(ck_rwlock_t *rw) +{ + + if (ck_pr_load_uint(&rw->writer) != 0) + return false; + + ck_pr_inc_uint(&rw->n_readers); + + /* + * Serialize with respect to concurrent write + * lock operation. + */ + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->writer) == 0) { + ck_pr_fence_lock(); + return true; + } + + ck_pr_dec_uint(&rw->n_readers); + return false; +} + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_rwlock_read, ck_rwlock_t, + ck_rwlock_locked_writer, ck_rwlock_read_trylock) + +CK_CC_INLINE static void +ck_rwlock_read_lock(ck_rwlock_t *rw) +{ + + for (;;) { + while (ck_pr_load_uint(&rw->writer) != 0) + ck_pr_stall(); + + ck_pr_inc_uint(&rw->n_readers); + + /* + * Serialize with respect to concurrent write + * lock operation. + */ + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->writer) == 0) + break; + + ck_pr_dec_uint(&rw->n_readers); + } + + /* Acquire semantics are necessary. */ + ck_pr_fence_load(); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_locked_reader(ck_rwlock_t *rw) +{ + + ck_pr_fence_load(); + return ck_pr_load_uint(&rw->n_readers); +} + +CK_CC_INLINE static void +ck_rwlock_read_unlock(ck_rwlock_t *rw) +{ + + ck_pr_fence_load_atomic(); + ck_pr_dec_uint(&rw->n_readers); + return; +} + +CK_ELIDE_PROTOTYPE(ck_rwlock_read, ck_rwlock_t, + ck_rwlock_locked_writer, ck_rwlock_read_lock, + ck_rwlock_locked_reader, ck_rwlock_read_unlock) + +/* + * Recursive writer reader-writer lock implementation. + */ +struct ck_rwlock_recursive { + struct ck_rwlock rw; + unsigned int wc; +}; +typedef struct ck_rwlock_recursive ck_rwlock_recursive_t; + +#define CK_RWLOCK_RECURSIVE_INITIALIZER {CK_RWLOCK_INITIALIZER, 0} + +CK_CC_INLINE static void +ck_rwlock_recursive_write_lock(ck_rwlock_recursive_t *rw, unsigned int tid) +{ + unsigned int o; + + o = ck_pr_load_uint(&rw->rw.writer); + if (o == tid) + goto leave; + + while (ck_pr_cas_uint(&rw->rw.writer, 0, tid) == false) + ck_pr_stall(); + + ck_pr_fence_atomic_load(); + + while (ck_pr_load_uint(&rw->rw.n_readers) != 0) + ck_pr_stall(); + + ck_pr_fence_lock(); +leave: + rw->wc++; + return; +} + +CK_CC_INLINE static bool +ck_rwlock_recursive_write_trylock(ck_rwlock_recursive_t *rw, unsigned int tid) +{ + unsigned int o; + + o = ck_pr_load_uint(&rw->rw.writer); + if (o == tid) + goto leave; + + if (ck_pr_cas_uint(&rw->rw.writer, 0, tid) == false) + return false; + + ck_pr_fence_atomic_load(); + + if (ck_pr_load_uint(&rw->rw.n_readers) != 0) { + ck_pr_store_uint(&rw->rw.writer, 0); + return false; + } + + ck_pr_fence_lock(); +leave: + rw->wc++; + return true; +} + +CK_CC_INLINE static void +ck_rwlock_recursive_write_unlock(ck_rwlock_recursive_t *rw) +{ + + if (--rw->wc == 0) { + ck_pr_fence_unlock(); + ck_pr_store_uint(&rw->rw.writer, 0); + } + + return; +} + +CK_CC_INLINE static void +ck_rwlock_recursive_read_lock(ck_rwlock_recursive_t *rw) +{ + + ck_rwlock_read_lock(&rw->rw); + return; +} + +CK_CC_INLINE static bool +ck_rwlock_recursive_read_trylock(ck_rwlock_recursive_t *rw) +{ + + return ck_rwlock_read_trylock(&rw->rw); +} + +CK_CC_INLINE static void +ck_rwlock_recursive_read_unlock(ck_rwlock_recursive_t *rw) +{ + + ck_rwlock_read_unlock(&rw->rw); + return; +} + +#endif /* CK_RWLOCK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_rwlock.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_sequence.h =================================================================== --- vendor-sys/ck/20161230/include/ck_sequence.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_sequence.h (revision 310844) @@ -0,0 +1,125 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SEQUENCE_H +#define CK_SEQUENCE_H + +#include +#include +#include + +struct ck_sequence { + unsigned int sequence; +}; +typedef struct ck_sequence ck_sequence_t; + +#define CK_SEQUENCE_INITIALIZER { .sequence = 0 } + +CK_CC_INLINE static void +ck_sequence_init(struct ck_sequence *sq) +{ + + ck_pr_store_uint(&sq->sequence, 0); + return; +} + +CK_CC_INLINE static unsigned int +ck_sequence_read_begin(const struct ck_sequence *sq) +{ + unsigned int version; + + for (;;) { + version = ck_pr_load_uint(&sq->sequence); + + /* + * If a sequence is even then associated data may be in a + * consistent state. + */ + if (CK_CC_LIKELY((version & 1) == 0)) + break; + + /* + * If a sequence is odd then a thread is in the middle of an + * update. Retry the read to avoid operating on inconsistent + * data. + */ + ck_pr_stall(); + } + + ck_pr_fence_load(); + return version; +} + +CK_CC_INLINE static bool +ck_sequence_read_retry(const struct ck_sequence *sq, unsigned int version) +{ + + /* + * If the sequence number was updated then a read should be + * re-attempted. + */ + ck_pr_fence_load(); + return ck_pr_load_uint(&sq->sequence) != version; +} + +#define CK_SEQUENCE_READ(seqlock, version) \ + for (*(version) = 1; \ + (*(version) != 0) && (*(version) = ck_sequence_read_begin(seqlock), 1); \ + *(version) = ck_sequence_read_retry(seqlock, *(version))) + +/* + * This must be called after a successful mutex acquisition. + */ +CK_CC_INLINE static void +ck_sequence_write_begin(struct ck_sequence *sq) +{ + + /* + * Increment the sequence to an odd number to indicate + * the beginning of a write update. + */ + ck_pr_store_uint(&sq->sequence, sq->sequence + 1); + ck_pr_fence_store(); + return; +} + +/* + * This must be called before mutex ownership is relinquished. + */ +CK_CC_INLINE static void +ck_sequence_write_end(struct ck_sequence *sq) +{ + + /* + * Increment the sequence to an even number to indicate + * completion of a write update. + */ + ck_pr_fence_store(); + ck_pr_store_uint(&sq->sequence, sq->sequence + 1); + return; +} + +#endif /* CK_SEQUENCE_H */ Property changes on: vendor-sys/ck/20161230/include/ck_sequence.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_spinlock.h =================================================================== --- vendor-sys/ck/20161230/include/ck_spinlock.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_spinlock.h (revision 310844) @@ -0,0 +1,61 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_H +#define CK_SPINLOCK_H + +#include "spinlock/anderson.h" +#include "spinlock/cas.h" +#include "spinlock/clh.h" +#include "spinlock/dec.h" +#include "spinlock/fas.h" +#include "spinlock/hclh.h" +#include "spinlock/mcs.h" +#include "spinlock/ticket.h" + +/* + * On tested x86, x86_64, PPC64 and SPARC64 targets, + * ck_spinlock_fas proved to have lowest latency + * in fast path testing or negligible degradation + * from faster but less robust implementations. + */ +#define CK_SPINLOCK_INITIALIZER CK_SPINLOCK_FAS_INITIALIZER +#define ck_spinlock_t ck_spinlock_fas_t +#define ck_spinlock_init(x) ck_spinlock_fas_init(x) +#define ck_spinlock_lock(x) ck_spinlock_fas_lock(x) +#define ck_spinlock_lock_eb(x) ck_spinlock_fas_lock_eb(x) +#define ck_spinlock_unlock(x) ck_spinlock_fas_unlock(x) +#define ck_spinlock_locked(x) ck_spinlock_fas_locked(x) +#define ck_spinlock_trylock(x) ck_spinlock_fas_trylock(x) + +CK_ELIDE_PROTOTYPE(ck_spinlock, ck_spinlock_t, + ck_spinlock_locked, ck_spinlock_lock, + ck_spinlock_locked, ck_spinlock_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock, ck_spinlock_t, + ck_spinlock_locked, ck_spinlock_trylock) + +#endif /* CK_SPINLOCK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_spinlock.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_stack.h =================================================================== --- vendor-sys/ck/20161230/include/ck_stack.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_stack.h (revision 310844) @@ -0,0 +1,357 @@ +/* + * Copyright 2009-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_STACK_H +#define CK_STACK_H + +#include +#include +#include +#include + +struct ck_stack_entry { + struct ck_stack_entry *next; +}; +typedef struct ck_stack_entry ck_stack_entry_t; + +struct ck_stack { + struct ck_stack_entry *head; + char *generation CK_CC_PACKED; +} CK_CC_ALIASED; +typedef struct ck_stack ck_stack_t; + +#define CK_STACK_INITIALIZER { NULL, NULL } + +#ifndef CK_F_STACK_PUSH_UPMC +#define CK_F_STACK_PUSH_UPMC +/* + * Stack producer operation safe for multiple unique producers and multiple consumers. + */ +CK_CC_INLINE static void +ck_stack_push_upmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + stack = ck_pr_load_ptr(&target->head); + entry->next = stack; + ck_pr_fence_store(); + + while (ck_pr_cas_ptr_value(&target->head, stack, entry, &stack) == false) { + entry->next = stack; + ck_pr_fence_store(); + } + + return; +} +#endif /* CK_F_STACK_PUSH_UPMC */ + +#ifndef CK_F_STACK_TRYPUSH_UPMC +#define CK_F_STACK_TRYPUSH_UPMC +/* + * Stack producer operation for multiple unique producers and multiple consumers. + * Returns true on success and false on failure. + */ +CK_CC_INLINE static bool +ck_stack_trypush_upmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + stack = ck_pr_load_ptr(&target->head); + entry->next = stack; + ck_pr_fence_store(); + + return ck_pr_cas_ptr(&target->head, stack, entry); +} +#endif /* CK_F_STACK_TRYPUSH_UPMC */ + +#ifndef CK_F_STACK_POP_UPMC +#define CK_F_STACK_POP_UPMC +/* + * Stack consumer operation safe for multiple unique producers and multiple consumers. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_upmc(struct ck_stack *target) +{ + struct ck_stack_entry *entry, *next; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return NULL; + + ck_pr_fence_load(); + next = entry->next; + while (ck_pr_cas_ptr_value(&target->head, entry, next, &entry) == false) { + if (entry == NULL) + break; + + ck_pr_fence_load(); + next = entry->next; + } + + return entry; +} +#endif + +#ifndef CK_F_STACK_TRYPOP_UPMC +#define CK_F_STACK_TRYPOP_UPMC +/* + * Stack production operation for multiple unique producers and multiple consumers. + * Returns true on success and false on failure. The value pointed to by the second + * argument is set to a valid ck_stack_entry_t reference if true is returned. If + * false is returned, then the value pointed to by the second argument is undefined. + */ +CK_CC_INLINE static bool +ck_stack_trypop_upmc(struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_load_ptr(&target->head); + if (entry == NULL) + return false; + + ck_pr_fence_load(); + if (ck_pr_cas_ptr(&target->head, entry, entry->next) == true) { + *r = entry; + return true; + } + + return false; +} +#endif /* CK_F_STACK_TRYPOP_UPMC */ + +#ifndef CK_F_STACK_BATCH_POP_UPMC +#define CK_F_STACK_BATCH_POP_UPMC +/* + * Pop all items off the stack. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_upmc(struct ck_stack *target) +{ + struct ck_stack_entry *entry; + + entry = ck_pr_fas_ptr(&target->head, NULL); + ck_pr_fence_load(); + return entry; +} +#endif /* CK_F_STACK_BATCH_POP_UPMC */ + +#ifndef CK_F_STACK_PUSH_MPMC +#define CK_F_STACK_PUSH_MPMC +/* + * Stack producer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static void +ck_stack_push_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + ck_stack_push_upmc(target, entry); + return; +} +#endif /* CK_F_STACK_PUSH_MPMC */ + +#ifndef CK_F_STACK_TRYPUSH_MPMC +#define CK_F_STACK_TRYPUSH_MPMC +/* + * Stack producer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static bool +ck_stack_trypush_mpmc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + return ck_stack_trypush_upmc(target, entry); +} +#endif /* CK_F_STACK_TRYPUSH_MPMC */ + +#ifdef CK_F_PR_CAS_PTR_2_VALUE +#ifndef CK_F_STACK_POP_MPMC +#define CK_F_STACK_POP_MPMC +/* + * Stack consumer operation safe for multiple producers and multiple consumers. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_mpmc(struct ck_stack *target) +{ + struct ck_stack original, update; + + original.generation = ck_pr_load_ptr(&target->generation); + ck_pr_fence_load(); + original.head = ck_pr_load_ptr(&target->head); + if (original.head == NULL) + return NULL; + + /* Order with respect to next pointer. */ + ck_pr_fence_load(); + + update.generation = original.generation + 1; + update.head = original.head->next; + + while (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == false) { + if (original.head == NULL) + return NULL; + + update.generation = original.generation + 1; + + /* Order with respect to next pointer. */ + ck_pr_fence_load(); + update.head = original.head->next; + } + + return original.head; +} +#endif /* CK_F_STACK_POP_MPMC */ + +#ifndef CK_F_STACK_TRYPOP_MPMC +#define CK_F_STACK_TRYPOP_MPMC +CK_CC_INLINE static bool +ck_stack_trypop_mpmc(struct ck_stack *target, struct ck_stack_entry **r) +{ + struct ck_stack original, update; + + original.generation = ck_pr_load_ptr(&target->generation); + ck_pr_fence_load(); + original.head = ck_pr_load_ptr(&target->head); + if (original.head == NULL) + return false; + + update.generation = original.generation + 1; + ck_pr_fence_load(); + update.head = original.head->next; + + if (ck_pr_cas_ptr_2_value(target, &original, &update, &original) == true) { + *r = original.head; + return true; + } + + return false; +} +#endif /* CK_F_STACK_TRYPOP_MPMC */ +#endif /* CK_F_PR_CAS_PTR_2_VALUE */ + +#ifndef CK_F_STACK_BATCH_POP_MPMC +#define CK_F_STACK_BATCH_POP_MPMC +/* + * This is equivalent to the UP/MC version as NULL does not need a + * a generation count. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_mpmc(struct ck_stack *target) +{ + + return ck_stack_batch_pop_upmc(target); +} +#endif /* CK_F_STACK_BATCH_POP_MPMC */ + +#ifndef CK_F_STACK_PUSH_MPNC +#define CK_F_STACK_PUSH_MPNC +/* + * Stack producer operation safe with no concurrent consumers. + */ +CK_CC_INLINE static void +ck_stack_push_mpnc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + struct ck_stack_entry *stack; + + entry->next = NULL; + ck_pr_fence_store_atomic(); + stack = ck_pr_fas_ptr(&target->head, entry); + ck_pr_store_ptr(&entry->next, stack); + ck_pr_fence_store(); + + return; +} +#endif /* CK_F_STACK_PUSH_MPNC */ + +/* + * Stack producer operation for single producer and no concurrent consumers. + */ +CK_CC_INLINE static void +ck_stack_push_spnc(struct ck_stack *target, struct ck_stack_entry *entry) +{ + + entry->next = target->head; + target->head = entry; + return; +} + +/* + * Stack consumer operation for no concurrent producers and single consumer. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_pop_npsc(struct ck_stack *target) +{ + struct ck_stack_entry *n; + + if (target->head == NULL) + return NULL; + + n = target->head; + target->head = n->next; + + return n; +} + +/* + * Pop all items off a stack. + */ +CK_CC_INLINE static struct ck_stack_entry * +ck_stack_batch_pop_npsc(struct ck_stack *target) +{ + struct ck_stack_entry *n; + + n = target->head; + target->head = NULL; + + return n; +} + +/* + * Stack initialization function. Guarantees initialization across processors. + */ +CK_CC_INLINE static void +ck_stack_init(struct ck_stack *stack) +{ + + stack->head = NULL; + stack->generation = NULL; + return; +} + +/* Defines a container_of functions for */ +#define CK_STACK_CONTAINER(T, M, N) CK_CC_CONTAINER(ck_stack_entry_t, T, M, N) + +#define CK_STACK_ISEMPTY(m) ((m)->head == NULL) +#define CK_STACK_FIRST(s) ((s)->head) +#define CK_STACK_NEXT(m) ((m)->next) +#define CK_STACK_FOREACH(stack, entry) \ + for ((entry) = CK_STACK_FIRST(stack); \ + (entry) != NULL; \ + (entry) = CK_STACK_NEXT(entry)) +#define CK_STACK_FOREACH_SAFE(stack, entry, T) \ + for ((entry) = CK_STACK_FIRST(stack); \ + (entry) != NULL && ((T) = (entry)->next, 1); \ + (entry) = (T)) + +#endif /* CK_STACK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_stack.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_stdbool.h =================================================================== --- vendor-sys/ck/20161230/include/ck_stdbool.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_stdbool.h (revision 310844) @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include +#else +#include +#endif Property changes on: vendor-sys/ck/20161230/include/ck_stdbool.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_stddef.h =================================================================== --- vendor-sys/ck/20161230/include/ck_stddef.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_stddef.h (revision 310844) @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include +#else +#include +#endif Property changes on: vendor-sys/ck/20161230/include/ck_stddef.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_stdint.h =================================================================== --- vendor-sys/ck/20161230/include/ck_stdint.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_stdint.h (revision 310844) @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__linux__) && defined(__KERNEL__) +#include +#include +#elif defined(__FreeBSD__) && defined(_KERNEL) +#include +#else +#include +#endif /* __linux__ && __KERNEL__ */ Property changes on: vendor-sys/ck/20161230/include/ck_stdint.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_stdlib.h =================================================================== --- vendor-sys/ck/20161230/include/ck_stdlib.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_stdlib.h (revision 310844) @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include +#else +#include +#endif Property changes on: vendor-sys/ck/20161230/include/ck_stdlib.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_string.h =================================================================== --- vendor-sys/ck/20161230/include/ck_string.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_string.h (revision 310844) @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Olivier Houchard. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(__FreeBSD__) && defined(_KERNEL) +#include +#else +#include +#endif Property changes on: vendor-sys/ck/20161230/include/ck_string.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_swlock.h =================================================================== --- vendor-sys/ck/20161230/include/ck_swlock.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_swlock.h (revision 310844) @@ -0,0 +1,218 @@ +/* + * Copyright 2014 Jaidev Sridhar. + * Copyright 2014 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SWLOCK_H +#define CK_SWLOCK_H + +#include +#include +#include +#include +#include + +struct ck_swlock { + uint32_t value; +}; +typedef struct ck_swlock ck_swlock_t; + +#define CK_SWLOCK_INITIALIZER {0} +#define CK_SWLOCK_WRITER_BIT (1UL << 31) +#define CK_SWLOCK_LATCH_BIT (1UL << 30) +#define CK_SWLOCK_WRITER_MASK (CK_SWLOCK_LATCH_BIT | CK_SWLOCK_WRITER_BIT) +#define CK_SWLOCK_READER_MASK (UINT32_MAX ^ CK_SWLOCK_WRITER_MASK) + +CK_CC_INLINE static void +ck_swlock_init(struct ck_swlock *rw) +{ + + rw->value = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_swlock_write_unlock(ck_swlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_and_32(&rw->value, CK_SWLOCK_READER_MASK); + return; +} + +CK_CC_INLINE static bool +ck_swlock_locked_writer(ck_swlock_t *rw) +{ + bool r; + + r = ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT; + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_swlock_write_downgrade(ck_swlock_t *rw) +{ + + ck_pr_inc_32(&rw->value); + ck_swlock_write_unlock(rw); + return; +} + +CK_CC_INLINE static bool +ck_swlock_locked(ck_swlock_t *rw) +{ + bool r; + + r = ck_pr_load_32(&rw->value); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static bool +ck_swlock_write_trylock(ck_swlock_t *rw) +{ + bool r; + + r = ck_pr_cas_32(&rw->value, 0, CK_SWLOCK_WRITER_BIT); + ck_pr_fence_lock(); + return r; +} + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_write, ck_swlock_t, + ck_swlock_locked, ck_swlock_write_trylock) + +CK_CC_INLINE static void +ck_swlock_write_lock(ck_swlock_t *rw) +{ + + ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT); + while (ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_swlock_write_latch(ck_swlock_t *rw) +{ + + /* Publish intent to acquire lock. */ + ck_pr_or_32(&rw->value, CK_SWLOCK_WRITER_BIT); + + /* Stall until readers have seen the writer and cleared. */ + while (ck_pr_cas_32(&rw->value, CK_SWLOCK_WRITER_BIT, + CK_SWLOCK_WRITER_MASK) == false) { + do { + ck_pr_stall(); + } while (ck_pr_load_32(&rw->value) != CK_SWLOCK_WRITER_BIT); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_swlock_write_unlatch(ck_swlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_store_32(&rw->value, 0); + return; +} + +CK_ELIDE_PROTOTYPE(ck_swlock_write, ck_swlock_t, + ck_swlock_locked, ck_swlock_write_lock, + ck_swlock_locked_writer, ck_swlock_write_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_swlock_read, ck_swlock_t, + ck_swlock_locked_writer, ck_swlock_read_trylock) + +CK_CC_INLINE static bool +ck_swlock_read_trylock(ck_swlock_t *rw) +{ + uint32_t l = ck_pr_load_32(&rw->value); + + if (l & CK_SWLOCK_WRITER_BIT) + return false; + + l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK; + if (l == CK_SWLOCK_WRITER_BIT) + ck_pr_dec_32(&rw->value); + + ck_pr_fence_lock(); + return l == 0; +} + +CK_CC_INLINE static void +ck_swlock_read_lock(ck_swlock_t *rw) +{ + uint32_t l; + + for (;;) { + while (ck_pr_load_32(&rw->value) & CK_SWLOCK_WRITER_BIT) + ck_pr_stall(); + + l = ck_pr_faa_32(&rw->value, 1) & CK_SWLOCK_WRITER_MASK; + if (l == 0) + break; + + /* + * If the latch bit has not been set, then the writer would + * have observed the reader and will wait to completion of + * read-side critical section. + */ + if (l == CK_SWLOCK_WRITER_BIT) + ck_pr_dec_32(&rw->value); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static bool +ck_swlock_locked_reader(ck_swlock_t *rw) +{ + + ck_pr_fence_load(); + return ck_pr_load_32(&rw->value) & CK_SWLOCK_READER_MASK; +} + +CK_CC_INLINE static void +ck_swlock_read_unlock(ck_swlock_t *rw) +{ + + ck_pr_fence_unlock(); + ck_pr_dec_32(&rw->value); + return; +} + +CK_ELIDE_PROTOTYPE(ck_swlock_read, ck_swlock_t, + ck_swlock_locked_writer, ck_swlock_read_lock, + ck_swlock_locked_reader, ck_swlock_read_unlock) + +#endif /* CK_SWLOCK_H */ Property changes on: vendor-sys/ck/20161230/include/ck_swlock.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/ck_tflock.h =================================================================== --- vendor-sys/ck/20161230/include/ck_tflock.h (nonexistent) +++ vendor-sys/ck/20161230/include/ck_tflock.h (revision 310844) @@ -0,0 +1,136 @@ +/* + * Copyright 2014 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_TFLOCK_TICKET_H +#define CK_TFLOCK_TICKET_H + +/* + * This is an implementation of task-fair locks derived from the work + * described in: + * John M. Mellor-Crummey and Michael L. Scott. 1991. + * Scalable reader-writer synchronization for shared-memory + * multiprocessors. SIGPLAN Not. 26, 7 (April 1991), 106-113. + */ + +#include +#include + +struct ck_tflock_ticket { + uint32_t request; + uint32_t completion; +}; +typedef struct ck_tflock_ticket ck_tflock_ticket_t; + +#define CK_TFLOCK_TICKET_INITIALIZER { 0, 0 } + +#define CK_TFLOCK_TICKET_RC_INCR 0x10000U /* Read-side increment. */ +#define CK_TFLOCK_TICKET_WC_INCR 0x1U /* Write-side increment. */ +#define CK_TFLOCK_TICKET_W_MASK 0xffffU /* Write-side mask. */ +#define CK_TFLOCK_TICKET_WC_TOPMSK 0x8000U /* Write clear mask for overflow. */ +#define CK_TFLOCK_TICKET_RC_TOPMSK 0x80000000U /* Read clear mask for overflow. */ + +CK_CC_INLINE static uint32_t +ck_tflock_ticket_fca_32(uint32_t *target, uint32_t mask, uint32_t delta) +{ + uint32_t snapshot = ck_pr_load_32(target); + uint32_t goal; + + for (;;) { + goal = (snapshot & ~mask) + delta; + if (ck_pr_cas_32_value(target, snapshot, goal, &snapshot) == true) + break; + + ck_pr_stall(); + } + + return snapshot; +} + +CK_CC_INLINE static void +ck_tflock_ticket_init(struct ck_tflock_ticket *pf) +{ + + pf->request = pf->completion = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_write_lock(struct ck_tflock_ticket *lock) +{ + uint32_t previous; + + previous = ck_tflock_ticket_fca_32(&lock->request, CK_TFLOCK_TICKET_WC_TOPMSK, + CK_TFLOCK_TICKET_WC_INCR); + ck_pr_fence_atomic_load(); + while (ck_pr_load_32(&lock->completion) != previous) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_write_unlock(struct ck_tflock_ticket *lock) +{ + + ck_pr_fence_unlock(); + ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_WC_TOPMSK, + CK_TFLOCK_TICKET_WC_INCR); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_read_lock(struct ck_tflock_ticket *lock) +{ + uint32_t previous; + + previous = ck_tflock_ticket_fca_32(&lock->request, + CK_TFLOCK_TICKET_RC_TOPMSK, CK_TFLOCK_TICKET_RC_INCR) & + CK_TFLOCK_TICKET_W_MASK; + + ck_pr_fence_atomic_load(); + + while ((ck_pr_load_32(&lock->completion) & + CK_TFLOCK_TICKET_W_MASK) != previous) { + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_tflock_ticket_read_unlock(struct ck_tflock_ticket *lock) +{ + + ck_pr_fence_unlock(); + ck_tflock_ticket_fca_32(&lock->completion, CK_TFLOCK_TICKET_RC_TOPMSK, + CK_TFLOCK_TICKET_RC_INCR); + return; +} + +#endif /* CK_TFLOCK_TICKET_H */ Property changes on: vendor-sys/ck/20161230/include/ck_tflock.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/anderson.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/anderson.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/anderson.h (revision 310844) @@ -0,0 +1,167 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_ANDERSON_H +#define CK_SPINLOCK_ANDERSON_H + +#include +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_ANDERSON +#define CK_F_SPINLOCK_ANDERSON +/* + * This is an implementation of Anderson's array-based queuing lock. + */ +struct ck_spinlock_anderson_thread { + unsigned int locked; + unsigned int position; +}; +typedef struct ck_spinlock_anderson_thread ck_spinlock_anderson_thread_t; + +struct ck_spinlock_anderson { + struct ck_spinlock_anderson_thread *slots; + unsigned int count; + unsigned int wrap; + unsigned int mask; + char pad[CK_MD_CACHELINE - sizeof(unsigned int) * 3 - sizeof(void *)]; + unsigned int next; +}; +typedef struct ck_spinlock_anderson ck_spinlock_anderson_t; + +CK_CC_INLINE static void +ck_spinlock_anderson_init(struct ck_spinlock_anderson *lock, + struct ck_spinlock_anderson_thread *slots, + unsigned int count) +{ + unsigned int i; + + slots[0].locked = false; + slots[0].position = 0; + for (i = 1; i < count; i++) { + slots[i].locked = true; + slots[i].position = i; + } + + lock->slots = slots; + lock->count = count; + lock->mask = count - 1; + lock->next = 0; + + /* + * If the number of threads is not a power of two then compute + * appropriate wrap-around value in the case of next slot counter + * overflow. + */ + if (count & (count - 1)) + lock->wrap = (UINT_MAX % count) + 1; + else + lock->wrap = 0; + + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_anderson_locked(struct ck_spinlock_anderson *lock) +{ + unsigned int position; + bool r; + + position = ck_pr_load_uint(&lock->next) & lock->mask; + r = ck_pr_load_uint(&lock->slots[position].locked); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_anderson_lock(struct ck_spinlock_anderson *lock, + struct ck_spinlock_anderson_thread **slot) +{ + unsigned int position, next; + unsigned int count = lock->count; + + /* + * If count is not a power of 2, then it is possible for an overflow + * to reallocate beginning slots to more than one thread. To avoid this + * use a compare-and-swap. + */ + if (lock->wrap != 0) { + position = ck_pr_load_uint(&lock->next); + + do { + if (position == UINT_MAX) + next = lock->wrap; + else + next = position + 1; + } while (ck_pr_cas_uint_value(&lock->next, position, + next, &position) == false); + + position %= count; + } else { + position = ck_pr_faa_uint(&lock->next, 1); + position &= lock->mask; + } + + /* Serialize with respect to previous thread's store. */ + ck_pr_fence_load(); + + /* + * Spin until slot is marked as unlocked. First slot is initialized to + * false. + */ + while (ck_pr_load_uint(&lock->slots[position].locked) == true) + ck_pr_stall(); + + /* Prepare slot for potential re-use by another thread. */ + ck_pr_store_uint(&lock->slots[position].locked, true); + ck_pr_fence_lock(); + + *slot = lock->slots + position; + return; +} + +CK_CC_INLINE static void +ck_spinlock_anderson_unlock(struct ck_spinlock_anderson *lock, + struct ck_spinlock_anderson_thread *slot) +{ + unsigned int position; + + ck_pr_fence_unlock(); + + /* Mark next slot as available. */ + if (lock->wrap == 0) + position = (slot->position + 1) & lock->mask; + else + position = (slot->position + 1) % lock->count; + + ck_pr_store_uint(&lock->slots[position].locked, false); + return; +} +#endif /* CK_F_SPINLOCK_ANDERSON */ +#endif /* CK_SPINLOCK_ANDERSON_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/anderson.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/cas.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/cas.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/cas.h (revision 310844) @@ -0,0 +1,119 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_CAS_H +#define CK_SPINLOCK_CAS_H + +#include +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_CAS +#define CK_F_SPINLOCK_CAS +/* + * This is a simple CACAS (TATAS) spinlock implementation. + */ +struct ck_spinlock_cas { + unsigned int value; +}; +typedef struct ck_spinlock_cas ck_spinlock_cas_t; + +#define CK_SPINLOCK_CAS_INITIALIZER {false} + +CK_CC_INLINE static void +ck_spinlock_cas_init(struct ck_spinlock_cas *lock) +{ + + lock->value = false; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_cas_trylock(struct ck_spinlock_cas *lock) +{ + unsigned int value; + + value = ck_pr_fas_uint(&lock->value, true); + ck_pr_fence_lock(); + return !value; +} + +CK_CC_INLINE static bool +ck_spinlock_cas_locked(struct ck_spinlock_cas *lock) +{ + bool r = ck_pr_load_uint(&lock->value); + + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_cas_lock(struct ck_spinlock_cas *lock) +{ + + while (ck_pr_cas_uint(&lock->value, false, true) == false) { + while (ck_pr_load_uint(&lock->value) == true) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_cas_lock_eb(struct ck_spinlock_cas *lock) +{ + ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; + + while (ck_pr_cas_uint(&lock->value, false, true) == false) + ck_backoff_eb(&backoff); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_cas_unlock(struct ck_spinlock_cas *lock) +{ + + /* Set lock state to unlocked. */ + ck_pr_fence_unlock(); + ck_pr_store_uint(&lock->value, false); + return; +} + +CK_ELIDE_PROTOTYPE(ck_spinlock_cas, ck_spinlock_cas_t, + ck_spinlock_cas_locked, ck_spinlock_cas_lock, + ck_spinlock_cas_locked, ck_spinlock_cas_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_cas, ck_spinlock_cas_t, + ck_spinlock_cas_locked, ck_spinlock_cas_trylock) + +#endif /* CK_F_SPINLOCK_CAS */ +#endif /* CK_SPINLOCK_CAS_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/cas.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/clh.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/clh.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/clh.h (revision 310844) @@ -0,0 +1,122 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_CLH_H +#define CK_SPINLOCK_CLH_H + +#include +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_CLH +#define CK_F_SPINLOCK_CLH + +struct ck_spinlock_clh { + unsigned int wait; + struct ck_spinlock_clh *previous; +}; +typedef struct ck_spinlock_clh ck_spinlock_clh_t; + +CK_CC_INLINE static void +ck_spinlock_clh_init(struct ck_spinlock_clh **lock, struct ck_spinlock_clh *unowned) +{ + + unowned->previous = NULL; + unowned->wait = false; + *lock = unowned; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_clh_locked(struct ck_spinlock_clh **queue) +{ + struct ck_spinlock_clh *head; + bool r; + + head = ck_pr_load_ptr(queue); + r = ck_pr_load_uint(&head->wait); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_clh_lock(struct ck_spinlock_clh **queue, struct ck_spinlock_clh *thread) +{ + struct ck_spinlock_clh *previous; + + /* Indicate to the next thread on queue that they will have to block. */ + thread->wait = true; + ck_pr_fence_store_atomic(); + + /* + * Mark current request as last request. Save reference to previous + * request. + */ + previous = ck_pr_fas_ptr(queue, thread); + thread->previous = previous; + + /* Wait until previous thread is done with lock. */ + ck_pr_fence_load(); + while (ck_pr_load_uint(&previous->wait) == true) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_clh_unlock(struct ck_spinlock_clh **thread) +{ + struct ck_spinlock_clh *previous; + + /* + * If there are waiters, they are spinning on the current node wait + * flag. The flag is cleared so that the successor may complete an + * acquisition. If the caller is pre-empted then the predecessor field + * may be updated by a successor's lock operation. In order to avoid + * this, save a copy of the predecessor before setting the flag. + */ + previous = thread[0]->previous; + + /* + * We have to pay this cost anyways, use it as a compiler barrier too. + */ + ck_pr_fence_unlock(); + ck_pr_store_uint(&(*thread)->wait, false); + + /* + * Predecessor is guaranteed not to be spinning on previous request, + * so update caller to use previous structure. This allows successor + * all the time in the world to successfully read updated wait flag. + */ + *thread = previous; + return; +} +#endif /* CK_F_SPINLOCK_CLH */ +#endif /* CK_SPINLOCK_CLH_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/clh.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/dec.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/dec.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/dec.h (revision 310844) @@ -0,0 +1,143 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_DEC_H +#define CK_SPINLOCK_DEC_H + +#include +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_DEC +#define CK_F_SPINLOCK_DEC +/* + * This is similar to the CACAS lock but makes use of an atomic decrement + * operation to check if the lock value was decremented to 0 from 1. The + * idea is that a decrement operation is cheaper than a compare-and-swap. + */ +struct ck_spinlock_dec { + unsigned int value; +}; +typedef struct ck_spinlock_dec ck_spinlock_dec_t; + +#define CK_SPINLOCK_DEC_INITIALIZER {1} + +CK_CC_INLINE static void +ck_spinlock_dec_init(struct ck_spinlock_dec *lock) +{ + + lock->value = 1; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_dec_trylock(struct ck_spinlock_dec *lock) +{ + unsigned int value; + + value = ck_pr_fas_uint(&lock->value, 0); + ck_pr_fence_lock(); + return value == 1; +} + +CK_CC_INLINE static bool +ck_spinlock_dec_locked(struct ck_spinlock_dec *lock) +{ + bool r; + + r = ck_pr_load_uint(&lock->value) != 1; + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_dec_lock(struct ck_spinlock_dec *lock) +{ + bool r; + + for (;;) { + /* + * Only one thread is guaranteed to decrement lock to 0. + * Overflow must be protected against. No more than + * UINT_MAX lock requests can happen while the lock is held. + */ + ck_pr_dec_uint_zero(&lock->value, &r); + if (r == true) + break; + + /* Load value without generating write cycles. */ + while (ck_pr_load_uint(&lock->value) != 1) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_dec_lock_eb(struct ck_spinlock_dec *lock) +{ + ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; + bool r; + + for (;;) { + ck_pr_dec_uint_zero(&lock->value, &r); + if (r == true) + break; + + ck_backoff_eb(&backoff); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_dec_unlock(struct ck_spinlock_dec *lock) +{ + + ck_pr_fence_unlock(); + + /* + * Unconditionally set lock value to 1 so someone can decrement lock + * to 0. + */ + ck_pr_store_uint(&lock->value, 1); + return; +} + +CK_ELIDE_PROTOTYPE(ck_spinlock_dec, ck_spinlock_dec_t, + ck_spinlock_dec_locked, ck_spinlock_dec_lock, + ck_spinlock_dec_locked, ck_spinlock_dec_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_dec, ck_spinlock_dec_t, + ck_spinlock_dec_locked, ck_spinlock_dec_trylock) + +#endif /* CK_F_SPINLOCK_DEC */ +#endif /* CK_SPINLOCK_DEC_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/dec.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/fas.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/fas.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/fas.h (revision 310844) @@ -0,0 +1,118 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_FAS_H +#define CK_SPINLOCK_FAS_H + +#include +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_FAS +#define CK_F_SPINLOCK_FAS + +struct ck_spinlock_fas { + unsigned int value; +}; +typedef struct ck_spinlock_fas ck_spinlock_fas_t; + +#define CK_SPINLOCK_FAS_INITIALIZER {false} + +CK_CC_INLINE static void +ck_spinlock_fas_init(struct ck_spinlock_fas *lock) +{ + + lock->value = false; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_fas_trylock(struct ck_spinlock_fas *lock) +{ + bool value; + + value = ck_pr_fas_uint(&lock->value, true); + ck_pr_fence_lock(); + + return !value; +} + +CK_CC_INLINE static bool +ck_spinlock_fas_locked(struct ck_spinlock_fas *lock) +{ + bool r; + + r = ck_pr_load_uint(&lock->value); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_fas_lock(struct ck_spinlock_fas *lock) +{ + + while (ck_pr_fas_uint(&lock->value, true) == true) { + while (ck_pr_load_uint(&lock->value) == true) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_fas_lock_eb(struct ck_spinlock_fas *lock) +{ + ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; + + while (ck_pr_fas_uint(&lock->value, true) == true) + ck_backoff_eb(&backoff); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_fas_unlock(struct ck_spinlock_fas *lock) +{ + + ck_pr_fence_unlock(); + ck_pr_store_uint(&lock->value, false); + return; +} + +CK_ELIDE_PROTOTYPE(ck_spinlock_fas, ck_spinlock_fas_t, + ck_spinlock_fas_locked, ck_spinlock_fas_lock, + ck_spinlock_fas_locked, ck_spinlock_fas_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_fas, ck_spinlock_fas_t, + ck_spinlock_fas_locked, ck_spinlock_fas_trylock) + +#endif /* CK_F_SPINLOCK_FAS */ +#endif /* CK_SPINLOCK_FAS_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/fas.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/hclh.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/hclh.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/hclh.h (revision 310844) @@ -0,0 +1,145 @@ +/* + * Copyright 2013-2015 Olivier Houchard + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_HCLH_H +#define CK_SPINLOCK_HCLH_H + +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_HCLH +#define CK_F_SPINLOCK_HCLH +struct ck_spinlock_hclh { + unsigned int wait; + unsigned int splice; + int cluster_id; + struct ck_spinlock_hclh *previous; +}; +typedef struct ck_spinlock_hclh ck_spinlock_hclh_t; + +CK_CC_INLINE static void +ck_spinlock_hclh_init(struct ck_spinlock_hclh **lock, + struct ck_spinlock_hclh *unowned, + int cluster_id) +{ + + unowned->previous = NULL; + unowned->wait = false; + unowned->splice = false; + unowned->cluster_id = cluster_id; + *lock = unowned; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_hclh_locked(struct ck_spinlock_hclh **queue) +{ + struct ck_spinlock_hclh *head; + bool r; + + head = ck_pr_load_ptr(queue); + r = ck_pr_load_uint(&head->wait); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_hclh_lock(struct ck_spinlock_hclh **glob_queue, + struct ck_spinlock_hclh **local_queue, + struct ck_spinlock_hclh *thread) +{ + struct ck_spinlock_hclh *previous, *local_tail; + + /* Indicate to the next thread on queue that they will have to block. */ + thread->wait = true; + thread->splice = false; + thread->cluster_id = (*local_queue)->cluster_id; + + /* Serialize with respect to update of local queue. */ + ck_pr_fence_store_atomic(); + + /* Mark current request as last request. Save reference to previous request. */ + previous = ck_pr_fas_ptr(local_queue, thread); + thread->previous = previous; + + /* Wait until previous thread from the local queue is done with lock. */ + ck_pr_fence_load(); + if (previous->previous != NULL && + previous->cluster_id == thread->cluster_id) { + while (ck_pr_load_uint(&previous->wait) == true) + ck_pr_stall(); + + /* We're head of the global queue, we're done */ + if (ck_pr_load_uint(&previous->splice) == false) + return; + } + + /* Now we need to splice the local queue into the global queue. */ + local_tail = ck_pr_load_ptr(local_queue); + previous = ck_pr_fas_ptr(glob_queue, local_tail); + + ck_pr_store_uint(&local_tail->splice, true); + + /* Wait until previous thread from the global queue is done with lock. */ + while (ck_pr_load_uint(&previous->wait) == true) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_hclh_unlock(struct ck_spinlock_hclh **thread) +{ + struct ck_spinlock_hclh *previous; + + /* + * If there are waiters, they are spinning on the current node wait + * flag. The flag is cleared so that the successor may complete an + * acquisition. If the caller is pre-empted then the predecessor field + * may be updated by a successor's lock operation. In order to avoid + * this, save a copy of the predecessor before setting the flag. + */ + previous = thread[0]->previous; + + /* We have to pay this cost anyways, use it as a compiler barrier too. */ + ck_pr_fence_unlock(); + ck_pr_store_uint(&(*thread)->wait, false); + + /* + * Predecessor is guaranteed not to be spinning on previous request, + * so update caller to use previous structure. This allows successor + * all the time in the world to successfully read updated wait flag. + */ + *thread = previous; + return; +} +#endif /* CK_F_SPINLOCK_HCLH */ +#endif /* CK_SPINLOCK_HCLH_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/hclh.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/mcs.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/mcs.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/mcs.h (revision 310844) @@ -0,0 +1,155 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_MCS_H +#define CK_SPINLOCK_MCS_H + +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_MCS +#define CK_F_SPINLOCK_MCS + +struct ck_spinlock_mcs { + unsigned int locked; + struct ck_spinlock_mcs *next; +}; +typedef struct ck_spinlock_mcs * ck_spinlock_mcs_t; +typedef struct ck_spinlock_mcs ck_spinlock_mcs_context_t; + +#define CK_SPINLOCK_MCS_INITIALIZER (NULL) + +CK_CC_INLINE static void +ck_spinlock_mcs_init(struct ck_spinlock_mcs **queue) +{ + + *queue = NULL; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_mcs_trylock(struct ck_spinlock_mcs **queue, + struct ck_spinlock_mcs *node) +{ + bool r; + + node->locked = true; + node->next = NULL; + ck_pr_fence_store_atomic(); + + r = ck_pr_cas_ptr(queue, NULL, node); + ck_pr_fence_lock(); + return r; +} + +CK_CC_INLINE static bool +ck_spinlock_mcs_locked(struct ck_spinlock_mcs **queue) +{ + bool r; + + r = ck_pr_load_ptr(queue) != NULL; + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_mcs_lock(struct ck_spinlock_mcs **queue, + struct ck_spinlock_mcs *node) +{ + struct ck_spinlock_mcs *previous; + + /* + * In the case that there is a successor, let them know they must + * wait for us to unlock. + */ + node->locked = true; + node->next = NULL; + ck_pr_fence_store_atomic(); + + /* + * Swap current tail with current lock request. If the swap operation + * returns NULL, it means the queue was empty. If the queue was empty, + * then the operation is complete. + */ + previous = ck_pr_fas_ptr(queue, node); + if (previous != NULL) { + /* + * Let the previous lock holder know that we are waiting on + * them. + */ + ck_pr_store_ptr(&previous->next, node); + while (ck_pr_load_uint(&node->locked) == true) + ck_pr_stall(); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_mcs_unlock(struct ck_spinlock_mcs **queue, + struct ck_spinlock_mcs *node) +{ + struct ck_spinlock_mcs *next; + + ck_pr_fence_unlock(); + + next = ck_pr_load_ptr(&node->next); + if (next == NULL) { + /* + * If there is no request following us then it is a possibilty + * that we are the current tail. In this case, we may just + * mark the spinlock queue as empty. + */ + if (ck_pr_load_ptr(queue) == node && + ck_pr_cas_ptr(queue, node, NULL) == true) { + return; + } + + /* + * If the node is not the current tail then a lock operation + * is in-progress. In this case, busy-wait until the queue is + * in a consistent state to wake up the incoming lock + * request. + */ + for (;;) { + next = ck_pr_load_ptr(&node->next); + if (next != NULL) + break; + + ck_pr_stall(); + } + } + + /* Allow the next lock operation to complete. */ + ck_pr_store_uint(&next->locked, false); + return; +} +#endif /* CK_F_SPINLOCK_MCS */ +#endif /* CK_SPINLOCK_MCS_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/mcs.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/include/spinlock/ticket.h =================================================================== --- vendor-sys/ck/20161230/include/spinlock/ticket.h (nonexistent) +++ vendor-sys/ck/20161230/include/spinlock/ticket.h (revision 310844) @@ -0,0 +1,296 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_SPINLOCK_TICKET_H +#define CK_SPINLOCK_TICKET_H + +#include +#include +#include +#include +#include +#include + +#ifndef CK_F_SPINLOCK_TICKET +#define CK_F_SPINLOCK_TICKET +/* + * If 16-bit or 32-bit increment is supported, implement support for + * trylock functionality on availability of 32-bit or 64-bit fetch-and-add + * and compare-and-swap. This code path is only applied to x86*. + */ +#if defined(CK_MD_TSO) && (defined(__x86__) || defined(__x86_64__)) +#if defined(CK_F_PR_FAA_32) && defined(CK_F_PR_INC_16) && defined(CK_F_PR_CAS_32) +#define CK_SPINLOCK_TICKET_TYPE uint32_t +#define CK_SPINLOCK_TICKET_TYPE_BASE uint16_t +#define CK_SPINLOCK_TICKET_INC(x) ck_pr_inc_16(x) +#define CK_SPINLOCK_TICKET_CAS(x, y, z) ck_pr_cas_32(x, y, z) +#define CK_SPINLOCK_TICKET_FAA(x, y) ck_pr_faa_32(x, y) +#define CK_SPINLOCK_TICKET_LOAD(x) ck_pr_load_32(x) +#define CK_SPINLOCK_TICKET_INCREMENT (0x00010000UL) +#define CK_SPINLOCK_TICKET_MASK (0xFFFFUL) +#define CK_SPINLOCK_TICKET_SHIFT (16) +#elif defined(CK_F_PR_FAA_64) && defined(CK_F_PR_INC_32) && defined(CK_F_PR_CAS_64) +#define CK_SPINLOCK_TICKET_TYPE uint64_t +#define CK_SPINLOCK_TICKET_TYPE_BASE uint32_t +#define CK_SPINLOCK_TICKET_INC(x) ck_pr_inc_32(x) +#define CK_SPINLOCK_TICKET_CAS(x, y, z) ck_pr_cas_64(x, y, z) +#define CK_SPINLOCK_TICKET_FAA(x, y) ck_pr_faa_64(x, y) +#define CK_SPINLOCK_TICKET_LOAD(x) ck_pr_load_64(x) +#define CK_SPINLOCK_TICKET_INCREMENT (0x0000000100000000ULL) +#define CK_SPINLOCK_TICKET_MASK (0xFFFFFFFFULL) +#define CK_SPINLOCK_TICKET_SHIFT (32) +#endif +#endif /* CK_MD_TSO */ + +#if defined(CK_SPINLOCK_TICKET_TYPE) +#define CK_F_SPINLOCK_TICKET_TRYLOCK + +struct ck_spinlock_ticket { + CK_SPINLOCK_TICKET_TYPE value; +}; +typedef struct ck_spinlock_ticket ck_spinlock_ticket_t; +#define CK_SPINLOCK_TICKET_INITIALIZER { .value = 0 } + +CK_CC_INLINE static void +ck_spinlock_ticket_init(struct ck_spinlock_ticket *ticket) +{ + + ticket->value = 0; + ck_pr_barrier(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_ticket_locked(struct ck_spinlock_ticket *ticket) +{ + CK_SPINLOCK_TICKET_TYPE request, position; + + request = CK_SPINLOCK_TICKET_LOAD(&ticket->value); + position = request & CK_SPINLOCK_TICKET_MASK; + request >>= CK_SPINLOCK_TICKET_SHIFT; + + ck_pr_fence_acquire(); + return request != position; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock(struct ck_spinlock_ticket *ticket) +{ + CK_SPINLOCK_TICKET_TYPE request, position; + + /* Get our ticket number and set next ticket number. */ + request = CK_SPINLOCK_TICKET_FAA(&ticket->value, + CK_SPINLOCK_TICKET_INCREMENT); + + position = request & CK_SPINLOCK_TICKET_MASK; + request >>= CK_SPINLOCK_TICKET_SHIFT; + + while (request != position) { + ck_pr_stall(); + position = CK_SPINLOCK_TICKET_LOAD(&ticket->value) & + CK_SPINLOCK_TICKET_MASK; + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c) +{ + CK_SPINLOCK_TICKET_TYPE request, position; + ck_backoff_t backoff; + + /* Get our ticket number and set next ticket number. */ + request = CK_SPINLOCK_TICKET_FAA(&ticket->value, + CK_SPINLOCK_TICKET_INCREMENT); + + position = request & CK_SPINLOCK_TICKET_MASK; + request >>= CK_SPINLOCK_TICKET_SHIFT; + + while (request != position) { + ck_pr_stall(); + position = CK_SPINLOCK_TICKET_LOAD(&ticket->value) & + CK_SPINLOCK_TICKET_MASK; + + backoff = (request - position) & CK_SPINLOCK_TICKET_MASK; + backoff <<= c; + ck_backoff_eb(&backoff); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static bool +ck_spinlock_ticket_trylock(struct ck_spinlock_ticket *ticket) +{ + CK_SPINLOCK_TICKET_TYPE snapshot, request, position; + + snapshot = CK_SPINLOCK_TICKET_LOAD(&ticket->value); + position = snapshot & CK_SPINLOCK_TICKET_MASK; + request = snapshot >> CK_SPINLOCK_TICKET_SHIFT; + + if (position != request) + return false; + + if (CK_SPINLOCK_TICKET_CAS(&ticket->value, + snapshot, snapshot + CK_SPINLOCK_TICKET_INCREMENT) == false) { + return false; + } + + ck_pr_fence_lock(); + return true; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_unlock(struct ck_spinlock_ticket *ticket) +{ + + ck_pr_fence_unlock(); + CK_SPINLOCK_TICKET_INC((CK_SPINLOCK_TICKET_TYPE_BASE *)(void *)&ticket->value); + return; +} + +#undef CK_SPINLOCK_TICKET_TYPE +#undef CK_SPINLOCK_TICKET_TYPE_BASE +#undef CK_SPINLOCK_TICKET_INC +#undef CK_SPINLOCK_TICKET_FAA +#undef CK_SPINLOCK_TICKET_LOAD +#undef CK_SPINLOCK_TICKET_INCREMENT +#undef CK_SPINLOCK_TICKET_MASK +#undef CK_SPINLOCK_TICKET_SHIFT +#else +/* + * MESI benefits from cacheline padding between next and current. This avoids + * invalidation of current from the cache due to incoming lock requests. + */ +struct ck_spinlock_ticket { + unsigned int next; + unsigned int position; +}; +typedef struct ck_spinlock_ticket ck_spinlock_ticket_t; + +#define CK_SPINLOCK_TICKET_INITIALIZER {.next = 0, .position = 0} + +CK_CC_INLINE static void +ck_spinlock_ticket_init(struct ck_spinlock_ticket *ticket) +{ + + ticket->next = 0; + ticket->position = 0; + ck_pr_barrier(); + + return; +} + +CK_CC_INLINE static bool +ck_spinlock_ticket_locked(struct ck_spinlock_ticket *ticket) +{ + bool r; + + r = ck_pr_load_uint(&ticket->position) != + ck_pr_load_uint(&ticket->next); + ck_pr_fence_acquire(); + return r; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock(struct ck_spinlock_ticket *ticket) +{ + unsigned int request; + + /* Get our ticket number and set next ticket number. */ + request = ck_pr_faa_uint(&ticket->next, 1); + + /* + * Busy-wait until our ticket number is current. + * We can get away without a fence here assuming + * our position counter does not overflow. + */ + while (ck_pr_load_uint(&ticket->position) != request) + ck_pr_stall(); + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_lock_pb(struct ck_spinlock_ticket *ticket, unsigned int c) +{ + ck_backoff_t backoff; + unsigned int request, position; + + request = ck_pr_faa_uint(&ticket->next, 1); + + for (;;) { + position = ck_pr_load_uint(&ticket->position); + if (position == request) + break; + + backoff = request - position; + backoff <<= c; + + /* + * Ideally, back-off from generating cache traffic for at least + * the amount of time necessary for the number of pending lock + * acquisition and relinquish operations (assuming an empty + * critical section). + */ + ck_backoff_eb(&backoff); + } + + ck_pr_fence_lock(); + return; +} + +CK_CC_INLINE static void +ck_spinlock_ticket_unlock(struct ck_spinlock_ticket *ticket) +{ + unsigned int update; + + ck_pr_fence_unlock(); + + /* + * Update current ticket value so next lock request can proceed. + * Overflow behavior is assumed to be roll-over, in which case, + * it is only an issue if there are 2^32 pending lock requests. + */ + update = ck_pr_load_uint(&ticket->position); + ck_pr_store_uint(&ticket->position, update + 1); + return; +} +#endif /* !CK_F_SPINLOCK_TICKET_TRYLOCK */ + +CK_ELIDE_PROTOTYPE(ck_spinlock_ticket, ck_spinlock_ticket_t, + ck_spinlock_ticket_locked, ck_spinlock_ticket_lock, + ck_spinlock_ticket_locked, ck_spinlock_ticket_unlock) + +CK_ELIDE_TRYLOCK_PROTOTYPE(ck_spinlock_ticket, ck_spinlock_ticket_t, + ck_spinlock_ticket_locked, ck_spinlock_ticket_trylock) + +#endif /* CK_F_SPINLOCK_TICKET */ +#endif /* CK_SPINLOCK_TICKET_H */ Property changes on: vendor-sys/ck/20161230/include/spinlock/ticket.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_array.c =================================================================== --- vendor-sys/ck/20161230/src/ck_array.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_array.c (revision 310844) @@ -0,0 +1,240 @@ +/* + * Copyright 2013-2015 Samy Al Bahra + * Copyright 2013-2014 AppNexus, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +static struct _ck_array * +ck_array_create(struct ck_malloc *allocator, unsigned int length) +{ + struct _ck_array *active; + + active = allocator->malloc(sizeof(struct _ck_array) + sizeof(void *) * length); + if (active == NULL) + return NULL; + + active->n_committed = 0; + active->length = length; + + return active; +} + +bool +ck_array_init(struct ck_array *array, unsigned int mode, struct ck_malloc *allocator, unsigned int length) +{ + struct _ck_array *active; + + (void)mode; + + if (allocator->realloc == NULL || + allocator->malloc == NULL || + allocator->free == NULL || + length == 0) + return false; + + active = ck_array_create(allocator, length); + if (active == NULL) + return false; + + array->n_entries = 0; + array->allocator = allocator; + array->active = active; + array->transaction = NULL; + return true; +} + +bool +ck_array_put(struct ck_array *array, void *value) +{ + struct _ck_array *target; + unsigned int size; + + /* + * If no transaction copy has been necessary, attempt to do in-place + * modification of the array. + */ + if (array->transaction == NULL) { + target = array->active; + + if (array->n_entries == target->length) { + size = target->length << 1; + + target = array->allocator->realloc(target, + sizeof(struct _ck_array) + sizeof(void *) * array->n_entries, + sizeof(struct _ck_array) + sizeof(void *) * size, + true); + + if (target == NULL) + return false; + + ck_pr_store_uint(&target->length, size); + + /* Serialize with respect to contents. */ + ck_pr_fence_store(); + ck_pr_store_ptr(&array->active, target); + } + + target->values[array->n_entries++] = value; + return true; + } + + target = array->transaction; + if (array->n_entries == target->length) { + size = target->length << 1; + + target = array->allocator->realloc(target, + sizeof(struct _ck_array) + sizeof(void *) * array->n_entries, + sizeof(struct _ck_array) + sizeof(void *) * size, + true); + + if (target == NULL) + return false; + + target->length = size; + array->transaction = target; + } + + target->values[array->n_entries++] = value; + return false; +} + +int +ck_array_put_unique(struct ck_array *array, void *value) +{ + unsigned int i, limit; + void **v; + + limit = array->n_entries; + if (array->transaction != NULL) { + v = array->transaction->values; + } else { + v = array->active->values; + } + + for (i = 0; i < limit; i++) { + if (v[i] == value) + return 1; + } + + return -!ck_array_put(array, value); +} + +bool +ck_array_remove(struct ck_array *array, void *value) +{ + struct _ck_array *target; + unsigned int i; + + if (array->transaction != NULL) { + target = array->transaction; + + for (i = 0; i < array->n_entries; i++) { + if (target->values[i] == value) { + target->values[i] = target->values[--array->n_entries]; + return true; + } + } + + return false; + } + + target = array->active; + + for (i = 0; i < array->n_entries; i++) { + if (target->values[i] == value) + break; + } + + if (i == array->n_entries) + return false; + + /* If there are pending additions, immediately eliminate the operation. */ + if (target->n_committed != array->n_entries) { + ck_pr_store_ptr(&target->values[i], target->values[--array->n_entries]); + return true; + } + + /* + * The assumption is that these allocations are small to begin with. + * If there is no immediate opportunity for transaction, allocate a + * transactional array which will be applied upon commit time. + */ + target = ck_array_create(array->allocator, array->n_entries); + if (target == NULL) + return false; + + memcpy(target->values, array->active->values, sizeof(void *) * array->n_entries); + target->length = array->n_entries; + target->n_committed = array->n_entries; + target->values[i] = target->values[--array->n_entries]; + + array->transaction = target; + return true; +} + +bool +ck_array_commit(ck_array_t *array) +{ + struct _ck_array *m = array->transaction; + + if (m != NULL) { + struct _ck_array *p; + + m->n_committed = array->n_entries; + ck_pr_fence_store(); + p = array->active; + ck_pr_store_ptr(&array->active, m); + array->allocator->free(p, sizeof(struct _ck_array) + + p->length * sizeof(void *), true); + array->transaction = NULL; + + return true; + } + + ck_pr_fence_store(); + ck_pr_store_uint(&array->active->n_committed, array->n_entries); + return true; +} + +void +ck_array_deinit(struct ck_array *array, bool defer) +{ + + array->allocator->free(array->active, + sizeof(struct _ck_array) + sizeof(void *) * array->active->length, defer); + + if (array->transaction != NULL) { + array->allocator->free(array->transaction, + sizeof(struct _ck_array) + sizeof(void *) * array->transaction->length, defer); + } + + array->transaction = array->active = NULL; + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_array.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_barrier_centralized.c =================================================================== --- vendor-sys/ck/20161230/src/ck_barrier_centralized.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_barrier_centralized.c (revision 310844) @@ -0,0 +1,59 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +void +ck_barrier_centralized(struct ck_barrier_centralized *barrier, + struct ck_barrier_centralized_state *state, + unsigned int n_threads) +{ + unsigned int sense, value; + + /* + * Every execution context has a sense associated with it. + * This sense is reversed when the barrier is entered. Every + * thread will spin on the global sense until the last thread + * reverses it. + */ + sense = state->sense = ~state->sense; + value = ck_pr_faa_uint(&barrier->value, 1); + if (value == n_threads - 1) { + ck_pr_store_uint(&barrier->value, 0); + ck_pr_fence_memory(); + ck_pr_store_uint(&barrier->sense, sense); + return; + } + + ck_pr_fence_atomic_load(); + while (sense != ck_pr_load_uint(&barrier->sense)) + ck_pr_stall(); + + ck_pr_fence_acquire(); + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_barrier_centralized.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_barrier_combining.c =================================================================== --- vendor-sys/ck/20161230/src/ck_barrier_combining.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_barrier_combining.c (revision 310844) @@ -0,0 +1,207 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +struct ck_barrier_combining_queue { + struct ck_barrier_combining_group *head; + struct ck_barrier_combining_group *tail; +}; + +CK_CC_INLINE static struct ck_barrier_combining_group * +ck_barrier_combining_queue_dequeue(struct ck_barrier_combining_queue *queue) +{ + struct ck_barrier_combining_group *front = NULL; + + if (queue->head != NULL) { + front = queue->head; + queue->head = queue->head->next; + } + + return front; +} + +CK_CC_INLINE static void +ck_barrier_combining_insert(struct ck_barrier_combining_group *parent, + struct ck_barrier_combining_group *tnode, + struct ck_barrier_combining_group **child) +{ + + *child = tnode; + tnode->parent = parent; + + /* + * After inserting, we must increment the parent group's count for + * number of threads expected to reach it; otherwise, the + * barrier may end prematurely. + */ + parent->k++; + return; +} + +/* + * This implementation of software combining tree barriers + * uses level order traversal to insert new thread groups + * into the barrier's tree. We use a queue to implement this + * traversal. + */ +CK_CC_INLINE static void +ck_barrier_combining_queue_enqueue(struct ck_barrier_combining_queue *queue, + struct ck_barrier_combining_group *node_value) +{ + + node_value->next = NULL; + if (queue->head == NULL) { + queue->head = queue->tail = node_value; + return; + } + + queue->tail->next = node_value; + queue->tail = node_value; + + return; +} + + +void +ck_barrier_combining_group_init(struct ck_barrier_combining *root, + struct ck_barrier_combining_group *tnode, + unsigned int nthr) +{ + struct ck_barrier_combining_group *node; + struct ck_barrier_combining_queue queue; + + queue.head = queue.tail = NULL; + + tnode->k = nthr; + tnode->count = 0; + tnode->sense = 0; + tnode->left = tnode->right = NULL; + + /* + * Finds the first available node for linkage into the combining + * tree. The use of a spinlock is excusable as this is a one-time + * initialization cost. + */ + ck_spinlock_fas_lock(&root->mutex); + ck_barrier_combining_queue_enqueue(&queue, root->root); + while (queue.head != NULL) { + node = ck_barrier_combining_queue_dequeue(&queue); + + /* If the left child is free, link the group there. */ + if (node->left == NULL) { + ck_barrier_combining_insert(node, tnode, &node->left); + goto leave; + } + + /* If the right child is free, link the group there. */ + if (node->right == NULL) { + ck_barrier_combining_insert(node, tnode, &node->right); + goto leave; + } + + /* + * If unsuccessful, try inserting as a child of the children of the + * current node. + */ + ck_barrier_combining_queue_enqueue(&queue, node->left); + ck_barrier_combining_queue_enqueue(&queue, node->right); + } + +leave: + ck_spinlock_fas_unlock(&root->mutex); + return; +} + +void +ck_barrier_combining_init(struct ck_barrier_combining *root, + struct ck_barrier_combining_group *init_root) +{ + + init_root->k = 0; + init_root->count = 0; + init_root->sense = 0; + init_root->parent = init_root->left = init_root->right = NULL; + ck_spinlock_fas_init(&root->mutex); + root->root = init_root; + return; +} + +static void +ck_barrier_combining_aux(struct ck_barrier_combining *barrier, + struct ck_barrier_combining_group *tnode, + unsigned int sense) +{ + + /* + * If this is the last thread in the group, it moves on to the parent group. + * Otherwise, it spins on this group's sense. + */ + if (ck_pr_faa_uint(&tnode->count, 1) == tnode->k - 1) { + /* + * If we are and will be the last thread entering the barrier for the + * current group then signal the parent group if one exists. + */ + if (tnode->parent != NULL) + ck_barrier_combining_aux(barrier, tnode->parent, sense); + + /* + * Once the thread returns from its parent(s), it reinitializes the group's + * arrival count and signals other threads to continue by flipping the group + * sense. Order of these operations is not important since we assume a static + * number of threads are members of a barrier for the lifetime of the barrier. + * Since count is explicitly reinitialized, it is guaranteed that at any point + * tnode->count is equivalent to tnode->k if and only if that many threads + * are at the barrier. + */ + ck_pr_store_uint(&tnode->count, 0); + ck_pr_fence_store(); + ck_pr_store_uint(&tnode->sense, ~tnode->sense); + } else { + ck_pr_fence_memory(); + while (sense != ck_pr_load_uint(&tnode->sense)) + ck_pr_stall(); + } + + return; +} + +void +ck_barrier_combining(struct ck_barrier_combining *barrier, + struct ck_barrier_combining_group *tnode, + struct ck_barrier_combining_state *state) +{ + + ck_barrier_combining_aux(barrier, tnode, state->sense); + + /* Reverse the execution context's sense for the next barrier. */ + state->sense = ~state->sense; + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_barrier_combining.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_barrier_dissemination.c =================================================================== --- vendor-sys/ck/20161230/src/ck_barrier_dissemination.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_barrier_dissemination.c (revision 310844) @@ -0,0 +1,130 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "ck_internal.h" + +void +ck_barrier_dissemination_init(struct ck_barrier_dissemination *barrier, + struct ck_barrier_dissemination_flag **barrier_internal, + unsigned int nthr) +{ + unsigned int i, j, k, size, offset; + bool p = nthr & (nthr - 1); + + barrier->nthr = nthr; + barrier->size = size = ck_internal_log(ck_internal_power_2(nthr)); + ck_pr_store_uint(&barrier->tid, 0); + + for (i = 0; i < nthr; ++i) { + barrier[i].flags[0] = barrier_internal[i]; + barrier[i].flags[1] = barrier_internal[i] + size; + } + + for (i = 0; i < nthr; ++i) { + for (k = 0, offset = 1; k < size; ++k, offset <<= 1) { + /* + * Determine the thread's partner, j, for the current round, k. + * Partners are chosen such that by the completion of the barrier, + * every thread has been directly (having one of its flag set) or + * indirectly (having one of its partners's flags set) signaled + * by every other thread in the barrier. + */ + if (p == false) + j = (i + offset) & (nthr - 1); + else + j = (i + offset) % nthr; + + /* Set the thread's partner for round k. */ + barrier[i].flags[0][k].pflag = &barrier[j].flags[0][k].tflag; + barrier[i].flags[1][k].pflag = &barrier[j].flags[1][k].tflag; + + /* Set the thread's flags to false. */ + barrier[i].flags[0][k].tflag = barrier[i].flags[1][k].tflag = 0; + } + } + + return; +} + +void +ck_barrier_dissemination_subscribe(struct ck_barrier_dissemination *barrier, + struct ck_barrier_dissemination_state *state) +{ + + state->parity = 0; + state->sense = ~0; + state->tid = ck_pr_faa_uint(&barrier->tid, 1); + return; +} + +unsigned int +ck_barrier_dissemination_size(unsigned int nthr) +{ + + return (ck_internal_log(ck_internal_power_2(nthr)) << 1); +} + +void +ck_barrier_dissemination(struct ck_barrier_dissemination *barrier, + struct ck_barrier_dissemination_state *state) +{ + unsigned int i; + unsigned int size = barrier->size; + + for (i = 0; i < size; ++i) { + unsigned int *pflag, *tflag; + + pflag = barrier[state->tid].flags[state->parity][i].pflag; + tflag = &barrier[state->tid].flags[state->parity][i].tflag; + + /* Unblock current partner. */ + ck_pr_store_uint(pflag, state->sense); + + /* Wait until some other thread unblocks this one. */ + while (ck_pr_load_uint(tflag) != state->sense) + ck_pr_stall(); + } + + /* + * Dissemination barriers use two sets of flags to prevent race conditions + * between successive calls to the barrier. Parity indicates which set will + * be used for the next barrier. They also use a sense reversal technique + * to avoid re-initialization of the flags for every two calls to the barrier. + */ + if (state->parity == 1) + state->sense = ~state->sense; + + state->parity = 1 - state->parity; + + ck_pr_fence_acquire(); + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_barrier_dissemination.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_barrier_mcs.c =================================================================== --- vendor-sys/ck/20161230/src/ck_barrier_mcs.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_barrier_mcs.c (revision 310844) @@ -0,0 +1,141 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +void +ck_barrier_mcs_init(struct ck_barrier_mcs *barrier, unsigned int nthr) +{ + unsigned int i, j; + + ck_pr_store_uint(&barrier->tid, 0); + + for (i = 0; i < nthr; ++i) { + for (j = 0; j < 4; ++j) { + /* + * If there are still threads that don't have parents, + * add it as a child. + */ + barrier[i].havechild[j] = ((i << 2) + j < nthr - 1) ? ~0 : 0; + + /* + * childnotready is initialized to havechild to ensure + * a thread does not wait for a child that does not exist. + */ + barrier[i].childnotready[j] = barrier[i].havechild[j]; + } + + /* The root thread does not have a parent. */ + barrier[i].parent = (i == 0) ? + &barrier[i].dummy : + &barrier[(i - 1) >> 2].childnotready[(i - 1) & 3]; + + /* Leaf threads do not have any children. */ + barrier[i].children[0] = ((i << 1) + 1 >= nthr) ? + &barrier[i].dummy : + &barrier[(i << 1) + 1].parentsense; + + barrier[i].children[1] = ((i << 1) + 2 >= nthr) ? + &barrier[i].dummy : + &barrier[(i << 1) + 2].parentsense; + + barrier[i].parentsense = 0; + } + + return; +} + +void +ck_barrier_mcs_subscribe(struct ck_barrier_mcs *barrier, struct ck_barrier_mcs_state *state) +{ + + state->sense = ~0; + state->vpid = ck_pr_faa_uint(&barrier->tid, 1); + return; +} + +CK_CC_INLINE static bool +ck_barrier_mcs_check_children(unsigned int *childnotready) +{ + + if (ck_pr_load_uint(&childnotready[0]) != 0) + return false; + if (ck_pr_load_uint(&childnotready[1]) != 0) + return false; + if (ck_pr_load_uint(&childnotready[2]) != 0) + return false; + if (ck_pr_load_uint(&childnotready[3]) != 0) + return false; + + return true; +} + +CK_CC_INLINE static void +ck_barrier_mcs_reinitialize_children(struct ck_barrier_mcs *node) +{ + + ck_pr_store_uint(&node->childnotready[0], node->havechild[0]); + ck_pr_store_uint(&node->childnotready[1], node->havechild[1]); + ck_pr_store_uint(&node->childnotready[2], node->havechild[2]); + ck_pr_store_uint(&node->childnotready[3], node->havechild[3]); + return; +} + +void +ck_barrier_mcs(struct ck_barrier_mcs *barrier, + struct ck_barrier_mcs_state *state) +{ + + /* + * Wait until all children have reached the barrier and are done waiting + * for their children. + */ + while (ck_barrier_mcs_check_children(barrier[state->vpid].childnotready) == false) + ck_pr_stall(); + + /* Reinitialize for next barrier. */ + ck_barrier_mcs_reinitialize_children(&barrier[state->vpid]); + + /* Inform parent thread and its children have arrived at the barrier. */ + ck_pr_store_uint(barrier[state->vpid].parent, 0); + + /* Wait until parent indicates all threads have arrived at the barrier. */ + if (state->vpid != 0) { + while (ck_pr_load_uint(&barrier[state->vpid].parentsense) != state->sense) + ck_pr_stall(); + } + + /* Inform children of successful barrier. */ + ck_pr_store_uint(barrier[state->vpid].children[0], state->sense); + ck_pr_store_uint(barrier[state->vpid].children[1], state->sense); + state->sense = ~state->sense; + ck_pr_fence_memory(); + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_barrier_mcs.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_barrier_tournament.c =================================================================== --- vendor-sys/ck/20161230/src/ck_barrier_tournament.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_barrier_tournament.c (revision 310844) @@ -0,0 +1,184 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +#include "ck_internal.h" + +/* + * This is a tournament barrier implementation. Threads are statically + * assigned roles to perform for each round of the barrier. Winners + * move on to the next round, while losers spin in their current rounds + * on their own flags. During the last round, the champion of the tournament + * sets the last flag that begins the wakeup process. + */ + +enum { + CK_BARRIER_TOURNAMENT_BYE, + CK_BARRIER_TOURNAMENT_CHAMPION, + CK_BARRIER_TOURNAMENT_DROPOUT, + CK_BARRIER_TOURNAMENT_LOSER, + CK_BARRIER_TOURNAMENT_WINNER +}; + +void +ck_barrier_tournament_subscribe(struct ck_barrier_tournament *barrier, + struct ck_barrier_tournament_state *state) +{ + + state->sense = ~0; + state->vpid = ck_pr_faa_uint(&barrier->tid, 1); + return; +} + +void +ck_barrier_tournament_init(struct ck_barrier_tournament *barrier, + struct ck_barrier_tournament_round **rounds, + unsigned int nthr) +{ + unsigned int i, k, size, twok, twokm1, imod2k; + + ck_pr_store_uint(&barrier->tid, 0); + barrier->size = size = ck_barrier_tournament_size(nthr); + + for (i = 0; i < nthr; ++i) { + /* The first role is always CK_BARRIER_TOURNAMENT_DROPOUT. */ + rounds[i][0].flag = 0; + rounds[i][0].role = CK_BARRIER_TOURNAMENT_DROPOUT; + for (k = 1, twok = 2, twokm1 = 1; k < size; ++k, twokm1 = twok, twok <<= 1) { + rounds[i][k].flag = 0; + + imod2k = i & (twok - 1); + if (imod2k == 0) { + if ((i + twokm1 < nthr) && (twok < nthr)) + rounds[i][k].role = CK_BARRIER_TOURNAMENT_WINNER; + else if (i + twokm1 >= nthr) + rounds[i][k].role = CK_BARRIER_TOURNAMENT_BYE; + } + + if (imod2k == twokm1) + rounds[i][k].role = CK_BARRIER_TOURNAMENT_LOSER; + else if ((i == 0) && (twok >= nthr)) + rounds[i][k].role = CK_BARRIER_TOURNAMENT_CHAMPION; + + if (rounds[i][k].role == CK_BARRIER_TOURNAMENT_LOSER) + rounds[i][k].opponent = &rounds[i - twokm1][k].flag; + else if (rounds[i][k].role == CK_BARRIER_TOURNAMENT_WINNER || + rounds[i][k].role == CK_BARRIER_TOURNAMENT_CHAMPION) + rounds[i][k].opponent = &rounds[i + twokm1][k].flag; + } + } + + ck_pr_store_ptr(&barrier->rounds, rounds); + return; +} + +unsigned int +ck_barrier_tournament_size(unsigned int nthr) +{ + + return (ck_internal_log(ck_internal_power_2(nthr)) + 1); +} + +void +ck_barrier_tournament(struct ck_barrier_tournament *barrier, + struct ck_barrier_tournament_state *state) +{ + struct ck_barrier_tournament_round **rounds = ck_pr_load_ptr(&barrier->rounds); + int round = 1; + + if (barrier->size == 1) + return; + + for (;; ++round) { + switch (rounds[state->vpid][round].role) { + case CK_BARRIER_TOURNAMENT_BYE: + break; + case CK_BARRIER_TOURNAMENT_CHAMPION: + /* + * The CK_BARRIER_TOURNAMENT_CHAMPION waits until it wins the tournament; it then + * sets the final flag before the wakeup phase of the barrier. + */ + while (ck_pr_load_uint(&rounds[state->vpid][round].flag) != state->sense) + ck_pr_stall(); + + ck_pr_store_uint(rounds[state->vpid][round].opponent, state->sense); + goto wakeup; + case CK_BARRIER_TOURNAMENT_DROPOUT: + /* NOTREACHED */ + break; + case CK_BARRIER_TOURNAMENT_LOSER: + /* + * CK_BARRIER_TOURNAMENT_LOSERs set the flags of their opponents and wait until + * their opponents release them after the tournament is over. + */ + ck_pr_store_uint(rounds[state->vpid][round].opponent, state->sense); + while (ck_pr_load_uint(&rounds[state->vpid][round].flag) != state->sense) + ck_pr_stall(); + + goto wakeup; + case CK_BARRIER_TOURNAMENT_WINNER: + /* + * CK_BARRIER_TOURNAMENT_WINNERs wait until their current opponent sets their flag; they then + * continue to the next round of the tournament. + */ + while (ck_pr_load_uint(&rounds[state->vpid][round].flag) != state->sense) + ck_pr_stall(); + break; + } + } + +wakeup: + for (round -= 1 ;; --round) { + switch (rounds[state->vpid][round].role) { + case CK_BARRIER_TOURNAMENT_BYE: + break; + case CK_BARRIER_TOURNAMENT_CHAMPION: + /* NOTREACHED */ + break; + case CK_BARRIER_TOURNAMENT_DROPOUT: + goto leave; + break; + case CK_BARRIER_TOURNAMENT_LOSER: + /* NOTREACHED */ + break; + case CK_BARRIER_TOURNAMENT_WINNER: + /* + * Winners inform their old opponents the tournament is over + * by setting their flags. + */ + ck_pr_store_uint(rounds[state->vpid][round].opponent, state->sense); + break; + } + } + +leave: + ck_pr_fence_memory(); + state->sense = ~state->sense; + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_barrier_tournament.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_epoch.c =================================================================== --- vendor-sys/ck/20161230/src/ck_epoch.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_epoch.c (revision 310844) @@ -0,0 +1,544 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * The implementation here is inspired from the work described in: + * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University + * of Cambridge Computing Laboratory. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* + * Only three distinct values are used for reclamation, but reclamation occurs + * at e+2 rather than e+1. Any thread in a "critical section" would have + * acquired some snapshot (e) of the global epoch value (e_g) and set an active + * flag. Any hazardous references will only occur after a full memory barrier. + * For example, assume an initial e_g value of 1, e value of 0 and active value + * of 0. + * + * ck_epoch_begin(...) + * e = e_g + * active = 1 + * memory_barrier(); + * + * Any serialized reads may observe e = 0 or e = 1 with active = 0, or e = 0 or + * e = 1 with active = 1. The e_g value can only go from 1 to 2 if every thread + * has already observed the value of "1" (or the value we are incrementing + * from). This guarantees us that for any given value e_g, any threads with-in + * critical sections (referred to as "active" threads from here on) would have + * an e value of e_g-1 or e_g. This also means that hazardous references may be + * shared in both e_g-1 and e_g even if they are logically deleted in e_g. + * + * For example, assume all threads have an e value of e_g. Another thread may + * increment to e_g to e_g+1. Older threads may have a reference to an object + * which is only deleted in e_g+1. It could be that reader threads are + * executing some hash table look-ups, while some other writer thread (which + * causes epoch counter tick) actually deletes the same items that reader + * threads are looking up (this writer thread having an e value of e_g+1). + * This is possible if the writer thread re-observes the epoch after the + * counter tick. + * + * Psuedo-code for writer: + * ck_epoch_begin() + * ht_delete(x) + * ck_epoch_end() + * ck_epoch_begin() + * ht_delete(x) + * ck_epoch_end() + * + * Psuedo-code for reader: + * for (;;) { + * x = ht_lookup(x) + * ck_pr_inc(&x->value); + * } + * + * Of course, it is also possible for references logically deleted at e_g-1 to + * still be accessed at e_g as threads are "active" at the same time + * (real-world time) mutating shared objects. + * + * Now, if the epoch counter is ticked to e_g+1, then no new hazardous + * references could exist to objects logically deleted at e_g-1. The reason for + * this is that at e_g+1, all epoch read-side critical sections started at + * e_g-1 must have been completed. If any epoch read-side critical sections at + * e_g-1 were still active, then we would never increment to e_g+1 (active != 0 + * ^ e != e_g). Additionally, e_g may still have hazardous references to + * objects logically deleted at e_g-1 which means objects logically deleted at + * e_g-1 cannot be deleted at e_g+1 unless all threads have observed e_g+1 + * (since it is valid for active threads to be at e_g and threads at e_g still + * require safe memory accesses). + * + * However, at e_g+2, all active threads must be either at e_g+1 or e_g+2. + * Though e_g+2 may share hazardous references with e_g+1, and e_g+1 shares + * hazardous references to e_g, no active threads are at e_g or e_g-1. This + * means no hazardous references could exist to objects deleted at e_g-1 (at + * e_g+2). + * + * To summarize these important points, + * 1) Active threads will always have a value of e_g or e_g-1. + * 2) Items that are logically deleted e_g or e_g-1 cannot be physically + * deleted. + * 3) Objects logically deleted at e_g-1 can be physically destroyed at e_g+2 + * or at e_g+1 if no threads are at e_g. + * + * Last but not least, if we are at e_g+2, then no active thread is at e_g + * which means it is safe to apply modulo-3 arithmetic to e_g value in order to + * re-use e_g to represent the e_g+3 state. This means it is sufficient to + * represent e_g using only the values 0, 1 or 2. Every time a thread re-visits + * a e_g (which can be determined with a non-empty deferral list) it can assume + * objects in the e_g deferral list involved at least three e_g transitions and + * are thus, safe, for physical deletion. + * + * Blocking semantics for epoch reclamation have additional restrictions. + * Though we only require three deferral lists, reasonable blocking semantics + * must be able to more gracefully handle bursty write work-loads which could + * easily cause e_g wrap-around if modulo-3 arithmetic is used. This allows for + * easy-to-trigger live-lock situations. The work-around to this is to not + * apply modulo arithmetic to e_g but only to deferral list indexing. + */ +#define CK_EPOCH_GRACE 3U + +enum { + CK_EPOCH_STATE_USED = 0, + CK_EPOCH_STATE_FREE = 1 +}; + +CK_STACK_CONTAINER(struct ck_epoch_record, record_next, + ck_epoch_record_container) +CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, + ck_epoch_entry_container) + +#define CK_EPOCH_SENSE_MASK (CK_EPOCH_SENSE - 1) + +void +_ck_epoch_delref(struct ck_epoch_record *record, + struct ck_epoch_section *section) +{ + struct ck_epoch_ref *current, *other; + unsigned int i = section->bucket; + + current = &record->local.bucket[i]; + current->count--; + + if (current->count > 0) + return; + + /* + * If the current bucket no longer has any references, then + * determine whether we have already transitioned into a newer + * epoch. If so, then make sure to update our shared snapshot + * to allow for forward progress. + * + * If no other active bucket exists, then the record will go + * inactive in order to allow for forward progress. + */ + other = &record->local.bucket[(i + 1) & + CK_EPOCH_SENSE_MASK]; + if (other->count > 0 && + ((int)(current->epoch - other->epoch) < 0)) { + /* + * The other epoch value is actually the newest, + * transition to it. + */ + ck_pr_store_uint(&record->epoch, other->epoch); + } + + return; +} + +void +_ck_epoch_addref(struct ck_epoch_record *record, + struct ck_epoch_section *section) +{ + struct ck_epoch *global = record->global; + struct ck_epoch_ref *ref; + unsigned int epoch, i; + + epoch = ck_pr_load_uint(&global->epoch); + i = epoch & CK_EPOCH_SENSE_MASK; + ref = &record->local.bucket[i]; + + if (ref->count++ == 0) { +#ifndef CK_MD_TSO + struct ck_epoch_ref *previous; + + /* + * The system has already ticked. If another non-zero bucket + * exists, make sure to order our observations with respect + * to it. Otherwise, it is possible to acquire a reference + * from the previous epoch generation. + * + * On TSO architectures, the monoticity of the global counter + * and load-{store, load} ordering are sufficient to guarantee + * this ordering. + */ + previous = &record->local.bucket[(i + 1) & + CK_EPOCH_SENSE_MASK]; + if (previous->count > 0) + ck_pr_fence_acqrel(); +#endif /* !CK_MD_TSO */ + + /* + * If this is this is a new reference into the current + * bucket then cache the associated epoch value. + */ + ref->epoch = epoch; + } + + section->bucket = i; + return; +} + +void +ck_epoch_init(struct ck_epoch *global) +{ + + ck_stack_init(&global->records); + global->epoch = 1; + global->n_free = 0; + ck_pr_fence_store(); + return; +} + +struct ck_epoch_record * +ck_epoch_recycle(struct ck_epoch *global) +{ + struct ck_epoch_record *record; + ck_stack_entry_t *cursor; + unsigned int state; + + if (ck_pr_load_uint(&global->n_free) == 0) + return NULL; + + CK_STACK_FOREACH(&global->records, cursor) { + record = ck_epoch_record_container(cursor); + + if (ck_pr_load_uint(&record->state) == CK_EPOCH_STATE_FREE) { + /* Serialize with respect to deferral list clean-up. */ + ck_pr_fence_load(); + state = ck_pr_fas_uint(&record->state, + CK_EPOCH_STATE_USED); + if (state == CK_EPOCH_STATE_FREE) { + ck_pr_dec_uint(&global->n_free); + return record; + } + } + } + + return NULL; +} + +void +ck_epoch_register(struct ck_epoch *global, struct ck_epoch_record *record) +{ + size_t i; + + record->global = global; + record->state = CK_EPOCH_STATE_USED; + record->active = 0; + record->epoch = 0; + record->n_dispatch = 0; + record->n_peak = 0; + record->n_pending = 0; + memset(&record->local, 0, sizeof record->local); + + for (i = 0; i < CK_EPOCH_LENGTH; i++) + ck_stack_init(&record->pending[i]); + + ck_pr_fence_store(); + ck_stack_push_upmc(&global->records, &record->record_next); + return; +} + +void +ck_epoch_unregister(struct ck_epoch_record *record) +{ + struct ck_epoch *global = record->global; + size_t i; + + record->active = 0; + record->epoch = 0; + record->n_dispatch = 0; + record->n_peak = 0; + record->n_pending = 0; + memset(&record->local, 0, sizeof record->local); + + for (i = 0; i < CK_EPOCH_LENGTH; i++) + ck_stack_init(&record->pending[i]); + + ck_pr_fence_store(); + ck_pr_store_uint(&record->state, CK_EPOCH_STATE_FREE); + ck_pr_inc_uint(&global->n_free); + return; +} + +static struct ck_epoch_record * +ck_epoch_scan(struct ck_epoch *global, + struct ck_epoch_record *cr, + unsigned int epoch, + bool *af) +{ + ck_stack_entry_t *cursor; + + *af = false; + if (cr == NULL) { + cursor = CK_STACK_FIRST(&global->records); + } else { + cursor = &cr->record_next; + } + + while (cursor != NULL) { + unsigned int state, active; + + cr = ck_epoch_record_container(cursor); + + state = ck_pr_load_uint(&cr->state); + if (state & CK_EPOCH_STATE_FREE) { + cursor = CK_STACK_NEXT(cursor); + continue; + } + + active = ck_pr_load_uint(&cr->active); + *af |= active; + + if (active != 0 && ck_pr_load_uint(&cr->epoch) != epoch) + return cr; + + cursor = CK_STACK_NEXT(cursor); + } + + return NULL; +} + +static void +ck_epoch_dispatch(struct ck_epoch_record *record, unsigned int e) +{ + unsigned int epoch = e & (CK_EPOCH_LENGTH - 1); + ck_stack_entry_t *head, *next, *cursor; + unsigned int i = 0; + + head = CK_STACK_FIRST(&record->pending[epoch]); + ck_stack_init(&record->pending[epoch]); + + for (cursor = head; cursor != NULL; cursor = next) { + struct ck_epoch_entry *entry = + ck_epoch_entry_container(cursor); + + next = CK_STACK_NEXT(cursor); + entry->function(entry); + i++; + } + + if (record->n_pending > record->n_peak) + record->n_peak = record->n_pending; + + record->n_dispatch += i; + record->n_pending -= i; + return; +} + +/* + * Reclaim all objects associated with a record. + */ +void +ck_epoch_reclaim(struct ck_epoch_record *record) +{ + unsigned int epoch; + + for (epoch = 0; epoch < CK_EPOCH_LENGTH; epoch++) + ck_epoch_dispatch(record, epoch); + + return; +} + +/* + * This function must not be called with-in read section. + */ +void +ck_epoch_synchronize(struct ck_epoch_record *record) +{ + struct ck_epoch *global = record->global; + struct ck_epoch_record *cr; + unsigned int delta, epoch, goal, i; + bool active; + + ck_pr_fence_memory(); + + /* + * The observation of the global epoch must be ordered with respect to + * all prior operations. The re-ordering of loads is permitted given + * monoticity of global epoch counter. + * + * If UINT_MAX concurrent mutations were to occur then it is possible + * to encounter an ABA-issue. If this is a concern, consider tuning + * write-side concurrency. + */ + delta = epoch = ck_pr_load_uint(&global->epoch); + goal = epoch + CK_EPOCH_GRACE; + + for (i = 0, cr = NULL; i < CK_EPOCH_GRACE - 1; cr = NULL, i++) { + bool r; + + /* + * Determine whether all threads have observed the current + * epoch with respect to the updates on invocation. + */ + while (cr = ck_epoch_scan(global, cr, delta, &active), + cr != NULL) { + unsigned int e_d; + + ck_pr_stall(); + + /* + * Another writer may have already observed a grace + * period. + */ + e_d = ck_pr_load_uint(&global->epoch); + if (e_d != delta) { + delta = e_d; + goto reload; + } + } + + /* + * If we have observed all threads as inactive, then we assume + * we are at a grace period. + */ + if (active == false) + break; + + /* + * Increment current epoch. CAS semantics are used to eliminate + * increment operations for synchronization that occurs for the + * same global epoch value snapshot. + * + * If we can guarantee there will only be one active barrier or + * epoch tick at a given time, then it is sufficient to use an + * increment operation. In a multi-barrier workload, however, + * it is possible to overflow the epoch value if we apply + * modulo-3 arithmetic. + */ + r = ck_pr_cas_uint_value(&global->epoch, delta, delta + 1, + &delta); + + /* Order subsequent thread active checks. */ + ck_pr_fence_atomic_load(); + + /* + * If CAS has succeeded, then set delta to latest snapshot. + * Otherwise, we have just acquired latest snapshot. + */ + delta = delta + r; + continue; + +reload: + if ((goal > epoch) & (delta >= goal)) { + /* + * Right now, epoch overflow is handled as an edge + * case. If we have already observed an epoch + * generation, then we can be sure no hazardous + * references exist to objects from this generation. We + * can actually avoid an addtional scan step at this + * point. + */ + break; + } + } + + /* + * A majority of use-cases will not require full barrier semantics. + * However, if non-temporal instructions are used, full barrier + * semantics are necessary. + */ + ck_pr_fence_memory(); + record->epoch = delta; + return; +} + +void +ck_epoch_barrier(struct ck_epoch_record *record) +{ + + ck_epoch_synchronize(record); + ck_epoch_reclaim(record); + return; +} + +/* + * It may be worth it to actually apply these deferral semantics to an epoch + * that was observed at ck_epoch_call time. The problem is that the latter + * would require a full fence. + * + * ck_epoch_call will dispatch to the latest epoch snapshot that was observed. + * There are cases where it will fail to reclaim as early as it could. If this + * becomes a problem, we could actually use a heap for epoch buckets but that + * is far from ideal too. + */ +bool +ck_epoch_poll(struct ck_epoch_record *record) +{ + bool active; + unsigned int epoch; + unsigned int snapshot; + struct ck_epoch_record *cr = NULL; + struct ck_epoch *global = record->global; + + epoch = ck_pr_load_uint(&global->epoch); + + /* Serialize epoch snapshots with respect to global epoch. */ + ck_pr_fence_memory(); + cr = ck_epoch_scan(global, cr, epoch, &active); + if (cr != NULL) { + record->epoch = epoch; + return false; + } + + /* We are at a grace period if all threads are inactive. */ + if (active == false) { + record->epoch = epoch; + for (epoch = 0; epoch < CK_EPOCH_LENGTH; epoch++) + ck_epoch_dispatch(record, epoch); + + return true; + } + + /* If an active thread exists, rely on epoch observation. */ + if (ck_pr_cas_uint_value(&global->epoch, epoch, epoch + 1, + &snapshot) == false) { + record->epoch = snapshot; + } else { + record->epoch = epoch + 1; + } + + ck_epoch_dispatch(record, epoch + 1); + return true; +} Property changes on: vendor-sys/ck/20161230/src/ck_epoch.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_hp.c =================================================================== --- vendor-sys/ck/20161230/src/ck_hp.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_hp.c (revision 310844) @@ -0,0 +1,323 @@ +/* + * Copyright 2010-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * (c) Copyright 2008, IBM Corporation. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This is an implementation of hazard pointers as detailed in: + * http://www.research.ibm.com/people/m/michael/ieeetpds-2004.pdf + * + * This API provides a publishing mechanism that defers destruction of + * hazard pointers until it is safe to do so. Preventing arbitrary re-use + * protects against the ABA problem and provides safe memory reclamation. + * The implementation was derived from the Hazard Pointers implementation + * from the Amino CBBS project. It has been heavily modified for Concurrency + * Kit. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +CK_STACK_CONTAINER(struct ck_hp_record, global_entry, ck_hp_record_container) +CK_STACK_CONTAINER(struct ck_hp_hazard, pending_entry, ck_hp_hazard_container) + +void +ck_hp_init(struct ck_hp *state, + unsigned int degree, + unsigned int threshold, + ck_hp_destructor_t destroy) +{ + + state->threshold = threshold; + state->degree = degree; + state->destroy = destroy; + state->n_subscribers = 0; + state->n_free = 0; + ck_stack_init(&state->subscribers); + ck_pr_fence_store(); + + return; +} + +void +ck_hp_set_threshold(struct ck_hp *state, unsigned int threshold) +{ + + ck_pr_store_uint(&state->threshold, threshold); + return; +} + +struct ck_hp_record * +ck_hp_recycle(struct ck_hp *global) +{ + struct ck_hp_record *record; + ck_stack_entry_t *entry; + int state; + + if (ck_pr_load_uint(&global->n_free) == 0) + return NULL; + + CK_STACK_FOREACH(&global->subscribers, entry) { + record = ck_hp_record_container(entry); + + if (ck_pr_load_int(&record->state) == CK_HP_FREE) { + ck_pr_fence_load(); + state = ck_pr_fas_int(&record->state, CK_HP_USED); + if (state == CK_HP_FREE) { + ck_pr_dec_uint(&global->n_free); + return record; + } + } + } + + return NULL; +} + +void +ck_hp_unregister(struct ck_hp_record *entry) +{ + + entry->n_pending = 0; + entry->n_peak = 0; + entry->n_reclamations = 0; + ck_stack_init(&entry->pending); + ck_pr_fence_store(); + ck_pr_store_int(&entry->state, CK_HP_FREE); + ck_pr_inc_uint(&entry->global->n_free); + return; +} + +void +ck_hp_register(struct ck_hp *state, + struct ck_hp_record *entry, + void **pointers) +{ + + entry->state = CK_HP_USED; + entry->global = state; + entry->pointers = pointers; + entry->n_pending = 0; + entry->n_peak = 0; + entry->n_reclamations = 0; + memset(pointers, 0, state->degree * sizeof(void *)); + ck_stack_init(&entry->pending); + ck_pr_fence_store(); + ck_stack_push_upmc(&state->subscribers, &entry->global_entry); + ck_pr_inc_uint(&state->n_subscribers); + return; +} + +static int +hazard_compare(const void *a, const void *b) +{ + void * const *x; + void * const *y; + + x = a; + y = b; + return ((*x > *y) - (*x < *y)); +} + +CK_CC_INLINE static bool +ck_hp_member_scan(ck_stack_entry_t *entry, unsigned int degree, void *pointer) +{ + struct ck_hp_record *record; + unsigned int i; + void *hazard; + + do { + record = ck_hp_record_container(entry); + if (ck_pr_load_int(&record->state) == CK_HP_FREE) + continue; + + if (ck_pr_load_ptr(&record->pointers) == NULL) + continue; + + for (i = 0; i < degree; i++) { + hazard = ck_pr_load_ptr(&record->pointers[i]); + if (hazard == pointer) + return (true); + } + } while ((entry = CK_STACK_NEXT(entry)) != NULL); + + return (false); +} + +CK_CC_INLINE static void * +ck_hp_member_cache(struct ck_hp *global, void **cache, unsigned int *n_hazards) +{ + struct ck_hp_record *record; + ck_stack_entry_t *entry; + unsigned int hazards = 0; + unsigned int i; + void *pointer; + + CK_STACK_FOREACH(&global->subscribers, entry) { + record = ck_hp_record_container(entry); + if (ck_pr_load_int(&record->state) == CK_HP_FREE) + continue; + + if (ck_pr_load_ptr(&record->pointers) == NULL) + continue; + + for (i = 0; i < global->degree; i++) { + if (hazards > CK_HP_CACHE) + break; + + pointer = ck_pr_load_ptr(&record->pointers[i]); + if (pointer != NULL) + cache[hazards++] = pointer; + } + } + + *n_hazards = hazards; + return (entry); +} + +void +ck_hp_reclaim(struct ck_hp_record *thread) +{ + struct ck_hp_hazard *hazard; + struct ck_hp *global = thread->global; + unsigned int n_hazards; + void **cache, *marker, *match; + ck_stack_entry_t *previous, *entry, *next; + + /* Store as many entries as possible in local array. */ + cache = thread->cache; + marker = ck_hp_member_cache(global, cache, &n_hazards); + + /* + * In theory, there is an n such that (n * (log n) ** 2) < np. + */ + qsort(cache, n_hazards, sizeof(void *), hazard_compare); + + previous = NULL; + CK_STACK_FOREACH_SAFE(&thread->pending, entry, next) { + hazard = ck_hp_hazard_container(entry); + match = bsearch(&hazard->pointer, cache, n_hazards, + sizeof(void *), hazard_compare); + if (match != NULL) { + previous = entry; + continue; + } + + if (marker != NULL && + ck_hp_member_scan(marker, global->degree, hazard->pointer)) { + previous = entry; + continue; + } + + thread->n_pending -= 1; + + /* Remove from the pending stack. */ + if (previous) + CK_STACK_NEXT(previous) = CK_STACK_NEXT(entry); + else + CK_STACK_FIRST(&thread->pending) = CK_STACK_NEXT(entry); + + /* The entry is now safe to destroy. */ + global->destroy(hazard->data); + thread->n_reclamations++; + } + + return; +} + +void +ck_hp_retire(struct ck_hp_record *thread, + struct ck_hp_hazard *hazard, + void *data, + void *pointer) +{ + + ck_pr_store_ptr(&hazard->pointer, pointer); + ck_pr_store_ptr(&hazard->data, data); + ck_stack_push_spnc(&thread->pending, &hazard->pending_entry); + + thread->n_pending += 1; + if (thread->n_pending > thread->n_peak) + thread->n_peak = thread->n_pending; + + return; +} + +void +ck_hp_free(struct ck_hp_record *thread, + struct ck_hp_hazard *hazard, + void *data, + void *pointer) +{ + struct ck_hp *global; + + global = ck_pr_load_ptr(&thread->global); + ck_pr_store_ptr(&hazard->data, data); + ck_pr_store_ptr(&hazard->pointer, pointer); + ck_stack_push_spnc(&thread->pending, &hazard->pending_entry); + + thread->n_pending += 1; + if (thread->n_pending > thread->n_peak) + thread->n_peak = thread->n_pending; + + if (thread->n_pending >= global->threshold) + ck_hp_reclaim(thread); + + return; +} + +void +ck_hp_purge(struct ck_hp_record *thread) +{ + ck_backoff_t backoff = CK_BACKOFF_INITIALIZER; + + while (thread->n_pending > 0) { + ck_hp_reclaim(thread); + if (thread->n_pending > 0) + ck_backoff_eb(&backoff); + } + + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_hp.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_hs.c =================================================================== --- vendor-sys/ck/20161230/src/ck_hs.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_hs.c (revision 310844) @@ -0,0 +1,941 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ck_internal.h" + +#ifndef CK_HS_PROBE_L1_SHIFT +#define CK_HS_PROBE_L1_SHIFT 3ULL +#endif /* CK_HS_PROBE_L1_SHIFT */ + +#define CK_HS_PROBE_L1 (1 << CK_HS_PROBE_L1_SHIFT) +#define CK_HS_PROBE_L1_MASK (CK_HS_PROBE_L1 - 1) + +#ifndef CK_HS_PROBE_L1_DEFAULT +#define CK_HS_PROBE_L1_DEFAULT CK_MD_CACHELINE +#endif + +#define CK_HS_VMA_MASK ((uintptr_t)((1ULL << CK_MD_VMA_BITS) - 1)) +#define CK_HS_VMA(x) \ + ((void *)((uintptr_t)(x) & CK_HS_VMA_MASK)) + +#define CK_HS_EMPTY NULL +#define CK_HS_TOMBSTONE ((void *)~(uintptr_t)0) +#define CK_HS_G (2) +#define CK_HS_G_MASK (CK_HS_G - 1) + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_STORE_8) +#define CK_HS_WORD uint8_t +#define CK_HS_WORD_MAX UINT8_MAX +#define CK_HS_STORE(x, y) ck_pr_store_8(x, y) +#define CK_HS_LOAD(x) ck_pr_load_8(x) +#elif defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_STORE_16) +#define CK_HS_WORD uint16_t +#define CK_HS_WORD_MAX UINT16_MAX +#define CK_HS_STORE(x, y) ck_pr_store_16(x, y) +#define CK_HS_LOAD(x) ck_pr_load_16(x) +#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32) +#define CK_HS_WORD uint32_t +#define CK_HS_WORD_MAX UINT32_MAX +#define CK_HS_STORE(x, y) ck_pr_store_32(x, y) +#define CK_HS_LOAD(x) ck_pr_load_32(x) +#else +#error "ck_hs is not supported on your platform." +#endif + +enum ck_hs_probe_behavior { + CK_HS_PROBE = 0, /* Default behavior. */ + CK_HS_PROBE_TOMBSTONE, /* Short-circuit on tombstone. */ + CK_HS_PROBE_INSERT /* Short-circuit on probe bound if tombstone found. */ +}; + +struct ck_hs_map { + unsigned int generation[CK_HS_G]; + unsigned int probe_maximum; + unsigned long mask; + unsigned long step; + unsigned int probe_limit; + unsigned int tombstones; + unsigned long n_entries; + unsigned long capacity; + unsigned long size; + CK_HS_WORD *probe_bound; + const void **entries; +}; + +static inline void +ck_hs_map_signal(struct ck_hs_map *map, unsigned long h) +{ + + h &= CK_HS_G_MASK; + ck_pr_store_uint(&map->generation[h], + map->generation[h] + 1); + ck_pr_fence_store(); + return; +} + +void +ck_hs_iterator_init(struct ck_hs_iterator *iterator) +{ + + iterator->cursor = NULL; + iterator->offset = 0; + return; +} + +bool +ck_hs_next(struct ck_hs *hs, struct ck_hs_iterator *i, void **key) +{ + struct ck_hs_map *map = hs->map; + void *value; + + if (i->offset >= map->capacity) + return false; + + do { + value = CK_CC_DECONST_PTR(map->entries[i->offset]); + if (value != CK_HS_EMPTY && value != CK_HS_TOMBSTONE) { +#ifdef CK_HS_PP + if (hs->mode & CK_HS_MODE_OBJECT) + value = CK_HS_VMA(value); +#endif + i->offset++; + *key = value; + return true; + } + } while (++i->offset < map->capacity); + + return false; +} + +void +ck_hs_stat(struct ck_hs *hs, struct ck_hs_stat *st) +{ + struct ck_hs_map *map = hs->map; + + st->n_entries = map->n_entries; + st->tombstones = map->tombstones; + st->probe_maximum = map->probe_maximum; + return; +} + +unsigned long +ck_hs_count(struct ck_hs *hs) +{ + + return hs->map->n_entries; +} + +static void +ck_hs_map_destroy(struct ck_malloc *m, struct ck_hs_map *map, bool defer) +{ + + m->free(map, map->size, defer); + return; +} + +void +ck_hs_destroy(struct ck_hs *hs) +{ + + ck_hs_map_destroy(hs->m, hs->map, false); + return; +} + +static struct ck_hs_map * +ck_hs_map_create(struct ck_hs *hs, unsigned long entries) +{ + struct ck_hs_map *map; + unsigned long size, n_entries, prefix, limit; + + n_entries = ck_internal_power_2(entries); + if (n_entries < CK_HS_PROBE_L1) + n_entries = CK_HS_PROBE_L1; + + size = sizeof(struct ck_hs_map) + (sizeof(void *) * n_entries + CK_MD_CACHELINE - 1); + + if (hs->mode & CK_HS_MODE_DELETE) { + prefix = sizeof(CK_HS_WORD) * n_entries; + size += prefix; + } else { + prefix = 0; + } + + map = hs->m->malloc(size); + if (map == NULL) + return NULL; + + map->size = size; + + /* We should probably use a more intelligent heuristic for default probe length. */ + limit = ck_internal_max(n_entries >> (CK_HS_PROBE_L1_SHIFT + 2), CK_HS_PROBE_L1_DEFAULT); + if (limit > UINT_MAX) + limit = UINT_MAX; + + map->probe_limit = (unsigned int)limit; + map->probe_maximum = 0; + map->capacity = n_entries; + map->step = ck_internal_bsf(n_entries); + map->mask = n_entries - 1; + map->n_entries = 0; + + /* Align map allocation to cache line. */ + map->entries = (void *)(((uintptr_t)&map[1] + prefix + + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + + memset(map->entries, 0, sizeof(void *) * n_entries); + memset(map->generation, 0, sizeof map->generation); + + if (hs->mode & CK_HS_MODE_DELETE) { + map->probe_bound = (CK_HS_WORD *)&map[1]; + memset(map->probe_bound, 0, prefix); + } else { + map->probe_bound = NULL; + } + + /* Commit entries purge with respect to map publication. */ + ck_pr_fence_store(); + return map; +} + +bool +ck_hs_reset_size(struct ck_hs *hs, unsigned long capacity) +{ + struct ck_hs_map *map, *previous; + + previous = hs->map; + map = ck_hs_map_create(hs, capacity); + if (map == NULL) + return false; + + ck_pr_store_ptr(&hs->map, map); + ck_hs_map_destroy(hs->m, previous, true); + return true; +} + +bool +ck_hs_reset(struct ck_hs *hs) +{ + struct ck_hs_map *previous; + + previous = hs->map; + return ck_hs_reset_size(hs, previous->capacity); +} + +static inline unsigned long +ck_hs_map_probe_next(struct ck_hs_map *map, + unsigned long offset, + unsigned long h, + unsigned long level, + unsigned long probes) +{ + unsigned long r, stride; + + r = (h >> map->step) >> level; + stride = (r & ~CK_HS_PROBE_L1_MASK) << 1 | (r & CK_HS_PROBE_L1_MASK); + + return (offset + (probes >> CK_HS_PROBE_L1_SHIFT) + + (stride | CK_HS_PROBE_L1)) & map->mask; +} + +static inline void +ck_hs_map_bound_set(struct ck_hs_map *m, + unsigned long h, + unsigned long n_probes) +{ + unsigned long offset = h & m->mask; + + if (n_probes > m->probe_maximum) + ck_pr_store_uint(&m->probe_maximum, n_probes); + + if (m->probe_bound != NULL && m->probe_bound[offset] < n_probes) { + if (n_probes > CK_HS_WORD_MAX) + n_probes = CK_HS_WORD_MAX; + + CK_HS_STORE(&m->probe_bound[offset], n_probes); + ck_pr_fence_store(); + } + + return; +} + +static inline unsigned int +ck_hs_map_bound_get(struct ck_hs_map *m, unsigned long h) +{ + unsigned long offset = h & m->mask; + unsigned int r = CK_HS_WORD_MAX; + + if (m->probe_bound != NULL) { + r = CK_HS_LOAD(&m->probe_bound[offset]); + if (r == CK_HS_WORD_MAX) + r = ck_pr_load_uint(&m->probe_maximum); + } else { + r = ck_pr_load_uint(&m->probe_maximum); + } + + return r; +} + +bool +ck_hs_grow(struct ck_hs *hs, + unsigned long capacity) +{ + struct ck_hs_map *map, *update; + unsigned long k, i, j, offset, probes; + const void *previous, **bucket; + +restart: + map = hs->map; + if (map->capacity > capacity) + return false; + + update = ck_hs_map_create(hs, capacity); + if (update == NULL) + return false; + + for (k = 0; k < map->capacity; k++) { + unsigned long h; + + previous = map->entries[k]; + if (previous == CK_HS_EMPTY || previous == CK_HS_TOMBSTONE) + continue; + +#ifdef CK_HS_PP + if (hs->mode & CK_HS_MODE_OBJECT) + previous = CK_HS_VMA(previous); +#endif + + h = hs->hf(previous, hs->seed); + offset = h & update->mask; + i = probes = 0; + + for (;;) { + bucket = (const void **)((uintptr_t)&update->entries[offset] & ~(CK_MD_CACHELINE - 1)); + + for (j = 0; j < CK_HS_PROBE_L1; j++) { + const void **cursor = bucket + ((j + offset) & (CK_HS_PROBE_L1 - 1)); + + if (probes++ == update->probe_limit) + break; + + if (CK_CC_LIKELY(*cursor == CK_HS_EMPTY)) { + *cursor = map->entries[k]; + update->n_entries++; + + ck_hs_map_bound_set(update, h, probes); + break; + } + } + + if (j < CK_HS_PROBE_L1) + break; + + offset = ck_hs_map_probe_next(update, offset, h, i++, probes); + } + + if (probes > update->probe_limit) { + /* + * We have hit the probe limit, map needs to be even larger. + */ + ck_hs_map_destroy(hs->m, update, false); + capacity <<= 1; + goto restart; + } + } + + ck_pr_fence_store(); + ck_pr_store_ptr(&hs->map, update); + ck_hs_map_destroy(hs->m, map, true); + return true; +} + +static void +ck_hs_map_postinsert(struct ck_hs *hs, struct ck_hs_map *map) +{ + + map->n_entries++; + if ((map->n_entries << 1) > map->capacity) + ck_hs_grow(hs, map->capacity << 1); + + return; +} + +bool +ck_hs_rebuild(struct ck_hs *hs) +{ + + return ck_hs_grow(hs, hs->map->capacity); +} + +static const void ** +ck_hs_map_probe(struct ck_hs *hs, + struct ck_hs_map *map, + unsigned long *n_probes, + const void ***priority, + unsigned long h, + const void *key, + const void **object, + unsigned long probe_limit, + enum ck_hs_probe_behavior behavior) +{ + const void **bucket, **cursor, *k, *compare; + const void **pr = NULL; + unsigned long offset, j, i, probes, opl; + +#ifdef CK_HS_PP + /* If we are storing object pointers, then we may leverage pointer packing. */ + unsigned long hv = 0; + + if (hs->mode & CK_HS_MODE_OBJECT) { + hv = (h >> 25) & CK_HS_KEY_MASK; + compare = CK_HS_VMA(key); + } else { + compare = key; + } +#else + compare = key; +#endif + + offset = h & map->mask; + *object = NULL; + i = probes = 0; + + opl = probe_limit; + if (behavior == CK_HS_PROBE_INSERT) + probe_limit = ck_hs_map_bound_get(map, h); + + for (;;) { + bucket = (const void **)((uintptr_t)&map->entries[offset] & ~(CK_MD_CACHELINE - 1)); + + for (j = 0; j < CK_HS_PROBE_L1; j++) { + cursor = bucket + ((j + offset) & (CK_HS_PROBE_L1 - 1)); + + if (probes++ == probe_limit) { + if (probe_limit == opl || pr != NULL) { + k = CK_HS_EMPTY; + goto leave; + } + + /* + * If no eligible slot has been found yet, continue probe + * sequence with original probe limit. + */ + probe_limit = opl; + } + + k = ck_pr_load_ptr(cursor); + if (k == CK_HS_EMPTY) + goto leave; + + if (k == CK_HS_TOMBSTONE) { + if (pr == NULL) { + pr = cursor; + *n_probes = probes; + + if (behavior == CK_HS_PROBE_TOMBSTONE) { + k = CK_HS_EMPTY; + goto leave; + } + } + + continue; + } + +#ifdef CK_HS_PP + if (hs->mode & CK_HS_MODE_OBJECT) { + if (((uintptr_t)k >> CK_MD_VMA_BITS) != hv) + continue; + + k = CK_HS_VMA(k); + } +#endif + + if (k == compare) + goto leave; + + if (hs->compare == NULL) + continue; + + if (hs->compare(k, key) == true) + goto leave; + } + + offset = ck_hs_map_probe_next(map, offset, h, i++, probes); + } + +leave: + if (probes > probe_limit) { + cursor = NULL; + } else { + *object = k; + } + + if (pr == NULL) + *n_probes = probes; + + *priority = pr; + return cursor; +} + +static inline const void * +ck_hs_marshal(unsigned int mode, const void *key, unsigned long h) +{ +#ifdef CK_HS_PP + const void *insert; + + if (mode & CK_HS_MODE_OBJECT) { + insert = (void *)((uintptr_t)CK_HS_VMA(key) | + ((h >> 25) << CK_MD_VMA_BITS)); + } else { + insert = key; + } + + return insert; +#else + (void)mode; + (void)h; + + return key; +#endif +} + +bool +ck_hs_gc(struct ck_hs *hs, unsigned long cycles, unsigned long seed) +{ + unsigned long size = 0; + unsigned long i; + struct ck_hs_map *map = hs->map; + unsigned int maximum; + CK_HS_WORD *bounds = NULL; + + if (map->n_entries == 0) { + ck_pr_store_uint(&map->probe_maximum, 0); + if (map->probe_bound != NULL) + memset(map->probe_bound, 0, sizeof(CK_HS_WORD) * map->capacity); + + return true; + } + + if (cycles == 0) { + maximum = 0; + + if (map->probe_bound != NULL) { + size = sizeof(CK_HS_WORD) * map->capacity; + bounds = hs->m->malloc(size); + if (bounds == NULL) + return false; + + memset(bounds, 0, size); + } + } else { + maximum = map->probe_maximum; + } + + for (i = 0; i < map->capacity; i++) { + const void **first, *object, **slot, *entry; + unsigned long n_probes, offset, h; + + entry = map->entries[(i + seed) & map->mask]; + if (entry == CK_HS_EMPTY || entry == CK_HS_TOMBSTONE) + continue; + +#ifdef CK_HS_PP + if (hs->mode & CK_HS_MODE_OBJECT) + entry = CK_HS_VMA(entry); +#endif + + h = hs->hf(entry, hs->seed); + offset = h & map->mask; + + slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, entry, &object, + ck_hs_map_bound_get(map, h), CK_HS_PROBE); + + if (first != NULL) { + const void *insert = ck_hs_marshal(hs->mode, entry, h); + + ck_pr_store_ptr(first, insert); + ck_hs_map_signal(map, h); + ck_pr_store_ptr(slot, CK_HS_TOMBSTONE); + } + + if (cycles == 0) { + if (n_probes > maximum) + maximum = n_probes; + + if (n_probes > CK_HS_WORD_MAX) + n_probes = CK_HS_WORD_MAX; + + if (bounds != NULL && n_probes > bounds[offset]) + bounds[offset] = n_probes; + } else if (--cycles == 0) + break; + } + + /* + * The following only apply to garbage collection involving + * a full scan of all entries. + */ + if (maximum != map->probe_maximum) + ck_pr_store_uint(&map->probe_maximum, maximum); + + if (bounds != NULL) { + for (i = 0; i < map->capacity; i++) + CK_HS_STORE(&map->probe_bound[i], bounds[i]); + + hs->m->free(bounds, size, false); + } + + return true; +} + +bool +ck_hs_fas(struct ck_hs *hs, + unsigned long h, + const void *key, + void **previous) +{ + const void **slot, **first, *object, *insert; + struct ck_hs_map *map = hs->map; + unsigned long n_probes; + + *previous = NULL; + slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, + ck_hs_map_bound_get(map, h), CK_HS_PROBE); + + /* Replacement semantics presume existence. */ + if (object == NULL) + return false; + + insert = ck_hs_marshal(hs->mode, key, h); + + if (first != NULL) { + ck_pr_store_ptr(first, insert); + ck_hs_map_signal(map, h); + ck_pr_store_ptr(slot, CK_HS_TOMBSTONE); + } else { + ck_pr_store_ptr(slot, insert); + } + + *previous = CK_CC_DECONST_PTR(object); + return true; +} + +/* + * An apply function takes two arguments. The first argument is a pointer to a + * pre-existing object. The second argument is a pointer to the fifth argument + * passed to ck_hs_apply. If a non-NULL pointer is passed to the first argument + * and the return value of the apply function is NULL, then the pre-existing + * value is deleted. If the return pointer is the same as the one passed to the + * apply function then no changes are made to the hash table. If the first + * argument is non-NULL and the return pointer is different than that passed to + * the apply function, then the pre-existing value is replaced. For + * replacement, it is required that the value itself is identical to the + * previous value. + */ +bool +ck_hs_apply(struct ck_hs *hs, + unsigned long h, + const void *key, + ck_hs_apply_fn_t *fn, + void *cl) +{ + const void **slot, **first, *object, *delta, *insert; + unsigned long n_probes; + struct ck_hs_map *map; + +restart: + map = hs->map; + + slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_HS_PROBE_INSERT); + if (slot == NULL && first == NULL) { + if (ck_hs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + + delta = fn(CK_CC_DECONST_PTR(object), cl); + if (delta == NULL) { + /* + * The apply function has requested deletion. If the object doesn't exist, + * then exit early. + */ + if (CK_CC_UNLIKELY(object == NULL)) + return true; + + /* Otherwise, mark slot as deleted. */ + ck_pr_store_ptr(slot, CK_HS_TOMBSTONE); + map->n_entries--; + map->tombstones++; + return true; + } + + /* The apply function has not requested hash set modification so exit early. */ + if (delta == object) + return true; + + /* A modification or insertion has been requested. */ + ck_hs_map_bound_set(map, h, n_probes); + + insert = ck_hs_marshal(hs->mode, delta, h); + if (first != NULL) { + /* + * This follows the same semantics as ck_hs_set, please refer to that + * function for documentation. + */ + ck_pr_store_ptr(first, insert); + + if (object != NULL) { + ck_hs_map_signal(map, h); + ck_pr_store_ptr(slot, CK_HS_TOMBSTONE); + } + } else { + /* + * If we are storing into same slot, then atomic store is sufficient + * for replacement. + */ + ck_pr_store_ptr(slot, insert); + } + + if (object == NULL) + ck_hs_map_postinsert(hs, map); + + return true; +} + +bool +ck_hs_set(struct ck_hs *hs, + unsigned long h, + const void *key, + void **previous) +{ + const void **slot, **first, *object, *insert; + unsigned long n_probes; + struct ck_hs_map *map; + + *previous = NULL; + +restart: + map = hs->map; + + slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_HS_PROBE_INSERT); + if (slot == NULL && first == NULL) { + if (ck_hs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + + ck_hs_map_bound_set(map, h, n_probes); + insert = ck_hs_marshal(hs->mode, key, h); + + if (first != NULL) { + /* If an earlier bucket was found, then store entry there. */ + ck_pr_store_ptr(first, insert); + + /* + * If a duplicate key was found, then delete it after + * signaling concurrent probes to restart. Optionally, + * it is possible to install tombstone after grace + * period if we can guarantee earlier position of + * duplicate key. + */ + if (object != NULL) { + ck_hs_map_signal(map, h); + ck_pr_store_ptr(slot, CK_HS_TOMBSTONE); + } + } else { + /* + * If we are storing into same slot, then atomic store is sufficient + * for replacement. + */ + ck_pr_store_ptr(slot, insert); + } + + if (object == NULL) + ck_hs_map_postinsert(hs, map); + + *previous = CK_CC_DECONST_PTR(object); + return true; +} + +CK_CC_INLINE static bool +ck_hs_put_internal(struct ck_hs *hs, + unsigned long h, + const void *key, + enum ck_hs_probe_behavior behavior) +{ + const void **slot, **first, *object, *insert; + unsigned long n_probes; + struct ck_hs_map *map; + +restart: + map = hs->map; + + slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, + map->probe_limit, behavior); + + if (slot == NULL && first == NULL) { + if (ck_hs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + + /* Fail operation if a match was found. */ + if (object != NULL) + return false; + + ck_hs_map_bound_set(map, h, n_probes); + insert = ck_hs_marshal(hs->mode, key, h); + + if (first != NULL) { + /* Insert key into first bucket in probe sequence. */ + ck_pr_store_ptr(first, insert); + } else { + /* An empty slot was found. */ + ck_pr_store_ptr(slot, insert); + } + + ck_hs_map_postinsert(hs, map); + return true; +} + +bool +ck_hs_put(struct ck_hs *hs, + unsigned long h, + const void *key) +{ + + return ck_hs_put_internal(hs, h, key, CK_HS_PROBE_INSERT); +} + +bool +ck_hs_put_unique(struct ck_hs *hs, + unsigned long h, + const void *key) +{ + + return ck_hs_put_internal(hs, h, key, CK_HS_PROBE_TOMBSTONE); +} + +void * +ck_hs_get(struct ck_hs *hs, + unsigned long h, + const void *key) +{ + const void **first, *object; + struct ck_hs_map *map; + unsigned long n_probes; + unsigned int g, g_p, probe; + unsigned int *generation; + + do { + map = ck_pr_load_ptr(&hs->map); + generation = &map->generation[h & CK_HS_G_MASK]; + g = ck_pr_load_uint(generation); + probe = ck_hs_map_bound_get(map, h); + ck_pr_fence_load(); + + ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, probe, CK_HS_PROBE); + + ck_pr_fence_load(); + g_p = ck_pr_load_uint(generation); + } while (g != g_p); + + return CK_CC_DECONST_PTR(object); +} + +void * +ck_hs_remove(struct ck_hs *hs, + unsigned long h, + const void *key) +{ + const void **slot, **first, *object; + struct ck_hs_map *map = hs->map; + unsigned long n_probes; + + slot = ck_hs_map_probe(hs, map, &n_probes, &first, h, key, &object, + ck_hs_map_bound_get(map, h), CK_HS_PROBE); + if (object == NULL) + return NULL; + + ck_pr_store_ptr(slot, CK_HS_TOMBSTONE); + map->n_entries--; + map->tombstones++; + return CK_CC_DECONST_PTR(object); +} + +bool +ck_hs_move(struct ck_hs *hs, + struct ck_hs *source, + ck_hs_hash_cb_t *hf, + ck_hs_compare_cb_t *compare, + struct ck_malloc *m) +{ + + if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL) + return false; + + hs->mode = source->mode; + hs->seed = source->seed; + hs->map = source->map; + hs->m = m; + hs->hf = hf; + hs->compare = compare; + return true; +} + +bool +ck_hs_init(struct ck_hs *hs, + unsigned int mode, + ck_hs_hash_cb_t *hf, + ck_hs_compare_cb_t *compare, + struct ck_malloc *m, + unsigned long n_entries, + unsigned long seed) +{ + + if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL) + return false; + + hs->m = m; + hs->mode = mode; + hs->seed = seed; + hs->hf = hf; + hs->compare = compare; + + hs->map = ck_hs_map_create(hs, n_entries); + return hs->map != NULL; +} Property changes on: vendor-sys/ck/20161230/src/ck_hs.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_ht.c =================================================================== --- vendor-sys/ck/20161230/src/ck_ht.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_ht.c (revision 310844) @@ -0,0 +1,1036 @@ +/* + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define CK_HT_IM +#include + +/* + * This implementation borrows several techniques from Josh Dybnis's + * nbds library which can be found at http://code.google.com/p/nbds + * + * This release currently only includes support for 64-bit platforms. + * We can address 32-bit platforms in a future release. + */ +#include +#include +#include +#include +#include +#include + +#include "ck_ht_hash.h" +#include "ck_internal.h" + +#ifndef CK_HT_BUCKET_LENGTH + +#ifdef CK_HT_PP +#define CK_HT_BUCKET_SHIFT 2ULL +#else +#define CK_HT_BUCKET_SHIFT 1ULL +#endif + +#define CK_HT_BUCKET_LENGTH (1U << CK_HT_BUCKET_SHIFT) +#define CK_HT_BUCKET_MASK (CK_HT_BUCKET_LENGTH - 1) +#endif + +#ifndef CK_HT_PROBE_DEFAULT +#define CK_HT_PROBE_DEFAULT 64ULL +#endif + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_STORE_8) +#define CK_HT_WORD uint8_t +#define CK_HT_WORD_MAX UINT8_MAX +#define CK_HT_STORE(x, y) ck_pr_store_8(x, y) +#define CK_HT_LOAD(x) ck_pr_load_8(x) +#elif defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_STORE_16) +#define CK_HT_WORD uint16_t +#define CK_HT_WORD_MAX UINT16_MAX +#define CK_HT_STORE(x, y) ck_pr_store_16(x, y) +#define CK_HT_LOAD(x) ck_pr_load_16(x) +#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32) +#define CK_HT_WORD uint32_t +#define CK_HT_WORD_MAX UINT32_MAX +#define CK_HT_STORE(x, y) ck_pr_store_32(x, y) +#define CK_HT_LOAD(x) ck_pr_load_32(x) +#else +#error "ck_ht is not supported on your platform." +#endif + +struct ck_ht_map { + unsigned int mode; + CK_HT_TYPE deletions; + CK_HT_TYPE probe_maximum; + CK_HT_TYPE probe_length; + CK_HT_TYPE probe_limit; + CK_HT_TYPE size; + CK_HT_TYPE n_entries; + CK_HT_TYPE mask; + CK_HT_TYPE capacity; + CK_HT_TYPE step; + CK_HT_WORD *probe_bound; + struct ck_ht_entry *entries; +}; + +void +ck_ht_stat(struct ck_ht *table, + struct ck_ht_stat *st) +{ + struct ck_ht_map *map = table->map; + + st->n_entries = map->n_entries; + st->probe_maximum = map->probe_maximum; + return; +} + +void +ck_ht_hash(struct ck_ht_hash *h, + struct ck_ht *table, + const void *key, + uint16_t key_length) +{ + + table->h(h, key, key_length, table->seed); + return; +} + +void +ck_ht_hash_direct(struct ck_ht_hash *h, + struct ck_ht *table, + uintptr_t key) +{ + + ck_ht_hash(h, table, &key, sizeof(key)); + return; +} + +static void +ck_ht_hash_wrapper(struct ck_ht_hash *h, + const void *key, + size_t length, + uint64_t seed) +{ + + h->value = (unsigned long)MurmurHash64A(key, length, seed); + return; +} + +static struct ck_ht_map * +ck_ht_map_create(struct ck_ht *table, CK_HT_TYPE entries) +{ + struct ck_ht_map *map; + CK_HT_TYPE size; + uintptr_t prefix; + uint32_t n_entries; + + n_entries = ck_internal_power_2(entries); + if (n_entries < CK_HT_BUCKET_LENGTH) + n_entries = CK_HT_BUCKET_LENGTH; + + size = sizeof(struct ck_ht_map) + + (sizeof(struct ck_ht_entry) * n_entries + CK_MD_CACHELINE - 1); + + if (table->mode & CK_HT_WORKLOAD_DELETE) { + prefix = sizeof(CK_HT_WORD) * n_entries; + size += prefix; + } else { + prefix = 0; + } + + map = table->m->malloc(size); + if (map == NULL) + return NULL; + + map->mode = table->mode; + map->size = size; + map->probe_limit = ck_internal_max_64(n_entries >> + (CK_HT_BUCKET_SHIFT + 2), CK_HT_PROBE_DEFAULT); + + map->deletions = 0; + map->probe_maximum = 0; + map->capacity = n_entries; + map->step = ck_internal_bsf_64(map->capacity); + map->mask = map->capacity - 1; + map->n_entries = 0; + map->entries = (struct ck_ht_entry *)(((uintptr_t)&map[1] + prefix + + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + + if (table->mode & CK_HT_WORKLOAD_DELETE) { + map->probe_bound = (CK_HT_WORD *)&map[1]; + memset(map->probe_bound, 0, prefix); + } else { + map->probe_bound = NULL; + } + + memset(map->entries, 0, sizeof(struct ck_ht_entry) * n_entries); + ck_pr_fence_store(); + return map; +} + +static inline void +ck_ht_map_bound_set(struct ck_ht_map *m, + struct ck_ht_hash h, + CK_HT_TYPE n_probes) +{ + CK_HT_TYPE offset = h.value & m->mask; + + if (n_probes > m->probe_maximum) + CK_HT_TYPE_STORE(&m->probe_maximum, n_probes); + + if (m->probe_bound != NULL && m->probe_bound[offset] < n_probes) { + if (n_probes >= CK_HT_WORD_MAX) + n_probes = CK_HT_WORD_MAX; + + CK_HT_STORE(&m->probe_bound[offset], n_probes); + ck_pr_fence_store(); + } + + return; +} + +static inline CK_HT_TYPE +ck_ht_map_bound_get(struct ck_ht_map *m, struct ck_ht_hash h) +{ + CK_HT_TYPE offset = h.value & m->mask; + CK_HT_TYPE r = CK_HT_WORD_MAX; + + if (m->probe_bound != NULL) { + r = CK_HT_LOAD(&m->probe_bound[offset]); + if (r == CK_HT_WORD_MAX) + r = CK_HT_TYPE_LOAD(&m->probe_maximum); + } else { + r = CK_HT_TYPE_LOAD(&m->probe_maximum); + } + + return r; +} + +static void +ck_ht_map_destroy(struct ck_malloc *m, struct ck_ht_map *map, bool defer) +{ + + m->free(map, map->size, defer); + return; +} + +static inline size_t +ck_ht_map_probe_next(struct ck_ht_map *map, size_t offset, ck_ht_hash_t h, size_t probes) +{ + ck_ht_hash_t r; + size_t stride; + unsigned long level = (unsigned long)probes >> CK_HT_BUCKET_SHIFT; + + r.value = (h.value >> map->step) >> level; + stride = (r.value & ~CK_HT_BUCKET_MASK) << 1 + | (r.value & CK_HT_BUCKET_MASK); + + return (offset + level + + (stride | CK_HT_BUCKET_LENGTH)) & map->mask; +} + +bool +ck_ht_init(struct ck_ht *table, + unsigned int mode, + ck_ht_hash_cb_t *h, + struct ck_malloc *m, + CK_HT_TYPE entries, + uint64_t seed) +{ + + if (m == NULL || m->malloc == NULL || m->free == NULL) + return false; + + table->m = m; + table->mode = mode; + table->seed = seed; + + if (h == NULL) { + table->h = ck_ht_hash_wrapper; + } else { + table->h = h; + } + + table->map = ck_ht_map_create(table, entries); + return table->map != NULL; +} + +static struct ck_ht_entry * +ck_ht_map_probe_wr(struct ck_ht_map *map, + ck_ht_hash_t h, + ck_ht_entry_t *snapshot, + ck_ht_entry_t **available, + const void *key, + uint16_t key_length, + CK_HT_TYPE *probe_limit, + CK_HT_TYPE *probe_wr) +{ + struct ck_ht_entry *bucket, *cursor; + struct ck_ht_entry *first = NULL; + size_t offset, i, j; + CK_HT_TYPE probes = 0; + CK_HT_TYPE limit; + + if (probe_limit == NULL) { + limit = ck_ht_map_bound_get(map, h); + } else { + limit = CK_HT_TYPE_MAX; + } + + offset = h.value & map->mask; + for (i = 0; i < map->probe_limit; i++) { + /* + * Probe on a complete cache line first. Scan forward and wrap around to + * the beginning of the cache line. Only when the complete cache line has + * been scanned do we move on to the next row. + */ + bucket = (void *)((uintptr_t)(map->entries + offset) & + ~(CK_MD_CACHELINE - 1)); + + for (j = 0; j < CK_HT_BUCKET_LENGTH; j++) { + uint16_t k; + + if (probes++ > limit) + break; + + cursor = bucket + ((j + offset) & (CK_HT_BUCKET_LENGTH - 1)); + + /* + * It is probably worth it to encapsulate probe state + * in order to prevent a complete reprobe sequence in + * the case of intermittent writers. + */ + if (cursor->key == CK_HT_KEY_TOMBSTONE) { + if (first == NULL) { + first = cursor; + *probe_wr = probes; + } + + continue; + } + + if (cursor->key == CK_HT_KEY_EMPTY) + goto leave; + + if (cursor->key == (uintptr_t)key) + goto leave; + + if (map->mode & CK_HT_MODE_BYTESTRING) { + void *pointer; + + /* + * Check memoized portion of hash value before + * expensive full-length comparison. + */ + k = ck_ht_entry_key_length(cursor); + if (k != key_length) + continue; + +#ifdef CK_HT_PP + if ((cursor->value >> CK_MD_VMA_BITS) != ((h.value >> 32) & CK_HT_KEY_MASK)) + continue; +#else + if (cursor->hash != h.value) + continue; +#endif + + pointer = ck_ht_entry_key(cursor); + if (memcmp(pointer, key, key_length) == 0) + goto leave; + } + } + + offset = ck_ht_map_probe_next(map, offset, h, probes); + } + + cursor = NULL; + +leave: + if (probe_limit != NULL) { + *probe_limit = probes; + } else if (first == NULL) { + *probe_wr = probes; + } + + *available = first; + + if (cursor != NULL) { + *snapshot = *cursor; + } + + return cursor; +} + +bool +ck_ht_gc(struct ck_ht *ht, unsigned long cycles, unsigned long seed) +{ + CK_HT_WORD *bounds = NULL; + struct ck_ht_map *map = ht->map; + CK_HT_TYPE maximum, i; + CK_HT_TYPE size = 0; + + if (map->n_entries == 0) { + CK_HT_TYPE_STORE(&map->probe_maximum, 0); + if (map->probe_bound != NULL) + memset(map->probe_bound, 0, sizeof(CK_HT_WORD) * map->capacity); + + return true; + } + + if (cycles == 0) { + maximum = 0; + + if (map->probe_bound != NULL) { + size = sizeof(CK_HT_WORD) * map->capacity; + bounds = ht->m->malloc(size); + if (bounds == NULL) + return false; + + memset(bounds, 0, size); + } + } else { + maximum = map->probe_maximum; + } + + for (i = 0; i < map->capacity; i++) { + struct ck_ht_entry *entry, *priority, snapshot; + struct ck_ht_hash h; + CK_HT_TYPE probes_wr; + CK_HT_TYPE offset; + + entry = &map->entries[(i + seed) & map->mask]; + if (entry->key == CK_HT_KEY_EMPTY || + entry->key == CK_HT_KEY_TOMBSTONE) { + continue; + } + + if (ht->mode & CK_HT_MODE_BYTESTRING) { +#ifndef CK_HT_PP + h.value = entry->hash; +#else + ht->h(&h, ck_ht_entry_key(entry), ck_ht_entry_key_length(entry), + ht->seed); +#endif + entry = ck_ht_map_probe_wr(map, h, &snapshot, &priority, + ck_ht_entry_key(entry), + ck_ht_entry_key_length(entry), + NULL, &probes_wr); + } else { +#ifndef CK_HT_PP + h.value = entry->hash; +#else + ht->h(&h, &entry->key, sizeof(entry->key), ht->seed); +#endif + entry = ck_ht_map_probe_wr(map, h, &snapshot, &priority, + (void *)entry->key, + sizeof(entry->key), + NULL, &probes_wr); + } + + offset = h.value & map->mask; + + if (priority != NULL) { + CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1); + ck_pr_fence_store(); +#ifndef CK_HT_PP + CK_HT_TYPE_STORE(&priority->key_length, entry->key_length); + CK_HT_TYPE_STORE(&priority->hash, entry->hash); +#endif + ck_pr_store_ptr_unsafe(&priority->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&priority->key, (void *)entry->key); + ck_pr_fence_store(); + CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1); + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&entry->key, (void *)CK_HT_KEY_TOMBSTONE); + ck_pr_fence_store(); + } + + if (cycles == 0) { + if (probes_wr > maximum) + maximum = probes_wr; + + if (probes_wr >= CK_HT_WORD_MAX) + probes_wr = CK_HT_WORD_MAX; + + if (bounds != NULL && probes_wr > bounds[offset]) + bounds[offset] = probes_wr; + } else if (--cycles == 0) + break; + } + + if (maximum != map->probe_maximum) + CK_HT_TYPE_STORE(&map->probe_maximum, maximum); + + if (bounds != NULL) { + for (i = 0; i < map->capacity; i++) + CK_HT_STORE(&map->probe_bound[i], bounds[i]); + + ht->m->free(bounds, size, false); + } + + return true; +} + +static struct ck_ht_entry * +ck_ht_map_probe_rd(struct ck_ht_map *map, + ck_ht_hash_t h, + ck_ht_entry_t *snapshot, + const void *key, + uint16_t key_length) +{ + struct ck_ht_entry *bucket, *cursor; + size_t offset, i, j; + CK_HT_TYPE probes = 0; + CK_HT_TYPE probe_maximum; + +#ifndef CK_HT_PP + CK_HT_TYPE d = 0; + CK_HT_TYPE d_prime = 0; +retry: +#endif + + probe_maximum = ck_ht_map_bound_get(map, h); + offset = h.value & map->mask; + + for (i = 0; i < map->probe_limit; i++) { + /* + * Probe on a complete cache line first. Scan forward and wrap around to + * the beginning of the cache line. Only when the complete cache line has + * been scanned do we move on to the next row. + */ + bucket = (void *)((uintptr_t)(map->entries + offset) & + ~(CK_MD_CACHELINE - 1)); + + for (j = 0; j < CK_HT_BUCKET_LENGTH; j++) { + uint16_t k; + + if (probes++ > probe_maximum) + return NULL; + + cursor = bucket + ((j + offset) & (CK_HT_BUCKET_LENGTH - 1)); + +#ifdef CK_HT_PP + snapshot->key = (uintptr_t)ck_pr_load_ptr(&cursor->key); + ck_pr_fence_load(); + snapshot->value = (uintptr_t)ck_pr_load_ptr(&cursor->value); +#else + d = CK_HT_TYPE_LOAD(&map->deletions); + snapshot->key = (uintptr_t)ck_pr_load_ptr(&cursor->key); + ck_pr_fence_load(); + snapshot->key_length = CK_HT_TYPE_LOAD(&cursor->key_length); + snapshot->hash = CK_HT_TYPE_LOAD(&cursor->hash); + snapshot->value = (uintptr_t)ck_pr_load_ptr(&cursor->value); +#endif + + /* + * It is probably worth it to encapsulate probe state + * in order to prevent a complete reprobe sequence in + * the case of intermittent writers. + */ + if (snapshot->key == CK_HT_KEY_TOMBSTONE) + continue; + + if (snapshot->key == CK_HT_KEY_EMPTY) + goto leave; + + if (snapshot->key == (uintptr_t)key) + goto leave; + + if (map->mode & CK_HT_MODE_BYTESTRING) { + void *pointer; + + /* + * Check memoized portion of hash value before + * expensive full-length comparison. + */ + k = ck_ht_entry_key_length(snapshot); + if (k != key_length) + continue; +#ifdef CK_HT_PP + if ((snapshot->value >> CK_MD_VMA_BITS) != ((h.value >> 32) & CK_HT_KEY_MASK)) + continue; +#else + if (snapshot->hash != h.value) + continue; + + d_prime = CK_HT_TYPE_LOAD(&map->deletions); + + /* + * It is possible that the slot was + * replaced, initiate a re-probe. + */ + if (d != d_prime) + goto retry; +#endif + + pointer = ck_ht_entry_key(snapshot); + if (memcmp(pointer, key, key_length) == 0) + goto leave; + } + } + + offset = ck_ht_map_probe_next(map, offset, h, probes); + } + + return NULL; + +leave: + return cursor; +} + +CK_HT_TYPE +ck_ht_count(struct ck_ht *table) +{ + struct ck_ht_map *map = ck_pr_load_ptr(&table->map); + + return CK_HT_TYPE_LOAD(&map->n_entries); +} + +bool +ck_ht_next(struct ck_ht *table, + struct ck_ht_iterator *i, + struct ck_ht_entry **entry) +{ + struct ck_ht_map *map = table->map; + uintptr_t key; + + if (i->offset >= map->capacity) + return false; + + do { + key = map->entries[i->offset].key; + if (key != CK_HT_KEY_EMPTY && key != CK_HT_KEY_TOMBSTONE) + break; + } while (++i->offset < map->capacity); + + if (i->offset >= map->capacity) + return false; + + *entry = map->entries + i->offset++; + return true; +} + +bool +ck_ht_reset_size_spmc(struct ck_ht *table, CK_HT_TYPE size) +{ + struct ck_ht_map *map, *update; + + map = table->map; + update = ck_ht_map_create(table, size); + if (update == NULL) + return false; + + ck_pr_store_ptr_unsafe(&table->map, update); + ck_ht_map_destroy(table->m, map, true); + return true; +} + +bool +ck_ht_reset_spmc(struct ck_ht *table) +{ + struct ck_ht_map *map = table->map; + + return ck_ht_reset_size_spmc(table, map->capacity); +} + +bool +ck_ht_grow_spmc(struct ck_ht *table, CK_HT_TYPE capacity) +{ + struct ck_ht_map *map, *update; + struct ck_ht_entry *bucket, *previous; + struct ck_ht_hash h; + size_t k, i, j, offset; + CK_HT_TYPE probes; + +restart: + map = table->map; + + if (map->capacity >= capacity) + return false; + + update = ck_ht_map_create(table, capacity); + if (update == NULL) + return false; + + for (k = 0; k < map->capacity; k++) { + previous = &map->entries[k]; + + if (previous->key == CK_HT_KEY_EMPTY || previous->key == CK_HT_KEY_TOMBSTONE) + continue; + + if (table->mode & CK_HT_MODE_BYTESTRING) { +#ifdef CK_HT_PP + void *key; + uint16_t key_length; + + key = ck_ht_entry_key(previous); + key_length = ck_ht_entry_key_length(previous); +#endif + +#ifndef CK_HT_PP + h.value = previous->hash; +#else + table->h(&h, key, key_length, table->seed); +#endif + } else { +#ifndef CK_HT_PP + h.value = previous->hash; +#else + table->h(&h, &previous->key, sizeof(previous->key), table->seed); +#endif + } + + offset = h.value & update->mask; + probes = 0; + + for (i = 0; i < update->probe_limit; i++) { + bucket = (void *)((uintptr_t)(update->entries + offset) & ~(CK_MD_CACHELINE - 1)); + + for (j = 0; j < CK_HT_BUCKET_LENGTH; j++) { + struct ck_ht_entry *cursor = bucket + ((j + offset) & (CK_HT_BUCKET_LENGTH - 1)); + + probes++; + if (CK_CC_LIKELY(cursor->key == CK_HT_KEY_EMPTY)) { + *cursor = *previous; + update->n_entries++; + ck_ht_map_bound_set(update, h, probes); + break; + } + } + + if (j < CK_HT_BUCKET_LENGTH) + break; + + offset = ck_ht_map_probe_next(update, offset, h, probes); + } + + if (i == update->probe_limit) { + /* + * We have hit the probe limit, the map needs to be even + * larger. + */ + ck_ht_map_destroy(table->m, update, false); + capacity <<= 1; + goto restart; + } + } + + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&table->map, update); + ck_ht_map_destroy(table->m, map, true); + return true; +} + +bool +ck_ht_remove_spmc(struct ck_ht *table, + ck_ht_hash_t h, + ck_ht_entry_t *entry) +{ + struct ck_ht_map *map; + struct ck_ht_entry *candidate, snapshot; + + map = table->map; + + if (table->mode & CK_HT_MODE_BYTESTRING) { + candidate = ck_ht_map_probe_rd(map, h, &snapshot, + ck_ht_entry_key(entry), + ck_ht_entry_key_length(entry)); + } else { + candidate = ck_ht_map_probe_rd(map, h, &snapshot, + (void *)entry->key, + sizeof(entry->key)); + } + + /* No matching entry was found. */ + if (candidate == NULL || snapshot.key == CK_HT_KEY_EMPTY) + return false; + + *entry = snapshot; + + ck_pr_store_ptr_unsafe(&candidate->key, (void *)CK_HT_KEY_TOMBSTONE); + ck_pr_fence_store(); + CK_HT_TYPE_STORE(&map->n_entries, map->n_entries - 1); + return true; +} + +bool +ck_ht_get_spmc(struct ck_ht *table, + ck_ht_hash_t h, + ck_ht_entry_t *entry) +{ + struct ck_ht_entry *candidate, snapshot; + struct ck_ht_map *map; + CK_HT_TYPE d, d_prime; + +restart: + map = ck_pr_load_ptr(&table->map); + + /* + * Platforms that cannot read key and key_length atomically must reprobe + * on the scan of any single entry. + */ + d = CK_HT_TYPE_LOAD(&map->deletions); + + if (table->mode & CK_HT_MODE_BYTESTRING) { + candidate = ck_ht_map_probe_rd(map, h, &snapshot, + ck_ht_entry_key(entry), ck_ht_entry_key_length(entry)); + } else { + candidate = ck_ht_map_probe_rd(map, h, &snapshot, + (void *)entry->key, sizeof(entry->key)); + } + + d_prime = CK_HT_TYPE_LOAD(&map->deletions); + if (d != d_prime) { + /* + * It is possible we have read (K, V'). Only valid states are + * (K, V), (K', V') and (T, V). Restart load operation in face + * of concurrent deletions or replacements. + */ + goto restart; + } + + if (candidate == NULL || snapshot.key == CK_HT_KEY_EMPTY) + return false; + + *entry = snapshot; + return true; +} + +bool +ck_ht_set_spmc(struct ck_ht *table, + ck_ht_hash_t h, + ck_ht_entry_t *entry) +{ + struct ck_ht_entry snapshot, *candidate, *priority; + struct ck_ht_map *map; + CK_HT_TYPE probes, probes_wr; + bool empty = false; + + for (;;) { + map = table->map; + + if (table->mode & CK_HT_MODE_BYTESTRING) { + candidate = ck_ht_map_probe_wr(map, h, &snapshot, &priority, + ck_ht_entry_key(entry), + ck_ht_entry_key_length(entry), + &probes, &probes_wr); + } else { + candidate = ck_ht_map_probe_wr(map, h, &snapshot, &priority, + (void *)entry->key, + sizeof(entry->key), + &probes, &probes_wr); + } + + if (priority != NULL) { + probes = probes_wr; + break; + } + + if (candidate != NULL) + break; + + if (ck_ht_grow_spmc(table, map->capacity << 1) == false) + return false; + } + + if (candidate == NULL) { + candidate = priority; + empty = true; + } + + if (candidate->key != CK_HT_KEY_EMPTY && + priority != NULL && candidate != priority) { + /* + * Entry is moved into another position in probe sequence. + * We avoid a state of (K, B) (where [K, B] -> [K', B]) by + * guaranteeing a forced reprobe before transitioning from K to + * T. (K, B) implies (K, B, D') so we will reprobe successfully + * from this transient state. + */ + probes = probes_wr; + +#ifndef CK_HT_PP + CK_HT_TYPE_STORE(&priority->key_length, entry->key_length); + CK_HT_TYPE_STORE(&priority->hash, entry->hash); +#endif + + /* + * Readers must observe version counter change before they + * observe re-use. If they observe re-use, it is at most + * a tombstone. + */ + if (priority->value == CK_HT_KEY_TOMBSTONE) { + CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1); + ck_pr_fence_store(); + } + + ck_pr_store_ptr_unsafe(&priority->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&priority->key, (void *)entry->key); + ck_pr_fence_store(); + + /* + * Make sure that readers who observe the tombstone would + * also observe counter change. + */ + CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1); + ck_pr_fence_store(); + + ck_pr_store_ptr_unsafe(&candidate->key, (void *)CK_HT_KEY_TOMBSTONE); + ck_pr_fence_store(); + } else { + /* + * In this case we are inserting a new entry or replacing + * an existing entry. Yes, this can be combined into above branch, + * but isn't because you are actually looking at dying code + * (ck_ht is effectively deprecated and is being replaced soon). + */ + bool replace = candidate->key != CK_HT_KEY_EMPTY && + candidate->key != CK_HT_KEY_TOMBSTONE; + + if (priority != NULL) { + if (priority->key == CK_HT_KEY_TOMBSTONE) { + CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1); + ck_pr_fence_store(); + } + + candidate = priority; + probes = probes_wr; + } + +#ifdef CK_HT_PP + ck_pr_store_ptr_unsafe(&candidate->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key); +#else + CK_HT_TYPE_STORE(&candidate->key_length, entry->key_length); + CK_HT_TYPE_STORE(&candidate->hash, entry->hash); + ck_pr_store_ptr_unsafe(&candidate->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key); +#endif + + /* + * If we are insert a new entry then increment number + * of entries associated with map. + */ + if (replace == false) + CK_HT_TYPE_STORE(&map->n_entries, map->n_entries + 1); + } + + ck_ht_map_bound_set(map, h, probes); + + /* Enforce a load factor of 0.5. */ + if (map->n_entries * 2 > map->capacity) + ck_ht_grow_spmc(table, map->capacity << 1); + + if (empty == true) { + entry->key = CK_HT_KEY_EMPTY; + } else { + *entry = snapshot; + } + + return true; +} + +bool +ck_ht_put_spmc(struct ck_ht *table, + ck_ht_hash_t h, + ck_ht_entry_t *entry) +{ + struct ck_ht_entry snapshot, *candidate, *priority; + struct ck_ht_map *map; + CK_HT_TYPE probes, probes_wr; + + for (;;) { + map = table->map; + + if (table->mode & CK_HT_MODE_BYTESTRING) { + candidate = ck_ht_map_probe_wr(map, h, &snapshot, &priority, + ck_ht_entry_key(entry), + ck_ht_entry_key_length(entry), + &probes, &probes_wr); + } else { + candidate = ck_ht_map_probe_wr(map, h, &snapshot, &priority, + (void *)entry->key, + sizeof(entry->key), + &probes, &probes_wr); + } + + if (candidate != NULL || priority != NULL) + break; + + if (ck_ht_grow_spmc(table, map->capacity << 1) == false) + return false; + } + + if (priority != NULL) { + /* Version counter is updated before re-use. */ + CK_HT_TYPE_STORE(&map->deletions, map->deletions + 1); + ck_pr_fence_store(); + + /* Re-use tombstone if one was found. */ + candidate = priority; + probes = probes_wr; + } else if (candidate->key != CK_HT_KEY_EMPTY && + candidate->key != CK_HT_KEY_TOMBSTONE) { + /* + * If the snapshot key is non-empty and the value field is not + * a tombstone then an identical key was found. As store does + * not implement replacement, we will fail. + */ + return false; + } + + ck_ht_map_bound_set(map, h, probes); + +#ifdef CK_HT_PP + ck_pr_store_ptr_unsafe(&candidate->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key); +#else + CK_HT_TYPE_STORE(&candidate->key_length, entry->key_length); + CK_HT_TYPE_STORE(&candidate->hash, entry->hash); + ck_pr_store_ptr_unsafe(&candidate->value, (void *)entry->value); + ck_pr_fence_store(); + ck_pr_store_ptr_unsafe(&candidate->key, (void *)entry->key); +#endif + + CK_HT_TYPE_STORE(&map->n_entries, map->n_entries + 1); + + /* Enforce a load factor of 0.5. */ + if (map->n_entries * 2 > map->capacity) + ck_ht_grow_spmc(table, map->capacity << 1); + + return true; +} + +void +ck_ht_destroy(struct ck_ht *table) +{ + + ck_ht_map_destroy(table->m, table->map, false); + return; +} Property changes on: vendor-sys/ck/20161230/src/ck_ht.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_ht_hash.h =================================================================== --- vendor-sys/ck/20161230/src/ck_ht_hash.h (nonexistent) +++ vendor-sys/ck/20161230/src/ck_ht_hash.h (revision 310844) @@ -0,0 +1,269 @@ +/* + * Copyright 2012-2015 Samy Al Bahra + * Copyright 2011-2014 AppNexus, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef CK_HT_HASH_H +#define CK_HT_HASH_H + +/* + * This is the Murmur hash written by Austin Appleby. + */ + +#include +#include + +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +#define FORCE_INLINE __forceinline + +#include + +#define ROTL32(x,y) _rotl(x,y) +#define ROTL64(x,y) _rotl64(x,y) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#define FORCE_INLINE inline __attribute__((always_inline)) + +static inline uint32_t rotl32 ( uint32_t x, int8_t r ) +{ + return (x << r) | (x >> (32 - r)); +} + +static inline uint64_t rotl64 ( uint64_t x, int8_t r ) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) +#define ROTL64(x,y) rotl64(x,y) + +#define BIG_CONSTANT(x) (x##LLU) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +FORCE_INLINE static uint32_t getblock ( const uint32_t * p, int i ) +{ + return p[i]; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +FORCE_INLINE static uint32_t fmix ( uint32_t h ) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +//----------------------------------------------------------------------------- + +static inline void MurmurHash3_x86_32 ( const void * key, int len, + uint32_t seed, uint32_t * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 4; + int i; + + uint32_t h1 = seed; + + uint32_t c1 = 0xcc9e2d51; + uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(const void *)(data + nblocks*4); + + for(i = -nblocks; i; i++) + { + uint32_t k1 = getblock(blocks,i); + + k1 *= c1; + k1 = ROTL32(k1,15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1,13); + h1 = h1*5+0xe6546b64; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*4); + + uint32_t k1 = 0; + + switch(len & 3) + { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h1 = fmix(h1); + + *(uint32_t *)out = h1; +} + +static inline uint64_t MurmurHash64A ( const void * key, int len, uint64_t seed ) +{ + const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); + const int r = 47; + + uint64_t h = seed ^ (len * m); + + const uint64_t * data = (const uint64_t *)key; + const uint64_t * end = data + (len/8); + + while(data != end) + { + uint64_t k; + + if (!((uintptr_t)data & 0x7)) + k = *data++; + else { + memcpy(&k, data, sizeof(k)); + data++; + } + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + const unsigned char * data2 = (const unsigned char*)data; + + switch(len & 7) + { + case 7: h ^= (uint64_t)(data2[6]) << 48; + case 6: h ^= (uint64_t)(data2[5]) << 40; + case 5: h ^= (uint64_t)(data2[4]) << 32; + case 4: h ^= (uint64_t)(data2[3]) << 24; + case 3: h ^= (uint64_t)(data2[2]) << 16; + case 2: h ^= (uint64_t)(data2[1]) << 8; + case 1: h ^= (uint64_t)(data2[0]); + h *= m; + }; + + h ^= h >> r; + h *= m; + h ^= h >> r; + + return h; +} + + +// 64-bit hash for 32-bit platforms + +static inline uint64_t MurmurHash64B ( const void * key, int len, uint64_t seed ) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + + uint32_t h1 = (uint32_t)(seed) ^ len; + uint32_t h2 = (uint32_t)(seed >> 32); + + const uint32_t * data = (const uint32_t *)key; + + while(len >= 8) + { + uint32_t k1 = *data++; + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + + uint32_t k2 = *data++; + k2 *= m; k2 ^= k2 >> r; k2 *= m; + h2 *= m; h2 ^= k2; + len -= 4; + } + + if(len >= 4) + { + uint32_t k1 = *data++; + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + } + + switch(len) + { + case 3: h2 ^= ((const unsigned char*)data)[2] << 16; + case 2: h2 ^= ((const unsigned char*)data)[1] << 8; + case 1: h2 ^= ((const unsigned char*)data)[0]; + h2 *= m; + }; + + h1 ^= h2 >> 18; h1 *= m; + h2 ^= h1 >> 22; h2 *= m; + h1 ^= h2 >> 17; h1 *= m; + h2 ^= h1 >> 19; h2 *= m; + + uint64_t h = h1; + + h = (h << 32) | h2; + + return h; +} + +#endif /* CK_HT_HASH_H */ Property changes on: vendor-sys/ck/20161230/src/ck_ht_hash.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_internal.h =================================================================== --- vendor-sys/ck/20161230/src/ck_internal.h (nonexistent) +++ vendor-sys/ck/20161230/src/ck_internal.h (revision 310844) @@ -0,0 +1,119 @@ +/* + * Copyright 2011-2015 Samy Al Bahra. + * Copyright 2011 David Joseph. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Several of these are from: http://graphics.stanford.edu/~seander/bithacks.html + */ + +#define CK_INTERNAL_LOG_0 (0xAAAAAAAA) +#define CK_INTERNAL_LOG_1 (0xCCCCCCCC) +#define CK_INTERNAL_LOG_2 (0xF0F0F0F0) +#define CK_INTERNAL_LOG_3 (0xFF00FF00) +#define CK_INTERNAL_LOG_4 (0xFFFF0000) + +CK_CC_INLINE static uint32_t +ck_internal_log(uint32_t v) +{ + uint32_t r = (v & CK_INTERNAL_LOG_0) != 0; + + r |= ((v & CK_INTERNAL_LOG_4) != 0) << 4; + r |= ((v & CK_INTERNAL_LOG_3) != 0) << 3; + r |= ((v & CK_INTERNAL_LOG_2) != 0) << 2; + r |= ((v & CK_INTERNAL_LOG_1) != 0) << 1; + return (r); +} + +CK_CC_INLINE static uint32_t +ck_internal_power_2(uint32_t v) +{ + + --v; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return (++v); +} + +CK_CC_INLINE static unsigned long +ck_internal_max(unsigned long x, unsigned long y) +{ + + return x ^ ((x ^ y) & -(x < y)); +} + +CK_CC_INLINE static uint64_t +ck_internal_max_64(uint64_t x, uint64_t y) +{ + + return x ^ ((x ^ y) & -(x < y)); +} + +CK_CC_INLINE static uint32_t +ck_internal_max_32(uint32_t x, uint32_t y) +{ + + return x ^ ((x ^ y) & -(x < y)); +} + +CK_CC_INLINE static unsigned long +ck_internal_bsf(unsigned long v) +{ +#if defined(__GNUC__) + return __builtin_ffs(v); +#else + unsigned int i; + const unsigned int s = sizeof(unsigned long) * 8 - 1; + + for (i = 0; i < s; i++) { + if (v & (1UL << (s - i))) + return sizeof(unsigned long) * 8 - i; + } + + return 1; +#endif /* !__GNUC__ */ +} + +CK_CC_INLINE static uint64_t +ck_internal_bsf_64(uint64_t v) +{ +#if defined(__GNUC__) + return __builtin_ffs(v); +#else + unsigned int i; + const unsigned int s = sizeof(unsigned long) * 8 - 1; + + for (i = 0; i < s; i++) { + if (v & (1ULL << (63U - i))) + return i; + } +#endif /* !__GNUC__ */ + + return 1; +} + Property changes on: vendor-sys/ck/20161230/src/ck_internal.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: vendor-sys/ck/20161230/src/ck_rhs.c =================================================================== --- vendor-sys/ck/20161230/src/ck_rhs.c (nonexistent) +++ vendor-sys/ck/20161230/src/ck_rhs.c (revision 310844) @@ -0,0 +1,1480 @@ +/* + * Copyright 2014-2015 Olivier Houchard. + * Copyright 2012-2015 Samy Al Bahra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ck_internal.h" + +#ifndef CK_RHS_PROBE_L1_SHIFT +#define CK_RHS_PROBE_L1_SHIFT 3ULL +#endif /* CK_RHS_PROBE_L1_SHIFT */ + +#define CK_RHS_PROBE_L1 (1 << CK_RHS_PROBE_L1_SHIFT) +#define CK_RHS_PROBE_L1_MASK (CK_RHS_PROBE_L1 - 1) + +#ifndef CK_RHS_PROBE_L1_DEFAULT +#define CK_RHS_PROBE_L1_DEFAULT CK_MD_CACHELINE +#endif + +#define CK_RHS_VMA_MASK ((uintptr_t)((1ULL << CK_MD_VMA_BITS) - 1)) +#define CK_RHS_VMA(x) \ + ((void *)((uintptr_t)(x) & CK_RHS_VMA_MASK)) + +#define CK_RHS_EMPTY NULL +#define CK_RHS_G (1024) +#define CK_RHS_G_MASK (CK_RHS_G - 1) + +#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_STORE_8) +#define CK_RHS_WORD uint8_t +#define CK_RHS_WORD_MAX UINT8_MAX +#define CK_RHS_STORE(x, y) ck_pr_store_8(x, y) +#define CK_RHS_LOAD(x) ck_pr_load_8(x) +#elif defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_STORE_16) +#define CK_RHS_WORD uint16_t +#define CK_RHS_WORD_MAX UINT16_MAX +#define CK_RHS_STORE(x, y) ck_pr_store_16(x, y) +#define CK_RHS_LOAD(x) ck_pr_load_16(x) +#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32) +#define CK_RHS_WORD uint32_t +#define CK_RHS_WORD_MAX UINT32_MAX +#define CK_RHS_STORE(x, y) ck_pr_store_32(x, y) +#define CK_RHS_LOAD(x) ck_pr_load_32(x) +#else +#error "ck_rhs is not supported on your platform." +#endif + +#define CK_RHS_MAX_WANTED 0xffff + +enum ck_rhs_probe_behavior { + CK_RHS_PROBE = 0, /* Default behavior. */ + CK_RHS_PROBE_RH, /* Short-circuit if RH slot found. */ + CK_RHS_PROBE_INSERT, /* Short-circuit on probe bound if tombstone found. */ + + CK_RHS_PROBE_ROBIN_HOOD,/* Look for the first slot available for the entry we are about to replace, only used to internally implement Robin Hood */ + CK_RHS_PROBE_NO_RH, /* Don't do the RH dance */ +}; +struct ck_rhs_entry_desc { + unsigned int probes; + unsigned short wanted; + CK_RHS_WORD probe_bound; + bool in_rh; + const void *entry; +} CK_CC_ALIGN(16); + +struct ck_rhs_no_entry_desc { + unsigned int probes; + unsigned short wanted; + CK_RHS_WORD probe_bound; + bool in_rh; +} CK_CC_ALIGN(8); + +typedef long ck_rhs_probe_cb_t(struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + long *priority, + unsigned long h, + const void *key, + const void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior); + +struct ck_rhs_map { + unsigned int generation[CK_RHS_G]; + unsigned int probe_maximum; + unsigned long mask; + unsigned long step; + unsigned int probe_limit; + unsigned long n_entries; + unsigned long capacity; + unsigned long size; + unsigned long max_entries; + char offset_mask; + union { + struct ck_rhs_entry_desc *descs; + struct ck_rhs_no_entry { + const void **entries; + struct ck_rhs_no_entry_desc *descs; + } no_entries; + } entries; + bool read_mostly; + ck_rhs_probe_cb_t *probe_func; +}; + +static CK_CC_INLINE const void * +ck_rhs_entry(struct ck_rhs_map *map, long offset) +{ + + if (map->read_mostly) + return (map->entries.no_entries.entries[offset]); + else + return (map->entries.descs[offset].entry); +} + +static CK_CC_INLINE const void ** +ck_rhs_entry_addr(struct ck_rhs_map *map, long offset) +{ + + if (map->read_mostly) + return (&map->entries.no_entries.entries[offset]); + else + return (&map->entries.descs[offset].entry); +} + +static CK_CC_INLINE struct ck_rhs_entry_desc * +ck_rhs_desc(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return ((struct ck_rhs_entry_desc *)(void *)&map->entries.no_entries.descs[offset]); + else + return (&map->entries.descs[offset]); +} + +static CK_CC_INLINE void +ck_rhs_wanted_inc(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].wanted++; + else + map->entries.descs[offset].wanted++; +} + +static CK_CC_INLINE unsigned int +ck_rhs_probes(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (map->entries.no_entries.descs[offset].probes); + else + return (map->entries.descs[offset].probes); +} + +static CK_CC_INLINE void +ck_rhs_set_probes(struct ck_rhs_map *map, long offset, unsigned int value) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].probes = value; + else + map->entries.descs[offset].probes = value; +} + +static CK_CC_INLINE CK_RHS_WORD +ck_rhs_probe_bound(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (map->entries.no_entries.descs[offset].probe_bound); + else + return (map->entries.descs[offset].probe_bound); +} + +static CK_CC_INLINE CK_RHS_WORD * +ck_rhs_probe_bound_addr(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (&map->entries.no_entries.descs[offset].probe_bound); + else + return (&map->entries.descs[offset].probe_bound); +} + + +static CK_CC_INLINE bool +ck_rhs_in_rh(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + return (map->entries.no_entries.descs[offset].in_rh); + else + return (map->entries.descs[offset].in_rh); +} + +static CK_CC_INLINE void +ck_rhs_set_rh(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].in_rh = true; + else + map->entries.descs[offset].in_rh = true; +} + +static CK_CC_INLINE void +ck_rhs_unset_rh(struct ck_rhs_map *map, long offset) +{ + + if (CK_CC_UNLIKELY(map->read_mostly)) + map->entries.no_entries.descs[offset].in_rh = false; + else + map->entries.descs[offset].in_rh = false; +} + + +#define CK_RHS_DEFAULT_LOAD_FACTOR 50 + +static ck_rhs_probe_cb_t ck_rhs_map_probe; +static ck_rhs_probe_cb_t ck_rhs_map_probe_rm; + +bool +ck_rhs_set_load_factor(struct ck_rhs *hs, unsigned int load_factor) +{ + struct ck_rhs_map *map = hs->map; + + if (load_factor == 0 || load_factor > 100) + return false; + + hs->load_factor = load_factor; + map->max_entries = (map->capacity * (unsigned long)hs->load_factor) / 100; + while (map->n_entries > map->max_entries) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) + return false; + map = hs->map; + } + return true; +} + +void +ck_rhs_iterator_init(struct ck_rhs_iterator *iterator) +{ + + iterator->cursor = NULL; + iterator->offset = 0; + return; +} + +bool +ck_rhs_next(struct ck_rhs *hs, struct ck_rhs_iterator *i, void **key) +{ + struct ck_rhs_map *map = hs->map; + void *value; + + if (i->offset >= map->capacity) + return false; + + do { + value = CK_CC_DECONST_PTR(ck_rhs_entry(map, i->offset)); + if (value != CK_RHS_EMPTY) { +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + value = CK_RHS_VMA(value); +#endif + i->offset++; + *key = value; + return true; + } + } while (++i->offset < map->capacity); + + return false; +} + +void +ck_rhs_stat(struct ck_rhs *hs, struct ck_rhs_stat *st) +{ + struct ck_rhs_map *map = hs->map; + + st->n_entries = map->n_entries; + st->probe_maximum = map->probe_maximum; + return; +} + +unsigned long +ck_rhs_count(struct ck_rhs *hs) +{ + + return hs->map->n_entries; +} + +static void +ck_rhs_map_destroy(struct ck_malloc *m, struct ck_rhs_map *map, bool defer) +{ + + m->free(map, map->size, defer); + return; +} + +void +ck_rhs_destroy(struct ck_rhs *hs) +{ + + ck_rhs_map_destroy(hs->m, hs->map, false); + return; +} + +static struct ck_rhs_map * +ck_rhs_map_create(struct ck_rhs *hs, unsigned long entries) +{ + struct ck_rhs_map *map; + unsigned long size, n_entries, limit; + + n_entries = ck_internal_power_2(entries); + if (n_entries < CK_RHS_PROBE_L1) + n_entries = CK_RHS_PROBE_L1; + + if (hs->mode & CK_RHS_MODE_READ_MOSTLY) + size = sizeof(struct ck_rhs_map) + + (sizeof(void *) * n_entries + + sizeof(struct ck_rhs_no_entry_desc) * n_entries + + 2 * CK_MD_CACHELINE - 1); + else + size = sizeof(struct ck_rhs_map) + + (sizeof(struct ck_rhs_entry_desc) * n_entries + + CK_MD_CACHELINE - 1); + map = hs->m->malloc(size); + if (map == NULL) + return NULL; + map->read_mostly = !!(hs->mode & CK_RHS_MODE_READ_MOSTLY); + + map->size = size; + /* We should probably use a more intelligent heuristic for default probe length. */ + limit = ck_internal_max(n_entries >> (CK_RHS_PROBE_L1_SHIFT + 2), CK_RHS_PROBE_L1_DEFAULT); + if (limit > UINT_MAX) + limit = UINT_MAX; + + map->probe_limit = (unsigned int)limit; + map->probe_maximum = 0; + map->capacity = n_entries; + map->step = ck_internal_bsf(n_entries); + map->mask = n_entries - 1; + map->n_entries = 0; + + map->max_entries = (map->capacity * (unsigned long)hs->load_factor) / 100; + /* Align map allocation to cache line. */ + if (map->read_mostly) { + map->entries.no_entries.entries = (void *)(((uintptr_t)&map[1] + + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + map->entries.no_entries.descs = (void *)(((uintptr_t)map->entries.no_entries.entries + (sizeof(void *) * n_entries) + CK_MD_CACHELINE - 1) &~ (CK_MD_CACHELINE - 1)); + memset(map->entries.no_entries.entries, 0, + sizeof(void *) * n_entries); + memset(map->entries.no_entries.descs, 0, + sizeof(struct ck_rhs_no_entry_desc) * n_entries); + map->offset_mask = (CK_MD_CACHELINE / sizeof(void *)) - 1; + map->probe_func = ck_rhs_map_probe_rm; + + } else { + map->entries.descs = (void *)(((uintptr_t)&map[1] + + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); + memset(map->entries.descs, 0, sizeof(struct ck_rhs_entry_desc) * n_entries); + map->offset_mask = (CK_MD_CACHELINE / sizeof(struct ck_rhs_entry_desc)) - 1; + map->probe_func = ck_rhs_map_probe; + } + memset(map->generation, 0, sizeof map->generation); + + /* Commit entries purge with respect to map publication. */ + ck_pr_fence_store(); + return map; +} + +bool +ck_rhs_reset_size(struct ck_rhs *hs, unsigned long capacity) +{ + struct ck_rhs_map *map, *previous; + + previous = hs->map; + map = ck_rhs_map_create(hs, capacity); + if (map == NULL) + return false; + + ck_pr_store_ptr(&hs->map, map); + ck_rhs_map_destroy(hs->m, previous, true); + return true; +} + +bool +ck_rhs_reset(struct ck_rhs *hs) +{ + struct ck_rhs_map *previous; + + previous = hs->map; + return ck_rhs_reset_size(hs, previous->capacity); +} + +static inline unsigned long +ck_rhs_map_probe_next(struct ck_rhs_map *map, + unsigned long offset, + unsigned long probes) +{ + + if (probes & map->offset_mask) { + offset = (offset &~ map->offset_mask) + + ((offset + 1) & map->offset_mask); + return offset; + } else + return (offset + probes) & map->mask; +} + +static inline unsigned long +ck_rhs_map_probe_prev(struct ck_rhs_map *map, unsigned long offset, + unsigned long probes) +{ + + if (probes & map->offset_mask) { + offset = (offset &~ map->offset_mask) + ((offset - 1) & + map->offset_mask); + return offset; + } else + return ((offset - probes) & map->mask); +} + + +static inline void +ck_rhs_map_bound_set(struct ck_rhs_map *m, + unsigned long h, + unsigned long n_probes) +{ + unsigned long offset = h & m->mask; + struct ck_rhs_entry_desc *desc; + + if (n_probes > m->probe_maximum) + ck_pr_store_uint(&m->probe_maximum, n_probes); + if (!(m->read_mostly)) { + desc = &m->entries.descs[offset]; + + if (desc->probe_bound < n_probes) { + if (n_probes > CK_RHS_WORD_MAX) + n_probes = CK_RHS_WORD_MAX; + + CK_RHS_STORE(&desc->probe_bound, n_probes); + ck_pr_fence_store(); + } + } + + return; +} + +static inline unsigned int +ck_rhs_map_bound_get(struct ck_rhs_map *m, unsigned long h) +{ + unsigned long offset = h & m->mask; + unsigned int r = CK_RHS_WORD_MAX; + + if (m->read_mostly) + r = ck_pr_load_uint(&m->probe_maximum); + else { + r = CK_RHS_LOAD(&m->entries.descs[offset].probe_bound); + if (r == CK_RHS_WORD_MAX) + r = ck_pr_load_uint(&m->probe_maximum); + } + return r; +} + +bool +ck_rhs_grow(struct ck_rhs *hs, + unsigned long capacity) +{ + struct ck_rhs_map *map, *update; + const void *previous, *prev_saved; + unsigned long k, offset, probes; + +restart: + map = hs->map; + if (map->capacity > capacity) + return false; + + update = ck_rhs_map_create(hs, capacity); + if (update == NULL) + return false; + + for (k = 0; k < map->capacity; k++) { + unsigned long h; + + prev_saved = previous = ck_rhs_entry(map, k); + if (previous == CK_RHS_EMPTY) + continue; + +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + previous = CK_RHS_VMA(previous); +#endif + + h = hs->hf(previous, hs->seed); + offset = h & update->mask; + probes = 0; + + for (;;) { + const void **cursor = ck_rhs_entry_addr(update, offset); + + if (probes++ == update->probe_limit) { + /* + * We have hit the probe limit, map needs to be even larger. + */ + ck_rhs_map_destroy(hs->m, update, false); + capacity <<= 1; + goto restart; + } + + if (CK_CC_LIKELY(*cursor == CK_RHS_EMPTY)) { + *cursor = prev_saved; + update->n_entries++; + ck_rhs_set_probes(update, offset, probes); + ck_rhs_map_bound_set(update, h, probes); + break; + } else if (ck_rhs_probes(update, offset) < probes) { + const void *tmp = prev_saved; + unsigned int old_probes; + prev_saved = previous = *cursor; +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + previous = CK_RHS_VMA(previous); +#endif + *cursor = tmp; + ck_rhs_map_bound_set(update, h, probes); + h = hs->hf(previous, hs->seed); + old_probes = ck_rhs_probes(update, offset); + ck_rhs_set_probes(update, offset, probes); + probes = old_probes - 1; + continue; + } + ck_rhs_wanted_inc(update, offset); + offset = ck_rhs_map_probe_next(update, offset, probes); + } + + } + + ck_pr_fence_store(); + ck_pr_store_ptr(&hs->map, update); + ck_rhs_map_destroy(hs->m, map, true); + return true; +} + +bool +ck_rhs_rebuild(struct ck_rhs *hs) +{ + + return ck_rhs_grow(hs, hs->map->capacity); +} + +static long +ck_rhs_map_probe_rm(struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + long *priority, + unsigned long h, + const void *key, + const void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior) +{ + const void *k; + const void *compare; + long pr = -1; + unsigned long offset, probes, opl; + +#ifdef CK_RHS_PP + /* If we are storing object pointers, then we may leverage pointer packing. */ + unsigned long hv = 0; + + if (hs->mode & CK_RHS_MODE_OBJECT) { + hv = (h >> 25) & CK_RHS_KEY_MASK; + compare = CK_RHS_VMA(key); + } else { + compare = key; + } +#else + compare = key; +#endif + *object = NULL; + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { + probes = 0; + offset = h & map->mask; + } else { + /* Restart from the bucket we were previously in */ + probes = *n_probes; + offset = ck_rhs_map_probe_next(map, *priority, + probes); + } + opl = probe_limit; + + for (;;) { + if (probes++ == probe_limit) { + if (probe_limit == opl || pr != -1) { + k = CK_RHS_EMPTY; + goto leave; + } + /* + * If no eligible slot has been found yet, continue probe + * sequence with original probe limit. + */ + probe_limit = opl; + } + k = ck_pr_load_ptr(&map->entries.no_entries.entries[offset]); + if (k == CK_RHS_EMPTY) + goto leave; + + if (behavior != CK_RHS_PROBE_NO_RH) { + struct ck_rhs_entry_desc *desc = (void *)&map->entries.no_entries.descs[offset]; + + if (pr == -1 && + desc->in_rh == false && desc->probes < probes) { + pr = offset; + *n_probes = probes; + + if (behavior == CK_RHS_PROBE_RH || + behavior == CK_RHS_PROBE_ROBIN_HOOD) { + k = CK_RHS_EMPTY; + goto leave; + } + } + } + + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) { + if (((uintptr_t)k >> CK_MD_VMA_BITS) != hv) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + k = CK_RHS_VMA(k); + } +#endif + + if (k == compare) + goto leave; + + if (hs->compare == NULL) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + if (hs->compare(k, key) == true) + goto leave; + } + offset = ck_rhs_map_probe_next(map, offset, probes); + } +leave: + if (probes > probe_limit) { + offset = -1; + } else { + *object = k; + } + + if (pr == -1) + *n_probes = probes; + + *priority = pr; + return offset; +} + +static long +ck_rhs_map_probe(struct ck_rhs *hs, + struct ck_rhs_map *map, + unsigned long *n_probes, + long *priority, + unsigned long h, + const void *key, + const void **object, + unsigned long probe_limit, + enum ck_rhs_probe_behavior behavior) +{ + const void *k; + const void *compare; + long pr = -1; + unsigned long offset, probes, opl; + +#ifdef CK_RHS_PP + /* If we are storing object pointers, then we may leverage pointer packing. */ + unsigned long hv = 0; + + if (hs->mode & CK_RHS_MODE_OBJECT) { + hv = (h >> 25) & CK_RHS_KEY_MASK; + compare = CK_RHS_VMA(key); + } else { + compare = key; + } +#else + compare = key; +#endif + + *object = NULL; + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { + probes = 0; + offset = h & map->mask; + } else { + /* Restart from the bucket we were previously in */ + probes = *n_probes; + offset = ck_rhs_map_probe_next(map, *priority, + probes); + } + opl = probe_limit; + if (behavior == CK_RHS_PROBE_INSERT) + probe_limit = ck_rhs_map_bound_get(map, h); + + for (;;) { + if (probes++ == probe_limit) { + if (probe_limit == opl || pr != -1) { + k = CK_RHS_EMPTY; + goto leave; + } + /* + * If no eligible slot has been found yet, continue probe + * sequence with original probe limit. + */ + probe_limit = opl; + } + k = ck_pr_load_ptr(&map->entries.descs[offset].entry); + if (k == CK_RHS_EMPTY) + goto leave; + if ((behavior != CK_RHS_PROBE_NO_RH)) { + struct ck_rhs_entry_desc *desc = &map->entries.descs[offset]; + + if (pr == -1 && + desc->in_rh == false && desc->probes < probes) { + pr = offset; + *n_probes = probes; + + if (behavior == CK_RHS_PROBE_RH || + behavior == CK_RHS_PROBE_ROBIN_HOOD) { + k = CK_RHS_EMPTY; + goto leave; + } + } + } + + if (behavior != CK_RHS_PROBE_ROBIN_HOOD) { +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) { + if (((uintptr_t)k >> CK_MD_VMA_BITS) != hv) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + k = CK_RHS_VMA(k); + } +#endif + + if (k == compare) + goto leave; + + if (hs->compare == NULL) { + offset = ck_rhs_map_probe_next(map, offset, probes); + continue; + } + + if (hs->compare(k, key) == true) + goto leave; + } + offset = ck_rhs_map_probe_next(map, offset, probes); + } +leave: + if (probes > probe_limit) { + offset = -1; + } else { + *object = k; + } + + if (pr == -1) + *n_probes = probes; + + *priority = pr; + return offset; +} + +static inline const void * +ck_rhs_marshal(unsigned int mode, const void *key, unsigned long h) +{ +#ifdef CK_RHS_PP + const void *insert; + + if (mode & CK_RHS_MODE_OBJECT) { + insert = (void *)((uintptr_t)CK_RHS_VMA(key) | ((h >> 25) << CK_MD_VMA_BITS)); + } else { + insert = key; + } + + return insert; +#else + (void)mode; + (void)h; + + return key; +#endif +} + +bool +ck_rhs_gc(struct ck_rhs *hs) +{ + unsigned long i; + struct ck_rhs_map *map = hs->map; + + unsigned int max_probes = 0; + for (i = 0; i < map->capacity; i++) { + if (ck_rhs_probes(map, i) > max_probes) + max_probes = ck_rhs_probes(map, i); + } + map->probe_maximum = max_probes; + return true; +} + +static void +ck_rhs_add_wanted(struct ck_rhs *hs, long end_offset, long old_slot, + unsigned long h) +{ + struct ck_rhs_map *map = hs->map; + long offset; + unsigned int probes = 1; + bool found_slot = false; + struct ck_rhs_entry_desc *desc; + + offset = h & map->mask; + + if (old_slot == -1) + found_slot = true; + while (offset != end_offset) { + if (offset == old_slot) + found_slot = true; + if (found_slot) { + desc = ck_rhs_desc(map, offset); + if (desc->wanted < CK_RHS_MAX_WANTED) + desc->wanted++; + } + offset = ck_rhs_map_probe_next(map, offset, probes); + probes++; + } +} + +static unsigned long +ck_rhs_remove_wanted(struct ck_rhs *hs, long offset, long limit) +{ + struct ck_rhs_map *map = hs->map; + int probes = ck_rhs_probes(map, offset); + bool do_remove = true; + struct ck_rhs_entry_desc *desc; + + while (probes > 1) { + probes--; + offset = ck_rhs_map_probe_prev(map, offset, probes); + if (offset == limit) + do_remove = false; + if (do_remove) { + desc = ck_rhs_desc(map, offset); + if (desc->wanted != CK_RHS_MAX_WANTED) + desc->wanted--; + } + } + return offset; +} + +static long +ck_rhs_get_first_offset(struct ck_rhs_map *map, unsigned long offset, unsigned int probes) +{ + while (probes > (unsigned long)map->offset_mask + 1) { + offset -= ((probes - 1) &~ map->offset_mask); + offset &= map->mask; + offset = (offset &~ map->offset_mask) + + ((offset - map->offset_mask) & map->offset_mask); + probes -= map->offset_mask + 1; + } + return ((offset &~ map->offset_mask) + ((offset - (probes - 1)) & map->offset_mask)); +} + +#define CK_RHS_MAX_RH 512 + +static int +ck_rhs_put_robin_hood(struct ck_rhs *hs, + long orig_slot, struct ck_rhs_entry_desc *desc) +{ + long slot, first; + const void *object, *insert; + unsigned long n_probes; + struct ck_rhs_map *map; + unsigned long h = 0; + long prev; + void *key; + long prevs[CK_RHS_MAX_RH]; + unsigned int prevs_nb = 0; + unsigned int i; + + map = hs->map; + first = orig_slot; + n_probes = desc->probes; +restart: + key = CK_CC_DECONST_PTR(ck_rhs_entry(map, first)); + insert = key; +#ifdef CK_RHS_PP + if (hs->mode & CK_RHS_MODE_OBJECT) + key = CK_RHS_VMA(key); +#endif + orig_slot = first; + ck_rhs_set_rh(map, orig_slot); + + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, + map->probe_limit, prevs_nb == CK_RHS_MAX_RH ? + CK_RHS_PROBE_NO_RH : CK_RHS_PROBE_ROBIN_HOOD); + + if (slot == -1 && first == -1) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) { + desc->in_rh = false; + + for (i = 0; i < prevs_nb; i++) + ck_rhs_unset_rh(map, prevs[i]); + + return -1; + } + + return 1; + } + + if (first != -1) { + desc = ck_rhs_desc(map, first); + int old_probes = desc->probes; + + desc->probes = n_probes; + h = ck_rhs_get_first_offset(map, first, n_probes); + ck_rhs_map_bound_set(map, h, n_probes); + prev = orig_slot; + prevs[prevs_nb++] = prev; + n_probes = old_probes; + goto restart; + } else { + /* An empty slot was found. */ + h = ck_rhs_get_first_offset(map, slot, n_probes); + ck_rhs_map_bound_set(map, h, n_probes); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + ck_rhs_set_probes(map, slot, n_probes); + desc->in_rh = 0; + ck_rhs_add_wanted(hs, slot, orig_slot, h); + } + while (prevs_nb > 0) { + prev = prevs[--prevs_nb]; + ck_pr_store_ptr(ck_rhs_entry_addr(map, orig_slot), + ck_rhs_entry(map, prev)); + h = ck_rhs_get_first_offset(map, orig_slot, + desc->probes); + ck_rhs_add_wanted(hs, orig_slot, prev, h); + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + orig_slot = prev; + desc->in_rh = false; + desc = ck_rhs_desc(map, orig_slot); + } + return 0; +} + +static void +ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot) +{ + struct ck_rhs_map *map = hs->map; + struct ck_rhs_entry_desc *desc, *new_desc = NULL; + unsigned long h; + + desc = ck_rhs_desc(map, slot); + h = ck_rhs_remove_wanted(hs, slot, -1); + while (desc->wanted > 0) { + unsigned long offset = 0, tmp_offset; + unsigned long wanted_probes = 1; + unsigned int probe = 0; + unsigned int max_probes; + + /* Find a successor */ + while (wanted_probes < map->probe_maximum) { + probe = wanted_probes; + offset = ck_rhs_map_probe_next(map, slot, probe); + while (probe < map->probe_maximum) { + new_desc = ck_rhs_desc(map, offset); + if (new_desc->probes == probe + 1) + break; + probe++; + offset = ck_rhs_map_probe_next(map, offset, + probe); + } + if (probe < map->probe_maximum) + break; + wanted_probes++; + } + if (!(wanted_probes < map->probe_maximum)) { + desc->wanted = 0; + break; + } + desc->probes = wanted_probes; + h = ck_rhs_remove_wanted(hs, offset, slot); + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), + ck_rhs_entry(map, offset)); + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + if (wanted_probes < CK_RHS_WORD_MAX) { + struct ck_rhs_entry_desc *hdesc = ck_rhs_desc(map, h); + if (hdesc->wanted == 1) + CK_RHS_STORE(&hdesc->probe_bound, + wanted_probes); + else if (hdesc->probe_bound == CK_RHS_WORD_MAX || + hdesc->probe_bound == new_desc->probes) { + probe++; + if (hdesc->probe_bound == CK_RHS_WORD_MAX) + max_probes = map->probe_maximum; + else { + max_probes = hdesc->probe_bound; + max_probes--; + } + tmp_offset = ck_rhs_map_probe_next(map, offset, + probe); + while (probe < max_probes) { + if (h == (unsigned long)ck_rhs_get_first_offset(map, tmp_offset, probe)) + break; + probe++; + tmp_offset = ck_rhs_map_probe_next(map, tmp_offset, probe); + } + if (probe == max_probes) + CK_RHS_STORE(&hdesc->probe_bound, + wanted_probes); + } + } + if (desc->wanted < CK_RHS_MAX_WANTED) + desc->wanted--; + slot = offset; + desc = new_desc; + } + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), CK_RHS_EMPTY); + if ((desc->probes - 1) < CK_RHS_WORD_MAX) + CK_RHS_STORE(ck_rhs_probe_bound_addr(map, h), + desc->probes - 1); + desc->probes = 0; +} + +bool +ck_rhs_fas(struct ck_rhs *hs, + unsigned long h, + const void *key, + void **previous) +{ + long slot, first; + const void *object; + const void *insert; + unsigned long n_probes; + struct ck_rhs_map *map = hs->map; + struct ck_rhs_entry_desc *desc, *desc2; + + *previous = NULL; +restart: + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, + ck_rhs_map_bound_get(map, h), CK_RHS_PROBE); + + /* Replacement semantics presume existence. */ + if (object == NULL) + return false; + + insert = ck_rhs_marshal(hs->mode, key, h); + + if (first != -1) { + int ret; + + desc = ck_rhs_desc(map, slot); + desc2 = ck_rhs_desc(map, first); + desc->in_rh = true; + ret = ck_rhs_put_robin_hood(hs, first, desc2); + desc->in_rh = false; + if (CK_CC_UNLIKELY(ret == 1)) + goto restart; + else if (CK_CC_UNLIKELY(ret != 0)) + return false; + ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + desc2->probes = n_probes; + ck_rhs_add_wanted(hs, first, -1, h); + ck_rhs_do_backward_shift_delete(hs, slot); + } else { + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_rhs_set_probes(map, slot, n_probes); + } + *previous = CK_CC_DECONST_PTR(object); + return true; +} + +/* + * An apply function takes two arguments. The first argument is a pointer to a + * pre-existing object. The second argument is a pointer to the fifth argument + * passed to ck_hs_apply. If a non-NULL pointer is passed to the first argument + * and the return value of the apply function is NULL, then the pre-existing + * value is deleted. If the return pointer is the same as the one passed to the + * apply function then no changes are made to the hash table. If the first + * argument is non-NULL and the return pointer is different than that passed to + * the apply function, then the pre-existing value is replaced. For + * replacement, it is required that the value itself is identical to the + * previous value. + */ +bool +ck_rhs_apply(struct ck_rhs *hs, + unsigned long h, + const void *key, + ck_rhs_apply_fn_t *fn, + void *cl) +{ + const void *insert; + const void *object, *delta = false; + unsigned long n_probes; + long slot, first; + struct ck_rhs_map *map; + bool delta_set = false; + +restart: + map = hs->map; + + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_RHS_PROBE_INSERT); + if (slot == -1 && first == -1) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + if (!delta_set) { + delta = fn(CK_CC_DECONST_PTR(object), cl); + delta_set = true; + } + + if (delta == NULL) { + /* + * The apply function has requested deletion. If the object doesn't exist, + * then exit early. + */ + if (CK_CC_UNLIKELY(object == NULL)) + return true; + + /* Otherwise, delete it. */ + ck_rhs_do_backward_shift_delete(hs, slot); + return true; + } + + /* The apply function has not requested hash set modification so exit early. */ + if (delta == object) + return true; + + /* A modification or insertion has been requested. */ + ck_rhs_map_bound_set(map, h, n_probes); + + insert = ck_rhs_marshal(hs->mode, delta, h); + if (first != -1) { + /* + * This follows the same semantics as ck_hs_set, please refer to that + * function for documentation. + */ + struct ck_rhs_entry_desc *desc = NULL, *desc2; + if (slot != -1) { + desc = ck_rhs_desc(map, slot); + desc->in_rh = true; + } + desc2 = ck_rhs_desc(map, first); + int ret = ck_rhs_put_robin_hood(hs, first, desc2); + if (slot != -1) + desc->in_rh = false; + + if (CK_CC_UNLIKELY(ret == 1)) + goto restart; + if (CK_CC_UNLIKELY(ret == -1)) + return false; + /* If an earlier bucket was found, then store entry there. */ + ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); + desc2->probes = n_probes; + /* + * If a duplicate key was found, then delete it after + * signaling concurrent probes to restart. Optionally, + * it is possible to install tombstone after grace + * period if we can guarantee earlier position of + * duplicate key. + */ + ck_rhs_add_wanted(hs, first, -1, h); + if (object != NULL) { + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + ck_rhs_do_backward_shift_delete(hs, slot); + } + } else { + /* + * If we are storing into same slot, then atomic store is sufficient + * for replacement. + */ + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_rhs_set_probes(map, slot, n_probes); + if (object == NULL) + ck_rhs_add_wanted(hs, slot, -1, h); + } + + if (object == NULL) { + map->n_entries++; + if ((map->n_entries ) > map->max_entries) + ck_rhs_grow(hs, map->capacity << 1); + } + return true; +} + +bool +ck_rhs_set(struct ck_rhs *hs, + unsigned long h, + const void *key, + void **previous) +{ + long slot, first; + const void *object; + const void *insert; + unsigned long n_probes; + struct ck_rhs_map *map; + + *previous = NULL; + +restart: + map = hs->map; + + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_RHS_PROBE_INSERT); + if (slot == -1 && first == -1) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + ck_rhs_map_bound_set(map, h, n_probes); + insert = ck_rhs_marshal(hs->mode, key, h); + + if (first != -1) { + struct ck_rhs_entry_desc *desc = NULL, *desc2; + if (slot != -1) { + desc = ck_rhs_desc(map, slot); + desc->in_rh = true; + } + desc2 = ck_rhs_desc(map, first); + int ret = ck_rhs_put_robin_hood(hs, first, desc2); + if (slot != -1) + desc->in_rh = false; + + if (CK_CC_UNLIKELY(ret == 1)) + goto restart; + if (CK_CC_UNLIKELY(ret == -1)) + return false; + /* If an earlier bucket was found, then store entry there. */ + ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); + desc2->probes = n_probes; + /* + * If a duplicate key was found, then delete it after + * signaling concurrent probes to restart. Optionally, + * it is possible to install tombstone after grace + * period if we can guarantee earlier position of + * duplicate key. + */ + ck_rhs_add_wanted(hs, first, -1, h); + if (object != NULL) { + ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]); + ck_pr_fence_atomic_store(); + ck_rhs_do_backward_shift_delete(hs, slot); + } + + } else { + /* + * If we are storing into same slot, then atomic store is sufficient + * for replacement. + */ + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_rhs_set_probes(map, slot, n_probes); + if (object == NULL) + ck_rhs_add_wanted(hs, slot, -1, h); + } + + if (object == NULL) { + map->n_entries++; + if ((map->n_entries ) > map->max_entries) + ck_rhs_grow(hs, map->capacity << 1); + } + + *previous = CK_CC_DECONST_PTR(object); + return true; +} + +static bool +ck_rhs_put_internal(struct ck_rhs *hs, + unsigned long h, + const void *key, + enum ck_rhs_probe_behavior behavior) +{ + long slot, first; + const void *object; + const void *insert; + unsigned long n_probes; + struct ck_rhs_map *map; + +restart: + map = hs->map; + + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, + map->probe_limit, behavior); + + if (slot == -1 && first == -1) { + if (ck_rhs_grow(hs, map->capacity << 1) == false) + return false; + + goto restart; + } + + /* Fail operation if a match was found. */ + if (object != NULL) + return false; + + ck_rhs_map_bound_set(map, h, n_probes); + insert = ck_rhs_marshal(hs->mode, key, h); + + if (first != -1) { + struct ck_rhs_entry_desc *desc = ck_rhs_desc(map, first); + int ret = ck_rhs_put_robin_hood(hs, first, desc); + if (CK_CC_UNLIKELY(ret == 1)) + return ck_rhs_put_internal(hs, h, key, behavior); + else if (CK_CC_UNLIKELY(ret == -1)) + return false; + /* Insert key into first bucket in probe sequence. */ + ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert); + desc->probes = n_probes; + ck_rhs_add_wanted(hs, first, -1, h); + } else { + /* An empty slot was found. */ + ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert); + ck_rhs_set_probes(map, slot, n_probes); + ck_rhs_add_wanted(hs, slot, -1, h); + } + + map->n_entries++; + if ((map->n_entries ) > map->max_entries) + ck_rhs_grow(hs, map->capacity << 1); + return true; +} + +bool +ck_rhs_put(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + + return ck_rhs_put_internal(hs, h, key, CK_RHS_PROBE_INSERT); +} + +bool +ck_rhs_put_unique(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + + return ck_rhs_put_internal(hs, h, key, CK_RHS_PROBE_RH); +} + +void * +ck_rhs_get(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + long first; + const void *object; + struct ck_rhs_map *map; + unsigned long n_probes; + unsigned int g, g_p, probe; + unsigned int *generation; + + do { + map = ck_pr_load_ptr(&hs->map); + generation = &map->generation[h & CK_RHS_G_MASK]; + g = ck_pr_load_uint(generation); + probe = ck_rhs_map_bound_get(map, h); + ck_pr_fence_load(); + + first = -1; + map->probe_func(hs, map, &n_probes, &first, h, key, &object, probe, CK_RHS_PROBE_NO_RH); + + ck_pr_fence_load(); + g_p = ck_pr_load_uint(generation); + } while (g != g_p); + + return CK_CC_DECONST_PTR(object); +} + +void * +ck_rhs_remove(struct ck_rhs *hs, + unsigned long h, + const void *key) +{ + long slot, first; + const void *object; + struct ck_rhs_map *map = hs->map; + unsigned long n_probes; + + slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, + ck_rhs_map_bound_get(map, h), CK_RHS_PROBE_NO_RH); + if (object == NULL) + return NULL; + + map->n_entries--; + ck_rhs_do_backward_shift_delete(hs, slot); + return CK_CC_DECONST_PTR(object); +} + +bool +ck_rhs_move(struct ck_rhs *hs, + struct ck_rhs *source, + ck_rhs_hash_cb_t *hf, + ck_rhs_compare_cb_t *compare, + struct ck_malloc *m) +{ + + if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL) + return false; + + hs->mode = source->mode; + hs->seed = source->seed; + hs->map = source->map; + hs->load_factor = source->load_factor; + hs->m = m; + hs->hf = hf; + hs->compare = compare; + return true; +} + +bool +ck_rhs_init(struct ck_rhs *hs, + unsigned int mode, + ck_rhs_hash_cb_t *hf, + ck_rhs_compare_cb_t *compare, + struct ck_malloc *m, + unsigned long n_entries, + unsigned long seed) +{ + + if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL) + return false; + + hs->m = m; + hs->mode = mode; + hs->seed = seed; + hs->hf = hf; + hs->compare = compare; + hs->load_factor = CK_RHS_DEFAULT_LOAD_FACTOR; + + hs->map = ck_rhs_map_create(hs, n_entries); + return hs->map != NULL; +} Property changes on: vendor-sys/ck/20161230/src/ck_rhs.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property