diff --git a/lib/libthr/Makefile b/lib/libthr/Makefile --- a/lib/libthr/Makefile +++ b/lib/libthr/Makefile @@ -44,6 +44,14 @@ CFLAGS+=-D_PTHREAD_FORCED_UNWIND .endif +.if ${MACHINE_ABI:Mlong64} +# Specify the number of elements in the per-thread pshared lock lookup +# cache (set it to zero to disable). This directly affects the size +# of struct pthread, so keep it small and prime for best results. +# +CFLAGS+=-D_PTHREAD_PSC_SIZE=11 +.endif + LDFLAGS+=-Wl,-znodelete VERSION_DEF=${SRCTOP}/lib/libc/Versions.def diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h --- a/lib/libthr/thread/thr_private.h +++ b/lib/libthr/thread/thr_private.h @@ -361,6 +361,17 @@ void (*destructor)(void *); }; +#if (_PTHREAD_PSC_SIZE > 0) +/* + * Per-thread pshared lock lookup cache entry. + */ +struct pthread_psc_entry { + u_long htgen; + void *key; + void *val; +}; +#endif + /* * lwpid_t is 32bit but kernel thr API exports tid as long type * to preserve the ABI for M:N model in very early date (r131431). @@ -532,6 +543,13 @@ int unwind_disabled; #endif +#if (_PTHREAD_PSC_SIZE > 0) + /* + * Per-thread pshared lock lookup cache hash table. + */ + struct pthread_psc_entry psc_ht[_PTHREAD_PSC_SIZE]; +#endif + /* * Magic value to help recognize a valid thread structure * from an invalid one: diff --git a/lib/libthr/thread/thr_pshared.c b/lib/libthr/thread/thr_pshared.c --- a/lib/libthr/thread/thr_pshared.c +++ b/lib/libthr/thread/thr_pshared.c @@ -52,6 +52,24 @@ static struct urwlock pshared_lock = DEFAULT_URWLOCK; static int page_size; +#if (_PTHREAD_PSC_SIZE > 0) +/* + * The pshared_hash[] hash table generation count is advanced/invalidated each + * time an entry is removed from the hash table. Threads performing a lookup + * will invalidate affected entries in their private cache when they detect + * that pshared_htgen has changed. + * + * Note that a thread performing a lookup running concurrently with a thread + * removing the same key could find and return a stale value (regardless of + * whether the pshared lock cache is enabled). This is a program error. + */ +static u_long pshared_htgen; + +#define PSHARED_HTGEN_INVALIDATE() atomic_add_rel_long(&pshared_htgen, 1) +#else +#define PSHARED_HTGEN_INVALIDATE() +#endif + void __thr_pshared_init(void) { @@ -116,6 +134,7 @@ h->val, NULL); if (error == 0) continue; + PSHARED_HTGEN_INVALIDATE(); LIST_REMOVE(h, link); munmap(h->val, page_size); free(h); @@ -125,17 +144,43 @@ } static void * -pshared_lookup(void *key) +pshared_lookup(struct pthread *curthread, void *key) { struct pshared_hash_head *hd; struct psh *h; + void *val; + +#if (_PTHREAD_PSC_SIZE > 0) + struct pthread_psc_entry *entry; + u_long htgen; + + entry = &curthread->psc_ht[(uintptr_t)key % _PTHREAD_PSC_SIZE]; + htgen = atomic_load_acq_long(&pshared_htgen); + + if (__predict_true(entry->htgen == htgen && entry->key == key)) + return (entry->val); +#endif + pshared_rlock(curthread); hd = &pshared_hash[PSHARED_KEY_HASH(key)]; + val = NULL; LIST_FOREACH(h, hd, link) { - if (h->key == key) - return (h->val); + if (h->key == key) { + val = h->val; + break; + } } - return (NULL); + pshared_unlock(curthread); + +#if (_PTHREAD_PSC_SIZE > 0) + if (val != NULL) { + entry->htgen = htgen; + entry->key = key; + entry->val = val; + } +#endif + + return (val); } static int @@ -195,6 +240,7 @@ hd = &pshared_hash[PSHARED_KEY_HASH(key)]; LIST_FOREACH(h, hd, link) { if (h->key == key) { + PSHARED_HTGEN_INVALIDATE(); LIST_REMOVE(h, link); val = h->val; free(h); @@ -232,13 +278,11 @@ int fd, ins_done; curthread = _get_curthread(); - if (doalloc) { + if (__predict_false(doalloc)) { pshared_destroy(curthread, key); res = NULL; } else { - pshared_rlock(curthread); - res = pshared_lookup(key); - pshared_unlock(curthread); + res = pshared_lookup(curthread, key); if (res != NULL) return (res); } @@ -275,6 +319,7 @@ { _thr_rwl_rdlock(&pshared_lock); + PSHARED_HTGEN_INVALIDATE(); } void