Index: lib/libc/amd64/static_tls.h =================================================================== --- /dev/null +++ lib/libc/amd64/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef LIBC_AMD64_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movq %%fs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} + +#endif Index: lib/libc/gen/Symbol.map =================================================================== --- lib/libc/gen/Symbol.map +++ lib/libc/gen/Symbol.map @@ -542,6 +542,7 @@ __libc_tcdrain; __elf_aux_vector; + __pthread_distribute_static_tls; __pthread_map_stacks_exec; __fillcontextx; __fillcontextx2; Index: lib/libc/gen/elf_utils.c =================================================================== --- lib/libc/gen/elf_utils.c +++ lib/libc/gen/elf_utils.c @@ -34,10 +34,13 @@ #include #include #include +#include #include "libc_private.h" +#include "static_tls.h" int __elf_phdr_match_addr(struct dl_phdr_info *, void *); void __pthread_map_stacks_exec(void); +void __pthread_distribute_static_tls(size_t, void *, size_t, size_t); int __elf_phdr_match_addr(struct dl_phdr_info *phdr_info, void *addr) @@ -97,3 +100,24 @@ ((void (*)(void))__libc_interposing[INTERPOS_map_stacks_exec])(); } + +void +__libc_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + uintptr_t tlsbase; + + tlsbase = _libc_get_static_tls_base(offset); + memcpy((void *)tlsbase, src, len); + memset((char *)tlsbase + len, 0, total_len - len); +} + +#pragma weak __pthread_distribute_static_tls +void +__pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + + ((void (*)(size_t, void *, size_t, size_t))__libc_interposing[ + INTERPOS_distribute_static_tls])(offset, src, len, total_len); +} Index: lib/libc/i386/static_tls.h =================================================================== --- /dev/null +++ lib/libc/i386/static_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef LIBC_I386_STATIC_TLS_H + +static __inline uintptr_t +_libc_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movl %%gs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} + +#endif Index: lib/libc/include/libc_private.h =================================================================== --- lib/libc/include/libc_private.h +++ lib/libc/include/libc_private.h @@ -235,6 +235,7 @@ INTERPOS_map_stacks_exec, INTERPOS_fdatasync, INTERPOS_clock_nanosleep, + INTERPOS_distribute_static_tls, INTERPOS_MAX }; @@ -413,6 +414,8 @@ int __elf_phdr_match_addr(struct dl_phdr_info *, void *); void __init_elf_aux_vector(void); void __libc_map_stacks_exec(void); +void __libc_distribute_static_tls(__size_t, void *, __size_t, __size_t); +__uintptr_t __libc_static_tls_base(__size_t); void _pthread_cancel_enter(int); void _pthread_cancel_leave(int); Index: lib/libc/sys/interposing_table.c =================================================================== --- lib/libc/sys/interposing_table.c +++ lib/libc/sys/interposing_table.c @@ -81,6 +81,7 @@ SLOT(map_stacks_exec, __libc_map_stacks_exec), SLOT(fdatasync, __sys_fdatasync), SLOT(clock_nanosleep, __sys_clock_nanosleep), + SLOT(distribute_static_tls, __libc_distribute_static_tls), }; #undef SLOT Index: lib/libthr/Makefile =================================================================== --- lib/libthr/Makefile +++ lib/libthr/Makefile @@ -17,8 +17,10 @@ NO_WTHREAD_SAFETY=1 NO_WCAST_ALIGN.gcc=1 # for gcc 4.2 CFLAGS+=-DPTHREAD_KERNEL -CFLAGS+=-I${SRCTOP}/lib/libc/include -I${.CURDIR}/thread \ - -I${SRCTOP}/include +CFLAGS+=-I${SRCTOP}/lib/libc/include +CFLAGS+=-I${SRCTOP}/lib/libc/${MACHINE_CPUARCH} +CFLAGS+=-I${.CURDIR}/thread +CFLAGS+=-I${SRCTOP}/include CFLAGS+=-I${.CURDIR}/arch/${MACHINE_CPUARCH}/include CFLAGS+=-I${.CURDIR}/sys CFLAGS+=-I${SRCTOP}/libexec/rtld-elf Index: lib/libthr/arch/amd64/include/pthread_tls.h =================================================================== --- /dev/null +++ lib/libthr/arch/amd64/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ARCH_AMD64_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase -= offset; + return (tlsbase); +} + +#endif Index: lib/libthr/arch/i386/include/pthread_tls.h =================================================================== --- /dev/null +++ lib/libthr/arch/i386/include/pthread_tls.h @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ARCH_PTHREAD_TLS_H + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase -= offset; + return (tlsbase); +} + +#endif Index: lib/libthr/pthread.map =================================================================== --- lib/libthr/pthread.map +++ lib/libthr/pthread.map @@ -137,6 +137,7 @@ __pthread_mutex_lock; __pthread_mutex_timedlock; __pthread_mutex_trylock; + __pthread_distribute_static_tls; _pthread_atfork; _pthread_barrier_destroy; _pthread_barrier_init; Index: lib/libthr/thread/thr_list.c =================================================================== --- lib/libthr/thread/thr_list.c +++ lib/libthr/thread/thr_list.c @@ -39,6 +39,7 @@ #include "libc_private.h" #include "thr_private.h" +#include "static_tls.h" /*#define DEBUG_THREAD_LIST */ #ifdef DEBUG_THREAD_LIST @@ -362,3 +363,35 @@ THREAD_LIST_UNLOCK(curthread); return (ret); } + +#include "pthread_tls.h" + +static void +thr_distribute_static_tls(uintptr_t tlsbase, void *src, size_t len, + size_t total_len) +{ + + memcpy((void *)tlsbase, src, len); + memset((char *)tlsbase + len, 0, total_len - len); +} + +void +__pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + struct pthread *curthread, *thrd; + uintptr_t tlsbase; + + if (!_thr_is_inited()) { + tlsbase = _libc_get_static_tls_base(offset); + thr_distribute_static_tls(tlsbase, src, len, total_len); + return; + } + curthread = _get_curthread(); + THREAD_LIST_RDLOCK(curthread); + TAILQ_FOREACH(thrd, &_thread_list, tle) { + tlsbase = _get_static_tls_base(thrd, offset); + thr_distribute_static_tls(tlsbase, src, len, total_len); + } + THREAD_LIST_UNLOCK(curthread); +} Index: lib/libthr/thread/thr_private.h =================================================================== --- lib/libthr/thread/thr_private.h +++ lib/libthr/thread/thr_private.h @@ -973,6 +973,8 @@ void _thr_tsd_unload(struct dl_phdr_info *phdr_info) __hidden; void _thr_sigact_unload(struct dl_phdr_info *phdr_info) __hidden; void _thr_stack_fix_protection(struct pthread *thrd); +void __pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len); int *__error_threaded(void) __hidden; void __thr_interpose_libc(void) __hidden; Index: libexec/rtld-elf/rtld.h =================================================================== --- libexec/rtld-elf/rtld.h +++ libexec/rtld-elf/rtld.h @@ -256,6 +256,8 @@ bool z_interpose : 1; /* Interpose all objects but main */ bool z_nodeflib : 1; /* Don't search default library path */ bool z_global : 1; /* Make the object global */ + bool static_tls : 1; /* Needs static TLS allocation */ + bool static_tls_copied : 1; /* Needs static TLS copying */ bool ref_nodel : 1; /* Refcount increased to prevent dlclose */ bool init_scanned: 1; /* Object is already on init list. */ bool on_fini_list: 1; /* Object is already on fini list. */ Index: libexec/rtld-elf/rtld.c =================================================================== --- libexec/rtld-elf/rtld.c +++ libexec/rtld-elf/rtld.c @@ -92,6 +92,7 @@ const Elf_Dyn *); static void digest_dynamic(Obj_Entry *, int); static Obj_Entry *digest_phdr(const Elf_Phdr *, int, caddr_t, const char *); +static void distribute_static_tls(Objlist *, RtldLockState *); static Obj_Entry *dlcheck(void *); static int dlclose_locked(void *, RtldLockState *); static Obj_Entry *dlopen_object(const char *name, int fd, Obj_Entry *refobj, @@ -1247,8 +1248,8 @@ obj->textrel = true; if (dynp->d_un.d_val & DF_BIND_NOW) obj->bind_now = true; - /*if (dynp->d_un.d_val & DF_STATIC_TLS) - ;*/ + if (dynp->d_un.d_val & DF_STATIC_TLS) + obj->static_tls = true; break; #ifdef __mips__ case DT_MIPS_LOCAL_GOTNO: @@ -3330,8 +3331,16 @@ if (globallist_next(old_obj_tail) != NULL) { /* We loaded something new. */ assert(globallist_next(old_obj_tail) == obj); - result = load_needed_objects(obj, - lo_flags & (RTLD_LO_DLOPEN | RTLD_LO_EARLY)); + result = 0; + if ((lo_flags & RTLD_LO_EARLY) == 0 && obj->static_tls && + !allocate_tls_offset(obj)) { + _rtld_error("%s: No space available " + "for static Thread Local Storage", obj->path); + result = -1; + } + if (result != -1) + result = load_needed_objects(obj, lo_flags & (RTLD_LO_DLOPEN | + RTLD_LO_EARLY)); init_dag(obj); ref_dag(obj); if (result != -1) @@ -3391,8 +3400,10 @@ name); GDB_STATE(RT_CONSISTENT,obj ? &obj->linkmap : NULL); - if (!(lo_flags & RTLD_LO_EARLY)) { + if ((lo_flags & RTLD_LO_EARLY) == 0) { map_stacks_exec(lockstate); + if (obj != NULL) + distribute_static_tls(&initlist, lockstate); } if (initlist_objects_ifunc(&initlist, (mode & RTLD_MODEMASK) == RTLD_NOW, @@ -5380,6 +5391,27 @@ } } +static void +distribute_static_tls(Objlist *list, RtldLockState *lockstate) +{ + Objlist_Entry *elm; + Obj_Entry *obj; + void (*distrib)(size_t, void *, size_t, size_t); + + distrib = (void (*)(size_t, void *, size_t, size_t))(uintptr_t) + get_program_var_addr("__pthread_distribute_static_tls", lockstate); + if (distrib == NULL) + return; + STAILQ_FOREACH(elm, list, link) { + obj = elm->obj; + if (obj->marker || !obj->tls_done || obj->static_tls_copied) + continue; + distrib(obj->tlsoffset, obj->tlsinit, obj->tlsinitsize, + obj->tlssize); + obj->static_tls_copied = true; + } +} + void symlook_init(SymLook *dst, const char *name) {