Index: lib/libc/amd64/gen/Makefile.inc =================================================================== --- lib/libc/amd64/gen/Makefile.inc +++ lib/libc/amd64/gen/Makefile.inc @@ -2,7 +2,7 @@ # $FreeBSD$ SRCS+= _setjmp.S _set_tp.c rfork_thread.S setjmp.S sigsetjmp.S \ - fabs.S getcontextx.c \ + fabs.S getcontextx.c get_static_tls_base.c \ infinity.c ldexp.c makecontext.c signalcontext.c \ flt_rounds.c fpgetmask.c fpsetmask.c fpgetprec.c fpsetprec.c \ fpgetround.c fpsetround.c fpgetsticky.c Index: lib/libc/amd64/gen/get_static_tls_base.c =================================================================== --- /dev/null +++ lib/libc/amd64/gen/get_static_tls_base.c @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include "libc_private.h" + +uintptr_t +__libc_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movq %%fs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} Index: lib/libc/gen/Symbol.map =================================================================== --- lib/libc/gen/Symbol.map +++ lib/libc/gen/Symbol.map @@ -542,6 +542,7 @@ __libc_tcdrain; __elf_aux_vector; + __pthread_distribute_static_tls; __pthread_map_stacks_exec; __fillcontextx; __fillcontextx2; Index: lib/libc/gen/elf_utils.c =================================================================== --- lib/libc/gen/elf_utils.c +++ lib/libc/gen/elf_utils.c @@ -34,10 +34,12 @@ #include #include #include +#include #include "libc_private.h" int __elf_phdr_match_addr(struct dl_phdr_info *, void *); void __pthread_map_stacks_exec(void); +void __pthread_distribute_static_tls(size_t, void *, size_t, size_t); int __elf_phdr_match_addr(struct dl_phdr_info *phdr_info, void *addr) @@ -97,3 +99,25 @@ ((void (*)(void))__libc_interposing[INTERPOS_map_stacks_exec])(); } + +#include +void +__libc_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + uintptr_t tlsbase; + + tlsbase = __libc_static_tls_base(offset); + memcpy((void *)tlsbase, src, len); + memset((char *)tlsbase + len, 0, total_len - len); +} + +#pragma weak __pthread_distribute_static_tls +void +__pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + + ((void (*)(size_t, void *, size_t, size_t))__libc_interposing[ + INTERPOS_distribute_static_tls])(offset, src, len, total_len); +} Index: lib/libc/i386/gen/Makefile.inc =================================================================== --- lib/libc/i386/gen/Makefile.inc +++ lib/libc/i386/gen/Makefile.inc @@ -2,5 +2,6 @@ # $FreeBSD$ SRCS+= _ctx_start.S _setjmp.S _set_tp.c fabs.S \ + get_static_tls_base.c \ flt_rounds.c getcontextx.c infinity.c ldexp.c makecontext.c \ rfork_thread.S setjmp.S signalcontext.c sigsetjmp.S Index: lib/libc/i386/gen/get_static_tls_base.c =================================================================== --- /dev/null +++ lib/libc/i386/gen/get_static_tls_base.c @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include "libc_private.h" + +uintptr_t +__libc_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movl %%gs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} Index: lib/libc/include/libc_private.h =================================================================== --- lib/libc/include/libc_private.h +++ lib/libc/include/libc_private.h @@ -235,6 +235,7 @@ INTERPOS_map_stacks_exec, INTERPOS_fdatasync, INTERPOS_clock_nanosleep, + INTERPOS_distribute_static_tls, INTERPOS_MAX }; @@ -413,6 +414,8 @@ int __elf_phdr_match_addr(struct dl_phdr_info *, void *); void __init_elf_aux_vector(void); void __libc_map_stacks_exec(void); +void __libc_distribute_static_tls(__size_t, void *, __size_t, __size_t); +__uintptr_t __libc_static_tls_base(__size_t); void _pthread_cancel_enter(int); void _pthread_cancel_leave(int); Index: lib/libc/sys/interposing_table.c =================================================================== --- lib/libc/sys/interposing_table.c +++ lib/libc/sys/interposing_table.c @@ -81,6 +81,7 @@ SLOT(map_stacks_exec, __libc_map_stacks_exec), SLOT(fdatasync, __sys_fdatasync), SLOT(clock_nanosleep, __sys_clock_nanosleep), + SLOT(distribute_static_tls, __libc_distribute_static_tls), }; #undef SLOT Index: lib/libthr/arch/amd64/include/pthread_tls.h =================================================================== --- /dev/null +++ lib/libthr/arch/amd64/include/pthread_tls.h @@ -0,0 +1,56 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ARCH_PTHREAD_TLS_H + +static __inline uintptr_t +_curthread_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movq %%fs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase -= offset; + return (tlsbase); +} + +#endif Index: lib/libthr/arch/i386/include/pthread_tls.h =================================================================== --- /dev/null +++ lib/libthr/arch/i386/include/pthread_tls.h @@ -0,0 +1,56 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ARCH_PTHREAD_TLS_H + +static __inline uintptr_t +_curthread_get_static_tls_base(size_t offset) +{ + uintptr_t tlsbase; + + __asm __volatile("movq %%gs:0, %0" : "=r" (tlsbase)); + tlsbase -= offset; + return (tlsbase); +} + +static __inline uintptr_t +_get_static_tls_base(struct pthread *thr, size_t offset) +{ + uintptr_t tlsbase; + + tlsbase = (uintptr_t)thr->tcb; + tlsbase -= offset; + return (tlsbase); +} + +#endif Index: lib/libthr/pthread.map =================================================================== --- lib/libthr/pthread.map +++ lib/libthr/pthread.map @@ -137,6 +137,7 @@ __pthread_mutex_lock; __pthread_mutex_timedlock; __pthread_mutex_trylock; + __pthread_distribute_static_tls; _pthread_atfork; _pthread_barrier_destroy; _pthread_barrier_init; Index: lib/libthr/thread/thr_list.c =================================================================== --- lib/libthr/thread/thr_list.c +++ lib/libthr/thread/thr_list.c @@ -362,3 +362,35 @@ THREAD_LIST_UNLOCK(curthread); return (ret); } + +#include "pthread_tls.h" + +static void +thr_distribute_static_tls(uintptr_t tlsbase, void *src, size_t len, + size_t total_len) +{ + + memcpy((void *)tlsbase, src, len); + memset((char *)tlsbase + len, 0, total_len - len); +} + +void +__pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len) +{ + struct pthread *curthread, *thrd; + uintptr_t tlsbase; + + if (!_thr_is_inited()) { + tlsbase = _curthread_get_static_tls_base(offset); + thr_distribute_static_tls(tlsbase, src, len, total_len); + return; + } + curthread = _get_curthread(); + THREAD_LIST_RDLOCK(curthread); + TAILQ_FOREACH(thrd, &_thread_list, tle) { + tlsbase = _get_static_tls_base(thrd, offset); + thr_distribute_static_tls(tlsbase, src, len, total_len); + } + THREAD_LIST_UNLOCK(curthread); +} Index: lib/libthr/thread/thr_private.h =================================================================== --- lib/libthr/thread/thr_private.h +++ lib/libthr/thread/thr_private.h @@ -973,6 +973,8 @@ void _thr_tsd_unload(struct dl_phdr_info *phdr_info) __hidden; void _thr_sigact_unload(struct dl_phdr_info *phdr_info) __hidden; void _thr_stack_fix_protection(struct pthread *thrd); +void __pthread_distribute_static_tls(size_t offset, void *src, size_t len, + size_t total_len); int *__error_threaded(void) __hidden; void __thr_interpose_libc(void) __hidden; Index: libexec/rtld-elf/amd64/reloc.c =================================================================== --- libexec/rtld-elf/amd64/reloc.c +++ libexec/rtld-elf/amd64/reloc.c @@ -249,43 +249,19 @@ *where = symval; break; case R_X86_64_TPOFF64: - /* - * We lazily allocate offsets for static TLS - * as we see the first relocation that - * references the TLS block. This allows us to - * support (small amounts of) static TLS in - * dynamically loaded modules. If we run out - * of space, we generate an error. - */ if (!defobj->tls_done) { - if (!allocate_tls_offset( - __DECONST(Obj_Entry *, defobj))) { - _rtld_error("%s: No space available " - "for static Thread Local Storage", - obj->path); - goto done; - } + _rtld_error("%s: Uses static TLS and no " + "DF_STATIC_TLS ", obj->path); + goto done; } *where = (Elf_Addr)(def->st_value - defobj->tlsoffset + rela->r_addend); break; case R_X86_64_TPOFF32: - /* - * We lazily allocate offsets for static TLS - * as we see the first relocation that - * references the TLS block. This allows us to - * support (small amounts of) static TLS in - * dynamically loaded modules. If we run out - * of space, we generate an error. - */ if (!defobj->tls_done) { - if (!allocate_tls_offset( - __DECONST(Obj_Entry *, defobj))) { - _rtld_error("%s: No space available " - "for static Thread Local Storage", - obj->path); - goto done; - } + _rtld_error("%s: Uses static TLS and no " + "DF_STATIC_TLS ", obj->path); + goto done; } *where32 = (Elf32_Addr)(def->st_value - defobj->tlsoffset + rela->r_addend); Index: libexec/rtld-elf/i386/reloc.c =================================================================== --- libexec/rtld-elf/i386/reloc.c +++ libexec/rtld-elf/i386/reloc.c @@ -234,22 +234,10 @@ break; case R_386_TLS_TPOFF: case R_386_TLS_TPOFF32: - /* - * We lazily allocate offsets for static TLS - * as we see the first relocation that - * references the TLS block. This allows us to - * support (small amounts of) static TLS in - * dynamically loaded modules. If we run out - * of space, we generate an error. - */ if (!defobj->tls_done) { - if (!allocate_tls_offset( - __DECONST(Obj_Entry *, defobj))) { - _rtld_error("%s: No space available " - "for static Thread Local Storage", - obj->path); - goto done; - } + _rtld_error("%s: Uses static TLS and no " + "DF_STATIC_TLS ", obj->path); + goto done; } add = (Elf_Addr)(def->st_value - defobj->tlsoffset); if (ELF_R_TYPE(rel->r_info) == R_386_TLS_TPOFF) Index: libexec/rtld-elf/rtld.h =================================================================== --- libexec/rtld-elf/rtld.h +++ libexec/rtld-elf/rtld.h @@ -256,6 +256,7 @@ bool z_interpose : 1; /* Interpose all objects but main */ bool z_nodeflib : 1; /* Don't search default library path */ bool z_global : 1; /* Make the object global */ + bool static_tls : 1; /* Needs static TLS allocation */ bool ref_nodel : 1; /* Refcount increased to prevent dlclose */ bool init_scanned: 1; /* Object is already on init list. */ bool on_fini_list: 1; /* Object is already on fini list. */ Index: libexec/rtld-elf/rtld.c =================================================================== --- libexec/rtld-elf/rtld.c +++ libexec/rtld-elf/rtld.c @@ -92,6 +92,7 @@ const Elf_Dyn *); static void digest_dynamic(Obj_Entry *, int); static Obj_Entry *digest_phdr(const Elf_Phdr *, int, caddr_t, const char *); +static void distribute_static_tls(Obj_Entry *, RtldLockState *); static Obj_Entry *dlcheck(void *); static int dlclose_locked(void *, RtldLockState *); static Obj_Entry *dlopen_object(const char *name, int fd, Obj_Entry *refobj, @@ -1247,8 +1248,8 @@ obj->textrel = true; if (dynp->d_un.d_val & DF_BIND_NOW) obj->bind_now = true; - /*if (dynp->d_un.d_val & DF_STATIC_TLS) - ;*/ + if (dynp->d_un.d_val & DF_STATIC_TLS) + obj->static_tls = true; break; #ifdef __mips__ case DT_MIPS_LOCAL_GOTNO: @@ -3330,10 +3331,18 @@ if (globallist_next(old_obj_tail) != NULL) { /* We loaded something new. */ assert(globallist_next(old_obj_tail) == obj); - result = load_needed_objects(obj, - lo_flags & (RTLD_LO_DLOPEN | RTLD_LO_EARLY)); + result = 0; init_dag(obj); ref_dag(obj); + if ((lo_flags & RTLD_LO_EARLY) == 0 && obj->static_tls && + !allocate_tls_offset(obj)) { + _rtld_error("%s: No space available " + "for static Thread Local Storage", obj->path); + result = -1; + } + if (result != -1) + result = load_needed_objects(obj, lo_flags & (RTLD_LO_DLOPEN | + RTLD_LO_EARLY)); if (result != -1) result = rtld_verify_versions(&obj->dagmembers); if (result != -1 && ld_tracing) @@ -3391,8 +3400,10 @@ name); GDB_STATE(RT_CONSISTENT,obj ? &obj->linkmap : NULL); - if (!(lo_flags & RTLD_LO_EARLY)) { + if ((lo_flags & RTLD_LO_EARLY) == 0) { map_stacks_exec(lockstate); + if (obj != NULL && obj->static_tls) + distribute_static_tls(obj, lockstate); } if (initlist_objects_ifunc(&initlist, (mode & RTLD_MODEMASK) == RTLD_NOW, @@ -5380,6 +5391,19 @@ } } +static void +distribute_static_tls(Obj_Entry *obj, RtldLockState *lockstate) +{ + void (*distrib)(size_t, void *, size_t, size_t); + + distrib = (void (*)(size_t, void *, size_t, size_t))(uintptr_t) + get_program_var_addr("__pthread_distribute_static_tls", lockstate); + if (distrib != NULL) { + distrib(obj->tlsoffset, obj->tlsinit, obj->tlsinitsize, + obj->tlssize); + } +} + void symlook_init(SymLook *dst, const char *name) {