diff --git a/lib/libc/Makefile b/lib/libc/Makefile --- a/lib/libc/Makefile +++ b/lib/libc/Makefile @@ -68,6 +68,14 @@ # Extras that live in either libc.a or libc_nonshared.a LIBC_NONSHARED_SRCS= +RTLD_ELF_DIR=${SRCTOP}/libexec/rtld-elf +.if exists(${RTLD_ELF_DIR}/${MACHINE_ARCH:S/powerpc64le/powerpc64/}) +RTLD_ARCH= ${MACHINE_ARCH:S/powerpc64le/powerpc64/} +.else +RTLD_ARCH= ${MACHINE_CPUARCH} +.endif +RTLD_HDRS= -I${RTLD_ELF_DIR}/${RTLD_ARCH} -I${RTLD_ELF_DIR} + # Define (empty) variables so that make doesn't give substitution # errors if the included makefiles don't change these: MDSRCS= diff --git a/lib/libc/aarch64/gen/Makefile.inc b/lib/libc/aarch64/gen/Makefile.inc --- a/lib/libc/aarch64/gen/Makefile.inc +++ b/lib/libc/aarch64/gen/Makefile.inc @@ -7,6 +7,7 @@ flt_rounds.c \ fpgetmask.c \ fpsetmask.c \ + _get_tp.c \ infinity.c \ ldexp.c \ makecontext.c \ diff --git a/lib/libc/aarch64/gen/_get_tp.c b/lib/libc/aarch64/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/aarch64/gen/_get_tp.c @@ -0,0 +1,45 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include "libc_private.h" + +void * +_get_tp(void) +{ + void *res; + + __asm __volatile("mrs %0, tpidr_el0" : "=r" (res)); + return (res); +} diff --git a/lib/libc/amd64/gen/Makefile.inc b/lib/libc/amd64/gen/Makefile.inc --- a/lib/libc/amd64/gen/Makefile.inc +++ b/lib/libc/amd64/gen/Makefile.inc @@ -1,7 +1,7 @@ # @(#)Makefile.inc 8.1 (Berkeley) 6/4/93 # $FreeBSD$ -SRCS+= _setjmp.S _set_tp.c rfork_thread.S setjmp.S sigsetjmp.S \ +SRCS+= _setjmp.S _get_tp.c _set_tp.c rfork_thread.S setjmp.S sigsetjmp.S \ fabs.S \ infinity.c ldexp.c makecontext.c signalcontext.c \ flt_rounds.c fpgetmask.c fpsetmask.c fpgetprec.c fpsetprec.c \ diff --git a/lib/libc/amd64/gen/_get_tp.c b/lib/libc/amd64/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/amd64/gen/_get_tp.c @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include "libc_private.h" + +void * +_get_tp(void) +{ + void **res; + + /* This function is used by rtld, avoid ifuncs. */ + __asm __volatile("movq %%fs:0, %0" : "=r" (res)); + return (&res[1]); +} diff --git a/lib/libc/arm/gen/Makefile.inc b/lib/libc/arm/gen/Makefile.inc --- a/lib/libc/arm/gen/Makefile.inc +++ b/lib/libc/arm/gen/Makefile.inc @@ -4,6 +4,7 @@ SRCS+= \ __aeabi_read_tp.S \ _ctx_start.S \ + _get_tp.c \ _set_tp.c \ _setjmp.S \ alloca.S \ diff --git a/lib/libc/arm/gen/_get_tp.c b/lib/libc/arm/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/arm/gen/_get_tp.c @@ -0,0 +1,46 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include "libc_private.h" + +void * +_get_tp(void) +{ + void *res; + + __asm __volatile("mrc p15, 0, %0, c13, c0, 3" : "=r" (res)); + return (res); +} diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc --- a/lib/libc/gen/Makefile.inc +++ b/lib/libc/gen/Makefile.inc @@ -172,6 +172,9 @@ CFLAGS.arc4random.c= -I${SRCTOP}/sys -I${SRCTOP}/sys/crypto/chacha20 +CFLAGS.dlfcn.c= ${RTLD_HDRS} +CFLAGS.tls.c= ${RTLD_HDRS} + .PATH: ${SRCTOP}/contrib/libc-pwcache SRCS+= pwcache.c pwcache.h diff --git a/lib/libc/gen/dlfcn.c b/lib/libc/gen/dlfcn.c --- a/lib/libc/gen/dlfcn.c +++ b/lib/libc/gen/dlfcn.c @@ -43,10 +43,11 @@ #include "namespace.h" #include #include "un-namespace.h" +#include "rtld.h" #include "libc_private.h" #include "reentrant.h" -static char sorry[] = "Service unavailable"; +static const char sorry[] = "Service unavailable"; void _rtld_thread_init(void *); void _rtld_atfork_pre(int *); @@ -91,7 +92,7 @@ dlerror(void) { - return (sorry); + return (__DECONST(char *, sorry)); } #pragma weak dllockinit @@ -195,8 +196,6 @@ for (i = 0; i < phdr_info.dlpi_phnum; i++) { if (phdr_info.dlpi_phdr[i].p_type == PT_TLS) { phdr_info.dlpi_tls_modid = 1; - phdr_info.dlpi_tls_data = - (void*)phdr_info.dlpi_phdr[i].p_vaddr; } } phdr_info.dlpi_adds = 1; @@ -209,13 +208,17 @@ void *data __unused) { #ifndef IN_LIBDL + tls_index ti; int ret; __init_elf_aux_vector(); if (__elf_aux_vector == NULL) return (1); _once(&dl_phdr_info_once, dl_init_phdr_info); + ti.ti_module = 1; + ti.ti_offset = 0; mutex_lock(&dl_phdr_info_lock); + phdr_info.dlpi_tls_data = __tls_get_addr(&ti); ret = callback(&phdr_info, sizeof(phdr_info), data); mutex_unlock(&dl_phdr_info_lock); return (ret); diff --git a/lib/libc/gen/tls.c b/lib/libc/gen/tls.c --- a/lib/libc/gen/tls.c +++ b/lib/libc/gen/tls.c @@ -41,6 +41,7 @@ #include #include +#include "rtld.h" #include "libc_private.h" #define tls_assert(cond) ((cond) ? (void) 0 : \ @@ -96,35 +97,42 @@ #ifndef PIC -static size_t tls_static_space; -static size_t tls_init_size; -static size_t tls_init_align; -static void *tls_init; +static size_t libc_tls_static_space; +static size_t libc_tls_init_size; +static size_t libc_tls_init_align; +static void *libc_tls_init; #endif +void * +__libc_tls_get_addr(void *vti) +{ + Elf_Addr **dtvp, *dtv; + tls_index *ti; + + dtvp = _get_tp(); + dtv = *dtvp; + ti = vti; + return ((char *)(dtv[ti->ti_module + 1] + ti->ti_offset) + + TLS_DTV_OFFSET); +} + #ifdef __i386__ /* GNU ABI */ __attribute__((__regparm__(1))) void * -___libc_tls_get_addr(void *ti __unused) +___libc_tls_get_addr(void *vti) { - return (0); + return (__libc_tls_get_addr(vti)); } #endif -void * -__libc_tls_get_addr(void *ti __unused) -{ - return (0); -} - #ifndef PIC static void * -malloc_aligned(size_t size, size_t align) +libc_malloc_aligned(size_t size, size_t align) { void *mem, *res; @@ -138,7 +146,7 @@ } static void -free_aligned(void *ptr) +libc_free_aligned(void *ptr) { void *mem; uintptr_t x; @@ -188,8 +196,6 @@ * [5] I'm not able to validate "values are biased" assertions. */ -#define TLS_TCB_SIZE (2 * sizeof(void *)) - /* * Return pointer to allocated TLS block */ @@ -201,12 +207,13 @@ /* Compute fragments sizes. */ extra_size = tcbsize - TLS_TCB_SIZE; #if defined(__aarch64__) || defined(__arm__) - post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE; + post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; #else post_size = 0; #endif tls_block_size = tcbsize + post_size; - pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size; + pre_size = roundup2(tls_block_size, libc_tls_init_align) - + tls_block_size; return ((char *)tcb - pre_size - extra_size); } @@ -225,7 +232,7 @@ tls = (Elf_Addr **)tcb; dtv = tls[0]; __je_bootstrap_free(dtv); - free_aligned(get_tls_block_ptr(tcb, tcbsize)); + libc_free_aligned(get_tls_block_ptr(tcb, tcbsize)); } /* @@ -259,21 +266,22 @@ return (oldtcb); tls_assert(tcbalign >= TLS_TCB_ALIGN); - maxalign = MAX(tcbalign, tls_init_align); + maxalign = MAX(tcbalign, libc_tls_init_align); /* Compute fragmets sizes. */ extra_size = tcbsize - TLS_TCB_SIZE; #if defined(__aarch64__) || defined(__arm__) - post_size = roundup2(TLS_TCB_SIZE, tls_init_align) - TLS_TCB_SIZE; + post_size = roundup2(TLS_TCB_SIZE, libc_tls_init_align) - TLS_TCB_SIZE; #else post_size = 0; #endif tls_block_size = tcbsize + post_size; - pre_size = roundup2(tls_block_size, tls_init_align) - tls_block_size; - tls_block_size += pre_size + tls_static_space; + pre_size = roundup2(tls_block_size, libc_tls_init_align) - + tls_block_size; + tls_block_size += pre_size + libc_tls_static_space; /* Allocate whole TLS block */ - tls_block = malloc_aligned(tls_block_size, maxalign); + tls_block = libc_malloc_aligned(tls_block_size, maxalign); if (tls_block == NULL) { tls_msg("__libc_allocate_tls: Out of memory.\n"); abort(); @@ -285,7 +293,7 @@ if (oldtcb != NULL) { memcpy(tls_block, get_tls_block_ptr(oldtcb, tcbsize), tls_block_size); - free_aligned(oldtcb); + libc_free_aligned(oldtcb); /* Adjust the DTV. */ dtv = tcb[0]; @@ -302,8 +310,8 @@ dtv[1] = 1; /* Segments count. */ dtv[2] = (Elf_Addr)(tls + DTV_OFFSET); - if (tls_init_size > 0) - memcpy(tls, tls_init, tls_init_size); + if (libc_tls_init_size > 0) + memcpy(tls, libc_tls_init, libc_tls_init_size); } return (tcb); @@ -329,13 +337,13 @@ * Figure out the size of the initial TLS block so that we can * find stuff which ___tls_get_addr() allocated dynamically. */ - tcbalign = MAX(tcbalign, tls_init_align); - size = roundup2(tls_static_space, tcbalign); + tcbalign = MAX(tcbalign, libc_tls_init_align); + size = roundup2(libc_tls_static_space, tcbalign); dtv = ((Elf_Addr**)tcb)[1]; tlsend = (Elf_Addr) tcb; tlsstart = tlsend - size; - free_aligned((void*)tlsstart); + libc_free_aligned((void*)tlsstart); __je_bootstrap_free(dtv); } @@ -350,12 +358,12 @@ Elf_Addr *dtv; Elf_Addr segbase, oldsegbase; - tcbalign = MAX(tcbalign, tls_init_align); - size = roundup2(tls_static_space, tcbalign); + tcbalign = MAX(tcbalign, libc_tls_init_align); + size = roundup2(libc_tls_static_space, tcbalign); if (tcbsize < 2 * sizeof(Elf_Addr)) tcbsize = 2 * sizeof(Elf_Addr); - tls = malloc_aligned(size + tcbsize, tcbalign); + tls = libc_malloc_aligned(size + tcbsize, tcbalign); if (tls == NULL) { tls_msg("__libc_allocate_tls: Out of memory.\n"); abort(); @@ -373,16 +381,16 @@ dtv[0] = 1; dtv[1] = 1; - dtv[2] = segbase - tls_static_space; + dtv[2] = segbase - libc_tls_static_space; if (oldtls) { /* * Copy the static TLS block over whole. */ oldsegbase = (Elf_Addr) oldtls; - memcpy((void *)(segbase - tls_static_space), - (const void *)(oldsegbase - tls_static_space), - tls_static_space); + memcpy((void *)(segbase - libc_tls_static_space), + (const void *)(oldsegbase - libc_tls_static_space), + libc_tls_static_space); /* * We assume that this block was the one we created with @@ -390,10 +398,11 @@ */ _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); } else { - memcpy((void *)(segbase - tls_static_space), - tls_init, tls_init_size); - memset((void *)(segbase - tls_static_space + tls_init_size), - 0, tls_static_space - tls_init_size); + memcpy((void *)(segbase - libc_tls_static_space), + libc_tls_init, libc_tls_init_size); + memset((void *)(segbase - libc_tls_static_space + + libc_tls_init_size), 0, + libc_tls_static_space - libc_tls_init_size); } return (void*) segbase; @@ -457,11 +466,11 @@ for (i = 0; (unsigned) i < phnum; i++) { if (phdr[i].p_type == PT_TLS) { - tls_static_space = roundup2(phdr[i].p_memsz, + libc_tls_static_space = roundup2(phdr[i].p_memsz, phdr[i].p_align); - tls_init_size = phdr[i].p_filesz; - tls_init_align = phdr[i].p_align; - tls_init = (void*) phdr[i].p_vaddr; + libc_tls_init_size = phdr[i].p_filesz; + libc_tls_init_align = phdr[i].p_align; + libc_tls_init = (void *)phdr[i].p_vaddr; break; } } diff --git a/lib/libc/i386/gen/Makefile.inc b/lib/libc/i386/gen/Makefile.inc --- a/lib/libc/i386/gen/Makefile.inc +++ b/lib/libc/i386/gen/Makefile.inc @@ -1,6 +1,6 @@ # @(#)Makefile.inc 8.1 (Berkeley) 6/4/93 # $FreeBSD$ -SRCS+= _ctx_start.S _setjmp.S _set_tp.c fabs.S \ +SRCS+= _ctx_start.S _setjmp.S _set_tp.c _get_tp.c fabs.S \ flt_rounds.c infinity.c ldexp.c makecontext.c \ rfork_thread.S setjmp.S signalcontext.c sigsetjmp.S diff --git a/lib/libc/i386/gen/_get_tp.c b/lib/libc/i386/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/i386/gen/_get_tp.c @@ -0,0 +1,45 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include "libc_private.h" + +void * +_get_tp(void) +{ + void **res; + + __asm __volatile("movl %%gs:0, %0" : "=r" (res)); + return (&res[1]); +} diff --git a/lib/libc/include/libc_private.h b/lib/libc/include/libc_private.h --- a/lib/libc/include/libc_private.h +++ b/lib/libc/include/libc_private.h @@ -261,8 +261,9 @@ int _once(pthread_once_t *, void (*)(void)); /* - * Set the TLS thread pointer + * Get/set the TLS thread pointer */ +void *_get_tp(void); void _set_tp(void *tp); /* diff --git a/lib/libc/mips/gen/Makefile.inc b/lib/libc/mips/gen/Makefile.inc --- a/lib/libc/mips/gen/Makefile.inc +++ b/lib/libc/mips/gen/Makefile.inc @@ -6,6 +6,6 @@ # SRCS+= flt_rounds.c fpgetmask.c fpgetround.c fpgetsticky.c fpsetmask.c \ # fpsetround.c fpsetsticky.c -SRCS+= _ctx_start.S _set_tp.c _setjmp.S makecontext.c \ +SRCS+= _ctx_start.S _get_tp.c _set_tp.c _setjmp.S makecontext.c \ setjmp.S signalcontext.c sigsetjmp.S \ trivial-getcontextx.c diff --git a/lib/libc/mips/gen/_get_tp.c b/lib/libc/mips/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/mips/gen/_get_tp.c @@ -0,0 +1,103 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include + +#include "libc_private.h" + +#ifdef __mips_n64 +static void * +_mips_get_tls(void) +{ + uint64_t _rv; + + __asm__ __volatile__ ( + ".set\tpush\n\t" + ".set\tmips64r2\n\t" + "rdhwr\t%0, $29\n\t" + ".set\tpop" + : "=r" (_rv)); + /* + * XXXSS See 'git show c6be4f4d2d1b71c04de5d3bbb6933ce2dbcdb317' + * + * Remove the offset since this really a request to get the TLS + * pointer via sysarch() (in theory). Of course, this may go away + * once the TLS code is rewritten. + */ + _rv = _rv - TLS_TP_OFFSET - TLS_TCB_SIZE; + + return (void *)_rv; +} + +#else /* mips 32 */ + +static void * +_mips_get_tls(void) +{ + uint32_t _rv; + + __asm__ __volatile__ ( + ".set\tpush\n\t" + ".set\tmips32r2\n\t" + "rdhwr\t%0, $29\n\t" + ".set\tpop" + : "=r" (_rv)); + /* + * XXXSS See 'git show c6be4f4d2d1b71c04de5d3bbb6933ce2dbcdb317' + * + * Remove the offset since this really a request to get the TLS + * pointer via sysarch() (in theory). Of course, this may go away + * once the TLS code is rewritten. + */ + _rv = _rv - TLS_TP_OFFSET - TLS_TCB_SIZE; + + return (void *)_rv; +} +#endif /* ! __mips_n64 */ + +void * +_get_tp(void) +{ + void *res; + +#ifdef TLS_USE_SYSARCH + sysarch(MIPS_GET_TLS, &res); +#else + res = _mips_get_tls(); +#endif + return (res); +} diff --git a/lib/libc/powerpc/gen/Makefile.common b/lib/libc/powerpc/gen/Makefile.common --- a/lib/libc/powerpc/gen/Makefile.common +++ b/lib/libc/powerpc/gen/Makefile.common @@ -3,4 +3,6 @@ .PATH: ${LIBC_SRCTOP}/powerpc/gen SRCS += _ctx_start.S eabi.S infinity.c ldexp.c makecontext.c \ - signalcontext.c syncicache.c _set_tp.c trivial-getcontextx.c + signalcontext.c syncicache.c _get_tp.c _set_tp.c trivial-getcontextx.c + +CFLAGS._get_tp.c+= ${RTLD_HDRS} diff --git a/lib/libc/powerpc/gen/_get_tp.c b/lib/libc/powerpc/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/powerpc/gen/_get_tp.c @@ -0,0 +1,47 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include "libc_private.h" +#include "rtld.h" + +void * +_get_tp(void) +{ + Elf_Addr tp; + + __asm __volatile("mr %0,2" : "=r" (tp)); + return ((void *)(tp - TLS_TP_OFFSET - TLS_TCB_SIZE)); +} diff --git a/lib/libc/powerpc64/gen/Makefile.inc b/lib/libc/powerpc64/gen/Makefile.inc --- a/lib/libc/powerpc64/gen/Makefile.inc +++ b/lib/libc/powerpc64/gen/Makefile.inc @@ -1,10 +1,11 @@ # $FreeBSD$ SRCS += _ctx_start.S fabs.S flt_rounds.c fpgetmask.c fpgetround.c \ - fpgetsticky.c fpsetmask.c fpsetround.c \ + fpgetsticky.c fpsetmask.c fpsetround.c _get_tp.c \ infinity.c ldexp.c makecontext.c _setjmp.S \ setjmp.S sigsetjmp.S signalcontext.c syncicache.c \ _set_tp.c \ trivial-getcontextx.c +CFLAGS._get_tp.c+= ${RTLD_HDRS} diff --git a/lib/libc/powerpc64/gen/_get_tp.c b/lib/libc/powerpc64/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/powerpc64/gen/_get_tp.c @@ -0,0 +1,47 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include "libc_private.h" +#include "rtld.h" + +void * +_get_tp(void) +{ + Elf_Addr tp; + + __asm __volatile("mr %0,13" : "=r" (tp)); + return ((void *)(tp - TLS_TP_OFFSET - TLS_TCB_SIZE)); +} diff --git a/lib/libc/riscv/gen/Makefile.inc b/lib/libc/riscv/gen/Makefile.inc --- a/lib/libc/riscv/gen/Makefile.inc +++ b/lib/libc/riscv/gen/Makefile.inc @@ -5,6 +5,7 @@ flt_rounds.c \ fpgetmask.c \ fpsetmask.c \ + _get_tp.c \ infinity.c \ ldexp.c \ makecontext.c \ @@ -13,3 +14,5 @@ setjmp.S \ sigsetjmp.S \ trivial-getcontextx.c + +CFLAGS._get_tp.c+= ${RTLD_HDRS} diff --git a/lib/libc/riscv/gen/_get_tp.c b/lib/libc/riscv/gen/_get_tp.c new file mode 100644 --- /dev/null +++ b/lib/libc/riscv/gen/_get_tp.c @@ -0,0 +1,47 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 The FreeBSD Foundation + * + * This software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include "libc_private.h" +#include "rtld.h" + +void * +_get_tp(void) +{ + Elf_Addr tp; + + __asm __volatile("mv %0, tp" : "=r" (tp)); + return ((void *)(tp - TLS_TP_OFFSET - TLS_TCB_SIZE)); +} diff --git a/lib/libdl/Makefile b/lib/libdl/Makefile --- a/lib/libdl/Makefile +++ b/lib/libdl/Makefile @@ -6,6 +6,13 @@ .PATH: ${SRCTOP}/lib/libc/gen CFLAGS+=-I${SRCTOP}/lib/libc/include +RTLD_ELF_DIR=${SRCTOP}/libexec/rtld-elf +.if exists(${RTLD_ELF_DIR}/${MACHINE_ARCH:S/powerpc64le/powerpc64/}) +RTLD_ARCH= ${MACHINE_ARCH:S/powerpc64le/powerpc64/} +.else +RTLD_ARCH= ${MACHINE_CPUARCH} +.endif +CFLAGS+= -I${RTLD_ELF_DIR}/${RTLD_ARCH} -I${RTLD_ELF_DIR} CFLAGS+=-DIN_LIBDL LDFLAGS+=-Wl,-F,libc.so.7 VERSION_DEF=${SRCTOP}/lib/libc/Versions.def diff --git a/libexec/rtld-elf/aarch64/rtld_machdep.h b/libexec/rtld-elf/aarch64/rtld_machdep.h --- a/libexec/rtld-elf/aarch64/rtld_machdep.h +++ b/libexec/rtld-elf/aarch64/rtld_machdep.h @@ -93,4 +93,6 @@ #define md_abi_variant_hook(x) +#define TLS_DTV_OFFSET 0 + #endif diff --git a/libexec/rtld-elf/amd64/reloc.c b/libexec/rtld-elf/amd64/reloc.c --- a/libexec/rtld-elf/amd64/reloc.c +++ b/libexec/rtld-elf/amd64/reloc.c @@ -538,13 +538,13 @@ sysarch(AMD64_SET_FSBASE, &addr); } -void *__tls_get_addr(tls_index *ti) +void * +__tls_get_addr(tls_index *ti) { - Elf_Addr** segbase; + Elf_Addr **dtvp; - __asm __volatile("movq %%fs:0, %0" : "=r" (segbase)); - - return tls_get_addr_common(&segbase[1], ti->ti_module, ti->ti_offset); + dtvp = _get_tp(); + return (tls_get_addr_common(dtvp, ti->ti_module, ti->ti_offset)); } size_t diff --git a/libexec/rtld-elf/amd64/rtld_machdep.h b/libexec/rtld-elf/amd64/rtld_machdep.h --- a/libexec/rtld-elf/amd64/rtld_machdep.h +++ b/libexec/rtld-elf/amd64/rtld_machdep.h @@ -73,6 +73,8 @@ #define md_abi_variant_hook(x) +#define TLS_DTV_OFFSET 0 + size_t calculate_first_tls_offset(size_t size, size_t align, size_t offset); size_t calculate_tls_offset(size_t prev_offset, size_t prev_size, size_t size, size_t align, size_t offset); diff --git a/libexec/rtld-elf/arm/rtld_machdep.h b/libexec/rtld-elf/arm/rtld_machdep.h --- a/libexec/rtld-elf/arm/rtld_machdep.h +++ b/libexec/rtld-elf/arm/rtld_machdep.h @@ -86,4 +86,6 @@ #define md_abi_variant_hook(x) #endif +#define TLS_DTV_OFFSET 0 + #endif diff --git a/libexec/rtld-elf/i386/reloc.c b/libexec/rtld-elf/i386/reloc.c --- a/libexec/rtld-elf/i386/reloc.c +++ b/libexec/rtld-elf/i386/reloc.c @@ -519,23 +519,23 @@ /* GNU ABI */ __attribute__((__regparm__(1))) -void *___tls_get_addr(tls_index *ti) +void * +___tls_get_addr(tls_index *ti) { - Elf_Addr** segbase; + Elf_Addr **dtvp; - __asm __volatile("movl %%gs:0, %0" : "=r" (segbase)); - - return tls_get_addr_common(&segbase[1], ti->ti_module, ti->ti_offset); + dtvp = _get_tp(); + return (tls_get_addr_common(dtvp, ti->ti_module, ti->ti_offset)); } /* Sun ABI */ -void *__tls_get_addr(tls_index *ti) +void * +__tls_get_addr(tls_index *ti) { - Elf_Addr** segbase; - - __asm __volatile("movl %%gs:0, %0" : "=r" (segbase)); + Elf_Addr **dtvp; - return tls_get_addr_common(&segbase[1], ti->ti_module, ti->ti_offset); + dtvp = _get_tp(); + return (tls_get_addr_common(dtvp, ti->ti_module, ti->ti_offset)); } size_t diff --git a/libexec/rtld-elf/i386/rtld_machdep.h b/libexec/rtld-elf/i386/rtld_machdep.h --- a/libexec/rtld-elf/i386/rtld_machdep.h +++ b/libexec/rtld-elf/i386/rtld_machdep.h @@ -74,6 +74,8 @@ #define md_abi_variant_hook(x) +#define TLS_DTV_OFFSET 0 + size_t calculate_first_tls_offset(size_t size, size_t align, size_t offset); size_t calculate_tls_offset(size_t prev_offset, size_t prev_size, size_t size, size_t align, size_t offset); diff --git a/libexec/rtld-elf/mips/reloc.c b/libexec/rtld-elf/mips/reloc.c --- a/libexec/rtld-elf/mips/reloc.c +++ b/libexec/rtld-elf/mips/reloc.c @@ -776,69 +776,15 @@ sysarch(MIPS_SET_TLS, tls); } -#ifdef __mips_n64 -void * -_mips_get_tls(void) -{ - uint64_t _rv; - - __asm__ __volatile__ ( - ".set\tpush\n\t" - ".set\tmips64r2\n\t" - "rdhwr\t%0, $29\n\t" - ".set\tpop" - : "=r" (_rv)); - /* - * XXXSS See 'git show c6be4f4d2d1b71c04de5d3bbb6933ce2dbcdb317' - * - * Remove the offset since this really a request to get the TLS - * pointer via sysarch() (in theory). Of course, this may go away - * once the TLS code is rewritten. - */ - _rv = _rv - TLS_TP_OFFSET - TLS_TCB_SIZE; - - return (void *)_rv; -} - -#else /* mips 32 */ - -void * -_mips_get_tls(void) -{ - uint32_t _rv; - - __asm__ __volatile__ ( - ".set\tpush\n\t" - ".set\tmips32r2\n\t" - "rdhwr\t%0, $29\n\t" - ".set\tpop" - : "=r" (_rv)); - /* - * XXXSS See 'git show c6be4f4d2d1b71c04de5d3bbb6933ce2dbcdb317' - * - * Remove the offset since this really a request to get the TLS - * pointer via sysarch() (in theory). Of course, this may go away - * once the TLS code is rewritten. - */ - _rv = _rv - TLS_TP_OFFSET - TLS_TCB_SIZE; - - return (void *)_rv; -} -#endif /* ! __mips_n64 */ - void * __tls_get_addr(tls_index* ti) { - Elf_Addr** tls; + Elf_Addr **tls; char *p; -#ifdef TLS_USE_SYSARCH - sysarch(MIPS_GET_TLS, &tls); -#else - tls = _mips_get_tls(); -#endif - - p = tls_get_addr_common(tls, ti->ti_module, ti->ti_offset + TLS_DTP_OFFSET); + tls = _get_tp(); + p = tls_get_addr_common(tls, ti->ti_module, ti->ti_offset + + TLS_DTP_OFFSET); return (p); } diff --git a/libexec/rtld-elf/mips/rtld_machdep.h b/libexec/rtld-elf/mips/rtld_machdep.h --- a/libexec/rtld-elf/mips/rtld_machdep.h +++ b/libexec/rtld-elf/mips/rtld_machdep.h @@ -44,8 +44,6 @@ const struct Struct_Obj_Entry *defobj, const struct Struct_Obj_Entry *obj, const Elf_Rel *rel); Elf_Addr _mips_rtld_bind(struct Struct_Obj_Entry *obj, Elf_Size reloff); -void *_mips_get_tls(void); - #define make_function_pointer(def, defobj) \ ((defobj)->relocbase + (def)->st_value) @@ -80,4 +78,6 @@ #define md_abi_variant_hook(x) +#define TLS_DTV_OFFSET 0 + #endif diff --git a/libexec/rtld-elf/rtld-libc/Makefile.inc b/libexec/rtld-elf/rtld-libc/Makefile.inc --- a/libexec/rtld-elf/rtld-libc/Makefile.inc +++ b/libexec/rtld-elf/rtld-libc/Makefile.inc @@ -67,6 +67,8 @@ _libc_other_objects+=syncicache .endif +_libc_other_objects+=_get_tp + # Extract all the .o files from libc_nossp_pic.a. This ensures that # we don't accidentally pull in the interposing table or similar by linking # directly against libc_nossp_pic.a diff --git a/libexec/rtld-elf/rtld.h b/libexec/rtld-elf/rtld.h --- a/libexec/rtld-elf/rtld.h +++ b/libexec/rtld-elf/rtld.h @@ -401,6 +401,7 @@ void free_tls_offset(Obj_Entry *obj); const Ver_Entry *fetch_ventry(const Obj_Entry *obj, unsigned long); int convert_prot(int elfflags); +void *_get_tp(void); /* libc implementation */ /* * MD function declarations. diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c --- a/libexec/rtld-elf/rtld.c +++ b/libexec/rtld-elf/rtld.c @@ -166,6 +166,7 @@ static int symlook_needed(SymLook *, const Needed_Entry *, DoneList *); static int symlook_obj1_sysv(SymLook *, const Obj_Entry *); static int symlook_obj1_gnu(SymLook *, const Obj_Entry *); +static void *tls_get_addr_slow(Elf_Addr **, int, size_t, bool) __noinline; static void trace_loaded_objects(Obj_Entry *); static void unlink_object(Obj_Entry *); static void unload_object(Obj_Entry *, RtldLockState *lockstate); @@ -3909,16 +3910,16 @@ static void rtld_fill_dl_phdr_info(const Obj_Entry *obj, struct dl_phdr_info *phdr_info) { - tls_index ti; + Elf_Addr **dtvp; phdr_info->dlpi_addr = (Elf_Addr)obj->relocbase; phdr_info->dlpi_name = obj->path; phdr_info->dlpi_phdr = obj->phdr; phdr_info->dlpi_phnum = obj->phsize / sizeof(obj->phdr[0]); phdr_info->dlpi_tls_modid = obj->tlsindex; - ti.ti_module = obj->tlsindex; - ti.ti_offset = 0; - phdr_info->dlpi_tls_data = __tls_get_addr(&ti); + dtvp = _get_tp(); + phdr_info->dlpi_tls_data = (char *)tls_get_addr_slow(dtvp, + obj->tlsindex, 0, true) + TLS_DTV_OFFSET; phdr_info->dlpi_adds = obj_loads; phdr_info->dlpi_subs = obj_loads - obj_count; } @@ -4871,39 +4872,42 @@ /* * Common code for MD __tls_get_addr(). */ -static void *tls_get_addr_slow(Elf_Addr **, int, size_t) __noinline; static void * -tls_get_addr_slow(Elf_Addr **dtvp, int index, size_t offset) +tls_get_addr_slow(Elf_Addr **dtvp, int index, size_t offset, bool locked) { - Elf_Addr *newdtv, *dtv; - RtldLockState lockstate; - int to_copy; + Elf_Addr *newdtv, *dtv; + RtldLockState lockstate; + int to_copy; - dtv = *dtvp; - /* Check dtv generation in case new modules have arrived */ - if (dtv[0] != tls_dtv_generation) { - wlock_acquire(rtld_bind_lock, &lockstate); - newdtv = xcalloc(tls_max_index + 2, sizeof(Elf_Addr)); - to_copy = dtv[1]; - if (to_copy > tls_max_index) - to_copy = tls_max_index; - memcpy(&newdtv[2], &dtv[2], to_copy * sizeof(Elf_Addr)); - newdtv[0] = tls_dtv_generation; - newdtv[1] = tls_max_index; - free(dtv); - lock_release(rtld_bind_lock, &lockstate); - dtv = *dtvp = newdtv; - } + dtv = *dtvp; + /* Check dtv generation in case new modules have arrived */ + if (dtv[0] != tls_dtv_generation) { + if (!locked) + wlock_acquire(rtld_bind_lock, &lockstate); + newdtv = xcalloc(tls_max_index + 2, sizeof(Elf_Addr)); + to_copy = dtv[1]; + if (to_copy > tls_max_index) + to_copy = tls_max_index; + memcpy(&newdtv[2], &dtv[2], to_copy * sizeof(Elf_Addr)); + newdtv[0] = tls_dtv_generation; + newdtv[1] = tls_max_index; + free(dtv); + if (!locked) + lock_release(rtld_bind_lock, &lockstate); + dtv = *dtvp = newdtv; + } - /* Dynamically allocate module TLS if necessary */ - if (dtv[index + 1] == 0) { - /* Signal safe, wlock will block out signals. */ - wlock_acquire(rtld_bind_lock, &lockstate); - if (!dtv[index + 1]) - dtv[index + 1] = (Elf_Addr)allocate_module_tls(index); - lock_release(rtld_bind_lock, &lockstate); - } - return ((void *)(dtv[index + 1] + offset)); + /* Dynamically allocate module TLS if necessary */ + if (dtv[index + 1] == 0) { + /* Signal safe, wlock will block out signals. */ + if (!locked) + wlock_acquire(rtld_bind_lock, &lockstate); + if (!dtv[index + 1]) + dtv[index + 1] = (Elf_Addr)allocate_module_tls(index); + if (!locked) + lock_release(rtld_bind_lock, &lockstate); + } + return ((void *)(dtv[index + 1] + offset)); } void * @@ -4916,7 +4920,7 @@ if (__predict_true(dtv[0] == tls_dtv_generation && dtv[index + 1] != 0)) return ((void *)(dtv[index + 1] + offset)); - return (tls_get_addr_slow(dtvp, index, offset)); + return (tls_get_addr_slow(dtvp, index, offset, false)); } #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \ diff --git a/libexec/rtld-elf/rtld_lock.h b/libexec/rtld-elf/rtld_lock.h --- a/libexec/rtld-elf/rtld_lock.h +++ b/libexec/rtld-elf/rtld_lock.h @@ -46,9 +46,13 @@ void (*at_fork)(void); }; -extern void _rtld_thread_init(struct RtldLockInfo *) __exported; -extern void _rtld_atfork_pre(int *) __exported; -extern void _rtld_atfork_post(int *) __exported; +#if defined(IN_RTLD) || defined(PTHREAD_KERNEL) + +void _rtld_thread_init(struct RtldLockInfo *) __exported; +void _rtld_atfork_pre(int *) __exported; +void _rtld_atfork_post(int *) __exported; + +#endif /* IN_RTLD || PTHREAD_KERNEL */ #ifdef IN_RTLD