diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c --- a/libexec/rtld-elf/rtld.c +++ b/libexec/rtld-elf/rtld.c @@ -82,9 +82,15 @@ char *msg; }; +struct tcb_list_entry { + TAILQ_ENTRY(tcb_list_entry) next; +}; + /* * Function declarations. */ +static bool allocate_tls_offset_common(size_t *offp, size_t tlssize, + size_t tlsalign, size_t tlspoffset); static const char *basename(const char *); static void digest_dynamic1(Obj_Entry *, int, const Elf_Dyn **, const Elf_Dyn **, const Elf_Dyn **); @@ -92,7 +98,7 @@ const Elf_Dyn *); static bool digest_dynamic(Obj_Entry *, int); static Obj_Entry *digest_phdr(const Elf_Phdr *, int, caddr_t, const char *); -static void distribute_static_tls(Objlist *, RtldLockState *); +static void distribute_static_tls(Objlist *); static Obj_Entry *dlcheck(void *); static int dlclose_locked(void *, RtldLockState *); static Obj_Entry *dlopen_object(const char *name, int fd, Obj_Entry *refobj, @@ -303,6 +309,10 @@ Elf_Addr tls_dtv_generation = 1; /* Used to detect when dtv size changes */ int tls_max_index = 1; /* Largest module index allocated */ +static TAILQ_HEAD(, tcb_list_entry) tcb_list = + TAILQ_HEAD_INITIALIZER(tcb_list); +static size_t tcb_list_entry_offset; + static bool ld_library_path_rpath = false; bool ld_fast_sigblock = false; @@ -929,6 +939,19 @@ allocate_tls_offset(entry->obj); } + if (!allocate_tls_offset_common(&tcb_list_entry_offset, + sizeof(struct tcb_list_entry), _Alignof(struct tcb_list_entry), + 0)) { + /* + * This should be impossible as the static block size is not + * yet fixed, but catch and diagnose it failing if that ever + * changes or somehow turns out to be false. + */ + _rtld_error("Could not allocate offset for tcb_list_entry"); + rtld_die(); + } + dbg("tcb_list_entry_offset %zu", tcb_list_entry_offset); + if (relocate_objects(obj_main, ld_bind_now != NULL && *ld_bind_now != '\0', &obj_rtld, SYMLOOK_EARLY, NULL) == -1) @@ -3973,7 +3996,7 @@ if ((lo_flags & RTLD_LO_EARLY) == 0) { map_stacks_exec(lockstate); if (obj != NULL) - distribute_static_tls(&initlist, lockstate); + distribute_static_tls(&initlist); } if (initlist_objects_ifunc(&initlist, (mode & RTLD_MODEMASK) == @@ -5400,6 +5423,44 @@ return (tls_get_addr_slow(tcb, index, offset, false)); } +static struct tcb * +tcb_from_tcb_list_entry(struct tcb_list_entry *tcbelm) +{ +#ifdef TLS_VARIANT_I + return ((struct tcb *)((char *)tcbelm - tcb_list_entry_offset)); +#else + return ((struct tcb *)((char *)tcbelm + tcb_list_entry_offset)); +#endif +} + +static struct tcb_list_entry * +tcb_list_entry_from_tcb(struct tcb *tcb) +{ +#ifdef TLS_VARIANT_I + return ((struct tcb_list_entry *)((char *)tcb + tcb_list_entry_offset)); +#else + return ((struct tcb_list_entry *)((char *)tcb - tcb_list_entry_offset)); +#endif +} + +static void +tcb_list_insert(struct tcb *tcb) +{ + struct tcb_list_entry *tcbelm; + + tcbelm = tcb_list_entry_from_tcb(tcb); + TAILQ_INSERT_TAIL(&tcb_list, tcbelm, next); +} + +static void +tcb_list_remove(struct tcb *tcb) +{ + struct tcb_list_entry *tcbelm; + + tcbelm = tcb_list_entry_from_tcb(tcb); + TAILQ_REMOVE(&tcb_list, tcbelm, next); +} + #ifdef TLS_VARIANT_I /* @@ -5513,6 +5574,7 @@ } } + tcb_list_insert(tcb); return (tcb); } @@ -5524,6 +5586,8 @@ size_t post_size; size_t i, tls_init_align __unused; + tcb_list_remove(tcb); + assert(tcbsize >= TLS_TCB_SIZE); tls_init_align = MAX(obj_main->tlsalign, 1); @@ -5624,6 +5688,7 @@ } } + tcb_list_insert(tcb); return (tcb); } @@ -5635,6 +5700,8 @@ size_t i; uintptr_t tlsstart, tlsend; + tcb_list_remove(tcb); + /* * Figure out the size of the initial TLS block so that we can * find stuff which ___tls_get_addr() allocated dynamically. @@ -6136,25 +6203,29 @@ } static void -distribute_static_tls(Objlist *list, RtldLockState *lockstate) +distribute_static_tls(Objlist *list) { - Objlist_Entry *elm; + struct tcb_list_entry *tcbelm; + Objlist_Entry *objelm; + struct tcb *tcb; Obj_Entry *obj; - void (*distrib)(size_t, void *, size_t, size_t); + char *tlsbase; - distrib = (void (*)(size_t, void *, size_t, size_t))( - uintptr_t)get_program_var_addr("__pthread_distribute_static_tls", - lockstate); - if (distrib == NULL) - return; - STAILQ_FOREACH(elm, list, link) { - obj = elm->obj; + STAILQ_FOREACH(objelm, list, link) { + obj = objelm->obj; if (obj->marker || !obj->tls_static || obj->static_tls_copied) continue; - lock_release(rtld_bind_lock, lockstate); - distrib(obj->tlsoffset, obj->tlsinit, obj->tlsinitsize, - obj->tlssize); - wlock_acquire(rtld_bind_lock, lockstate); + TAILQ_FOREACH(tcbelm, &tcb_list, next) { + tcb = tcb_from_tcb_list_entry(tcbelm); +#ifdef TLS_VARIANT_I + tlsbase = (char *)tcb + obj->tlsoffset; +#else + tlsbase = (char *)tcb - obj->tlsoffset; +#endif + memcpy(tlsbase, obj->tlsinit, obj->tlsinitsize); + memset(tlsbase + obj->tlsinitsize, 0, + obj->tlssize - obj->tlsinitsize); + } obj->static_tls_copied = true; } }