diff --git a/lib/libthr/pthread.map b/lib/libthr/pthread.map index 5e36239e8e22..9f82a1b7ca4d 100644 --- a/lib/libthr/pthread.map +++ b/lib/libthr/pthread.map @@ -1,406 +1,408 @@ /* * $FreeBSD$ */ /* * Use the same naming scheme as libc. */ FBSD_1.0 { __error; accept; aio_suspend; close; connect; creat; execve; fcntl; fork; fsync; msync; nanosleep; open; pause; poll; pselect; pthread_atfork; pthread_barrier_destroy; pthread_barrier_init; pthread_barrier_wait; pthread_barrierattr_destroy; pthread_barrierattr_getpshared; pthread_barrierattr_init; pthread_barrierattr_setpshared; pthread_attr_destroy; pthread_attr_get_np; pthread_attr_getdetachstate; pthread_attr_getguardsize; pthread_attr_getinheritsched; pthread_attr_getschedparam; pthread_attr_getschedpolicy; pthread_attr_getscope; pthread_attr_getstack; pthread_attr_getstackaddr; pthread_attr_getstacksize; pthread_attr_init; pthread_attr_setcreatesuspend_np; pthread_attr_setdetachstate; pthread_attr_setguardsize; pthread_attr_setinheritsched; pthread_attr_setschedparam; pthread_attr_setschedpolicy; pthread_attr_setscope; pthread_attr_setstack; pthread_attr_setstackaddr; pthread_attr_setstacksize; pthread_cancel; pthread_cleanup_pop; pthread_cleanup_push; pthread_cond_broadcast; pthread_cond_destroy; pthread_cond_init; pthread_cond_signal; pthread_cond_timedwait; pthread_cond_wait; pthread_condattr_destroy; pthread_condattr_getclock; pthread_condattr_getpshared; pthread_condattr_init; pthread_condattr_setclock; pthread_condattr_setpshared; pthread_create; pthread_detach; pthread_equal; pthread_exit; pthread_getconcurrency; pthread_getprio; pthread_getschedparam; pthread_getspecific; pthread_join; pthread_key_create; pthread_key_delete; pthread_kill; pthread_main_np; pthread_multi_np; pthread_mutex_destroy; pthread_mutex_getprioceiling; pthread_mutex_init; pthread_mutex_lock; pthread_mutex_setprioceiling; pthread_mutex_timedlock; pthread_mutex_trylock; pthread_mutex_unlock; pthread_mutexattr_destroy; pthread_mutexattr_getkind_np; pthread_mutexattr_getprioceiling; pthread_mutexattr_getpshared; pthread_mutexattr_getprotocol; pthread_mutexattr_gettype; pthread_mutexattr_init; pthread_mutexattr_setkind_np; pthread_mutexattr_setprioceiling; pthread_mutexattr_setprotocol; pthread_mutexattr_setpshared; pthread_mutexattr_settype; pthread_once; pthread_resume_all_np; pthread_resume_np; pthread_rwlock_destroy; pthread_rwlock_init; pthread_rwlock_rdlock; pthread_rwlock_timedrdlock; pthread_rwlock_timedwrlock; pthread_rwlock_tryrdlock; pthread_rwlock_trywrlock; pthread_rwlock_unlock; pthread_rwlock_wrlock; pthread_rwlockattr_destroy; pthread_rwlockattr_getpshared; pthread_rwlockattr_init; pthread_rwlockattr_setpshared; pthread_set_name_np; pthread_self; pthread_setcancelstate; pthread_setcanceltype; pthread_setconcurrency; pthread_setprio; pthread_setschedparam; pthread_setspecific; pthread_sigmask; pthread_single_np; pthread_spin_destroy; pthread_spin_init; pthread_spin_lock; pthread_spin_trylock; pthread_spin_unlock; pthread_suspend_all_np; pthread_suspend_np; pthread_switch_add_np; pthread_switch_delete_np; pthread_testcancel; pthread_timedjoin_np; pthread_yield; raise; read; readv; recvfrom; recvmsg; select; sendmsg; sendto; sigaction; sigprocmask; sigsuspend; sigwait; sigwaitinfo; sigtimedwait; sleep; system; tcdrain; usleep; wait; wait3; wait4; waitpid; write; writev; }; /* * List the private interfaces reserved for use in FreeBSD libraries. * These are not part of our application ABI. */ FBSDprivate_1.0 { ___creat; ___pause; ___pselect; ___sigwait; ___sleep; ___system; ___tcdrain; ___usleep; ___wait; ___waitpid; __accept; __aio_suspend; __close; __connect; __fcntl; __fsync; __msync; __nanosleep; __open; __openat; __poll; __pthread_cond_timedwait; __pthread_cond_wait; __pthread_cxa_finalize; __pthread_mutex_init; __pthread_mutex_lock; __pthread_mutex_timedlock; __pthread_mutex_trylock; __read; __readv; __recvfrom; __recvmsg; __select; __sendmsg; __sendto; __sigsuspend; __sigtimedwait; __sigwaitinfo; __wait3; __wait4; __write; __writev; _fork; _pthread_atfork; _pthread_barrier_destroy; _pthread_barrier_init; _pthread_barrier_wait; _pthread_barrierattr_destroy; _pthread_barrierattr_getpshared; _pthread_barrierattr_init; _pthread_barrierattr_setpshared; _pthread_attr_destroy; _pthread_attr_get_np; _pthread_attr_getaffinity_np; _pthread_attr_getdetachstate; _pthread_attr_getguardsize; _pthread_attr_getinheritsched; _pthread_attr_getschedparam; _pthread_attr_getschedpolicy; _pthread_attr_getscope; _pthread_attr_getstack; _pthread_attr_getstackaddr; _pthread_attr_getstacksize; _pthread_attr_init; _pthread_attr_setaffinity_np; _pthread_attr_setcreatesuspend_np; _pthread_attr_setdetachstate; _pthread_attr_setguardsize; _pthread_attr_setinheritsched; _pthread_attr_setschedparam; _pthread_attr_setschedpolicy; _pthread_attr_setscope; _pthread_attr_setstack; _pthread_attr_setstackaddr; _pthread_attr_setstacksize; _pthread_cancel; _pthread_cancel_enter; _pthread_cancel_leave; _pthread_cleanup_pop; _pthread_cleanup_push; _pthread_cond_broadcast; _pthread_cond_destroy; _pthread_cond_init; _pthread_cond_signal; _pthread_cond_timedwait; _pthread_cond_wait; _pthread_condattr_destroy; _pthread_condattr_getclock; _pthread_condattr_getpshared; _pthread_condattr_init; _pthread_condattr_setclock; _pthread_condattr_setpshared; _pthread_create; _pthread_detach; _pthread_equal; _pthread_exit; _pthread_getaffinity_np; _pthread_getconcurrency; _pthread_getcpuclockid; _pthread_getprio; _pthread_getschedparam; _pthread_getspecific; _pthread_join; _pthread_key_create; _pthread_key_delete; _pthread_kill; _pthread_main_np; _pthread_multi_np; _pthread_mutex_destroy; _pthread_mutex_getprioceiling; _pthread_mutex_getspinloops_np; _pthread_mutex_getyieldloops_np; _pthread_mutex_init; _pthread_mutex_init_calloc_cb; _pthread_mutex_isowned_np; _pthread_mutex_lock; _pthread_mutex_setprioceiling; _pthread_mutex_setspinloops_np; _pthread_mutex_setyieldloops_np; _pthread_mutex_timedlock; _pthread_mutex_trylock; _pthread_mutex_unlock; _pthread_mutexattr_destroy; _pthread_mutexattr_getkind_np; _pthread_mutexattr_getprioceiling; _pthread_mutexattr_getprotocol; _pthread_mutexattr_getpshared; _pthread_mutexattr_gettype; _pthread_mutexattr_init; _pthread_mutexattr_setkind_np; _pthread_mutexattr_setprioceiling; _pthread_mutexattr_setprotocol; _pthread_mutexattr_setpshared; _pthread_mutexattr_settype; _pthread_once; _pthread_resume_all_np; _pthread_resume_np; _pthread_rwlock_destroy; _pthread_rwlock_init; _pthread_rwlock_rdlock; _pthread_rwlock_timedrdlock; _pthread_rwlock_timedwrlock; _pthread_rwlock_tryrdlock; _pthread_rwlock_trywrlock; _pthread_rwlock_unlock; _pthread_rwlock_wrlock; _pthread_rwlockattr_destroy; _pthread_rwlockattr_getpshared; _pthread_rwlockattr_init; _pthread_rwlockattr_setpshared; _pthread_self; _pthread_set_name_np; _pthread_setaffinity_np; _pthread_setcancelstate; _pthread_setcanceltype; _pthread_setconcurrency; _pthread_setprio; _pthread_setschedparam; _pthread_setspecific; _pthread_sigmask; _pthread_single_np; _pthread_spin_destroy; _pthread_spin_init; _pthread_spin_lock; _pthread_spin_trylock; _pthread_spin_unlock; _pthread_suspend_all_np; _pthread_suspend_np; _pthread_switch_add_np; _pthread_switch_delete_np; _pthread_testcancel; _pthread_timedjoin_np; _pthread_yield; _raise; _setcontext; _sigaction; _sigprocmask; _sigsuspend; _sigtimedwait; _sigwait; _sigwaitinfo; _spinlock; _spinlock_debug; _spinunlock; _swapcontext; /* Debugger needs these. */ _libthr_debug; _thread_active_threads; _thread_bp_create; _thread_bp_death; _thread_event_mask; _thread_keytable; _thread_last_event; _thread_list; _thread_max_keys; _thread_off_attr_flags; _thread_off_dtv; _thread_off_event_buf; _thread_off_event_mask; _thread_off_key_allocated; _thread_off_key_destructor; _thread_off_linkmap; _thread_off_next; _thread_off_report_events; _thread_off_state; _thread_off_tcb; _thread_off_tid; _thread_off_tlsindex; _thread_size_key; _thread_state_running; _thread_state_zoombie; + + __pthread_map_stacks_exec; }; FBSD_1.1 { __pthread_cleanup_pop_imp; __pthread_cleanup_push_imp; pthread_attr_getaffinity_np; pthread_attr_setaffinity_np; pthread_getaffinity_np; pthread_getcpuclockid; pthread_setaffinity_np; pthread_mutex_getspinloops_np; pthread_mutex_getyieldloops_np; pthread_mutex_isowned_np; pthread_mutex_setspinloops_np; pthread_mutex_setyieldloops_np; }; FBSD_1.2 { openat; setcontext; swapcontext; }; diff --git a/lib/libthr/thread/thr_create.c b/lib/libthr/thread/thr_create.c index f0af20b0badf..a41b33f6039b 100644 --- a/lib/libthr/thread/thr_create.c +++ b/lib/libthr/thread/thr_create.c @@ -1,277 +1,288 @@ /* * Copyright (c) 2003 Daniel M. Eischen * Copyright (c) 2005, David Xu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #include "namespace.h" #include #include #include #include +#include #include #include #include #include #include #include "un-namespace.h" #include "thr_private.h" static int create_stack(struct pthread_attr *pattr); static void thread_start(struct pthread *curthread); __weak_reference(_pthread_create, pthread_create); int _pthread_create(pthread_t * thread, const pthread_attr_t * attr, void *(*start_routine) (void *), void *arg) { struct pthread *curthread, *new_thread; struct thr_param param; struct sched_param sched_param; struct rtprio rtp; int ret = 0, locked, create_suspended; sigset_t set, oset; cpuset_t *cpusetp = NULL; int cpusetsize = 0; + int old_stack_prot; _thr_check_init(); /* * Tell libc and others now they need lock to protect their data. */ if (_thr_isthreaded() == 0 && _thr_setthreaded(1)) return (EAGAIN); curthread = _get_curthread(); if ((new_thread = _thr_alloc(curthread)) == NULL) return (EAGAIN); memset(¶m, 0, sizeof(param)); if (attr == NULL || *attr == NULL) /* Use the default thread attributes: */ new_thread->attr = _pthread_attr_default; else { new_thread->attr = *(*attr); cpusetp = new_thread->attr.cpuset; cpusetsize = new_thread->attr.cpusetsize; new_thread->attr.cpuset = NULL; new_thread->attr.cpusetsize = 0; } if (new_thread->attr.sched_inherit == PTHREAD_INHERIT_SCHED) { /* inherit scheduling contention scope */ if (curthread->attr.flags & PTHREAD_SCOPE_SYSTEM) new_thread->attr.flags |= PTHREAD_SCOPE_SYSTEM; else new_thread->attr.flags &= ~PTHREAD_SCOPE_SYSTEM; new_thread->attr.prio = curthread->attr.prio; new_thread->attr.sched_policy = curthread->attr.sched_policy; } new_thread->tid = TID_TERMINATED; + old_stack_prot = _rtld_get_stack_prot(); if (create_stack(&new_thread->attr) != 0) { /* Insufficient memory to create a stack: */ _thr_free(curthread, new_thread); return (EAGAIN); } /* * Write a magic value to the thread structure * to help identify valid ones: */ new_thread->magic = THR_MAGIC; new_thread->start_routine = start_routine; new_thread->arg = arg; new_thread->cancel_enable = 1; new_thread->cancel_async = 0; /* Initialize the mutex queue: */ TAILQ_INIT(&new_thread->mutexq); TAILQ_INIT(&new_thread->pp_mutexq); /* Initialise hooks in the thread structure: */ if (new_thread->attr.suspend == THR_CREATE_SUSPENDED) { new_thread->flags = THR_FLAGS_NEED_SUSPEND; create_suspended = 1; } else { create_suspended = 0; } new_thread->state = PS_RUNNING; if (new_thread->attr.flags & PTHREAD_CREATE_DETACHED) new_thread->flags |= THR_FLAGS_DETACHED; /* Add the new thread. */ new_thread->refcount = 1; _thr_link(curthread, new_thread); + + /* + * Handle the race between __pthread_map_stacks_exec and + * thread linkage. + */ + if (old_stack_prot != _rtld_get_stack_prot()) + _thr_stack_fix_protection(new_thread); + /* Return thread pointer eariler so that new thread can use it. */ (*thread) = new_thread; if (SHOULD_REPORT_EVENT(curthread, TD_CREATE) || cpusetp != NULL) { THR_THREAD_LOCK(curthread, new_thread); locked = 1; } else locked = 0; param.start_func = (void (*)(void *)) thread_start; param.arg = new_thread; param.stack_base = new_thread->attr.stackaddr_attr; param.stack_size = new_thread->attr.stacksize_attr; param.tls_base = (char *)new_thread->tcb; param.tls_size = sizeof(struct tcb); param.child_tid = &new_thread->tid; param.parent_tid = &new_thread->tid; param.flags = 0; if (new_thread->attr.flags & PTHREAD_SCOPE_SYSTEM) param.flags |= THR_SYSTEM_SCOPE; if (new_thread->attr.sched_inherit == PTHREAD_INHERIT_SCHED) param.rtp = NULL; else { sched_param.sched_priority = new_thread->attr.prio; _schedparam_to_rtp(new_thread->attr.sched_policy, &sched_param, &rtp); param.rtp = &rtp; } /* Schedule the new thread. */ if (create_suspended) { SIGFILLSET(set); SIGDELSET(set, SIGTRAP); __sys_sigprocmask(SIG_SETMASK, &set, &oset); new_thread->sigmask = oset; SIGDELSET(new_thread->sigmask, SIGCANCEL); } ret = thr_new(¶m, sizeof(param)); if (ret != 0) { ret = errno; /* * Translate EPROCLIM into well-known POSIX code EAGAIN. */ if (ret == EPROCLIM) ret = EAGAIN; } if (create_suspended) __sys_sigprocmask(SIG_SETMASK, &oset, NULL); if (ret != 0) { if (!locked) THR_THREAD_LOCK(curthread, new_thread); new_thread->state = PS_DEAD; new_thread->tid = TID_TERMINATED; new_thread->flags |= THR_FLAGS_DETACHED; new_thread->refcount--; if (new_thread->flags & THR_FLAGS_NEED_SUSPEND) { new_thread->cycle++; _thr_umtx_wake(&new_thread->cycle, INT_MAX, 0); } _thr_try_gc(curthread, new_thread); /* thread lock released */ atomic_add_int(&_thread_active_threads, -1); } else if (locked) { if (cpusetp != NULL) { if (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, TID(new_thread), cpusetsize, cpusetp)) { ret = errno; /* kill the new thread */ new_thread->force_exit = 1; new_thread->flags |= THR_FLAGS_DETACHED; _thr_try_gc(curthread, new_thread); /* thread lock released */ goto out; } } _thr_report_creation(curthread, new_thread); THR_THREAD_UNLOCK(curthread, new_thread); } out: if (ret) (*thread) = 0; return (ret); } static int create_stack(struct pthread_attr *pattr) { int ret; /* Check if a stack was specified in the thread attributes: */ if ((pattr->stackaddr_attr) != NULL) { pattr->guardsize_attr = 0; pattr->flags |= THR_STACK_USER; ret = 0; } else ret = _thr_stack_alloc(pattr); return (ret); } static void thread_start(struct pthread *curthread) { sigset_t set; if (curthread->attr.suspend == THR_CREATE_SUSPENDED) set = curthread->sigmask; /* * This is used as a serialization point to allow parent * to report 'new thread' event to debugger or tweak new thread's * attributes before the new thread does real-world work. */ THR_LOCK(curthread); THR_UNLOCK(curthread); if (curthread->force_exit) _pthread_exit(PTHREAD_CANCELED); if (curthread->attr.suspend == THR_CREATE_SUSPENDED) { #if 0 /* Done in THR_UNLOCK() */ _thr_ast(curthread); #endif /* * Parent thread have stored signal mask for us, * we should restore it now. */ __sys_sigprocmask(SIG_SETMASK, &set, NULL); } #ifdef _PTHREAD_FORCED_UNWIND curthread->unwind_stackend = (char *)curthread->attr.stackaddr_attr + curthread->attr.stacksize_attr; #endif /* Run the current thread's start routine with argument: */ _pthread_exit(curthread->start_routine(curthread->arg)); /* This point should never be reached. */ PANIC("Thread has resumed after exit"); } diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h index 9df97aa43118..9b9227b2bece 100644 --- a/lib/libthr/thread/thr_private.h +++ b/lib/libthr/thread/thr_private.h @@ -1,904 +1,905 @@ /* * Copyright (C) 2005 Daniel M. Eischen * Copyright (c) 2005 David Xu * Copyright (c) 1995-1998 John Birrell . * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _THR_PRIVATE_H #define _THR_PRIVATE_H /* * Include files. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SYM_FB10(sym) __CONCAT(sym, _fb10) #define SYM_FBP10(sym) __CONCAT(sym, _fbp10) #define WEAK_REF(sym, alias) __weak_reference(sym, alias) #define SYM_COMPAT(sym, impl, ver) __sym_compat(sym, impl, ver) #define SYM_DEFAULT(sym, impl, ver) __sym_default(sym, impl, ver) #define FB10_COMPAT(func, sym) \ WEAK_REF(func, SYM_FB10(sym)); \ SYM_COMPAT(sym, SYM_FB10(sym), FBSD_1.0) #define FB10_COMPAT_PRIVATE(func, sym) \ WEAK_REF(func, SYM_FBP10(sym)); \ SYM_DEFAULT(sym, SYM_FBP10(sym), FBSDprivate_1.0) #include "pthread_md.h" #include "thr_umtx.h" #include "thread_db.h" #ifdef _PTHREAD_FORCED_UNWIND #define _BSD_SOURCE #include #endif typedef TAILQ_HEAD(pthreadlist, pthread) pthreadlist; typedef TAILQ_HEAD(atfork_head, pthread_atfork) atfork_head; TAILQ_HEAD(mutex_queue, pthread_mutex); /* Signal to do cancellation */ #define SIGCANCEL 32 /* * Kernel fatal error handler macro. */ #define PANIC(string) _thread_exit(__FILE__,__LINE__,string) /* Output debug messages like this: */ #define stdout_debug(args...) _thread_printf(STDOUT_FILENO, ##args) #define stderr_debug(args...) _thread_printf(STDERR_FILENO, ##args) #ifdef _PTHREADS_INVARIANTS #define THR_ASSERT(cond, msg) do { \ if (__predict_false(!(cond))) \ PANIC(msg); \ } while (0) #else #define THR_ASSERT(cond, msg) #endif #ifdef PIC # define STATIC_LIB_REQUIRE(name) #else # define STATIC_LIB_REQUIRE(name) __asm (".globl " #name) #endif #define TIMESPEC_ADD(dst, src, val) \ do { \ (dst)->tv_sec = (src)->tv_sec + (val)->tv_sec; \ (dst)->tv_nsec = (src)->tv_nsec + (val)->tv_nsec; \ if ((dst)->tv_nsec >= 1000000000) { \ (dst)->tv_sec++; \ (dst)->tv_nsec -= 1000000000; \ } \ } while (0) #define TIMESPEC_SUB(dst, src, val) \ do { \ (dst)->tv_sec = (src)->tv_sec - (val)->tv_sec; \ (dst)->tv_nsec = (src)->tv_nsec - (val)->tv_nsec; \ if ((dst)->tv_nsec < 0) { \ (dst)->tv_sec--; \ (dst)->tv_nsec += 1000000000; \ } \ } while (0) /* XXX These values should be same as those defined in pthread.h */ #define THR_MUTEX_INITIALIZER ((struct pthread_mutex *)NULL) #define THR_ADAPTIVE_MUTEX_INITIALIZER ((struct pthread_mutex *)1) #define THR_MUTEX_DESTROYED ((struct pthread_mutex *)2) #define THR_COND_INITIALIZER ((struct pthread_cond *)NULL) #define THR_COND_DESTROYED ((struct pthread_cond *)1) #define THR_RWLOCK_INITIALIZER ((struct pthread_rwlock *)NULL) #define THR_RWLOCK_DESTROYED ((struct pthread_rwlock *)1) #define PMUTEX_FLAG_TYPE_MASK 0x0ff #define PMUTEX_FLAG_PRIVATE 0x100 #define PMUTEX_FLAG_DEFERED 0x200 #define PMUTEX_TYPE(mtxflags) ((mtxflags) & PMUTEX_FLAG_TYPE_MASK) #define MAX_DEFER_WAITERS 50 struct pthread_mutex { /* * Lock for accesses to this structure. */ struct umutex m_lock; int m_flags; struct pthread *m_owner; int m_count; int m_spinloops; int m_yieldloops; /* * Link for all mutexes a thread currently owns. */ TAILQ_ENTRY(pthread_mutex) m_qe; }; struct pthread_mutex_attr { enum pthread_mutextype m_type; int m_protocol; int m_ceiling; }; #define PTHREAD_MUTEXATTR_STATIC_INITIALIZER \ { PTHREAD_MUTEX_DEFAULT, PTHREAD_PRIO_NONE, 0, MUTEX_FLAGS_PRIVATE } struct pthread_cond { __uint32_t __has_user_waiters; __uint32_t __has_kern_waiters; __uint32_t __flags; __uint32_t __clock_id; }; struct pthread_cond_attr { int c_pshared; int c_clockid; }; struct pthread_barrier { struct umutex b_lock; struct ucond b_cv; volatile int64_t b_cycle; volatile int b_count; volatile int b_waiters; }; struct pthread_barrierattr { int pshared; }; struct pthread_spinlock { struct umutex s_lock; }; /* * Flags for condition variables. */ #define COND_FLAGS_PRIVATE 0x01 #define COND_FLAGS_INITED 0x02 #define COND_FLAGS_BUSY 0x04 /* * Cleanup definitions. */ struct pthread_cleanup { struct pthread_cleanup *prev; void (*routine)(void *); void *routine_arg; int onheap; }; #define THR_CLEANUP_PUSH(td, func, arg) { \ struct pthread_cleanup __cup; \ \ __cup.routine = func; \ __cup.routine_arg = arg; \ __cup.onheap = 0; \ __cup.prev = (td)->cleanup; \ (td)->cleanup = &__cup; #define THR_CLEANUP_POP(td, exec) \ (td)->cleanup = __cup.prev; \ if ((exec) != 0) \ __cup.routine(__cup.routine_arg); \ } struct pthread_atfork { TAILQ_ENTRY(pthread_atfork) qe; void (*prepare)(void); void (*parent)(void); void (*child)(void); }; struct pthread_attr { #define pthread_attr_start_copy sched_policy int sched_policy; int sched_inherit; int prio; int suspend; #define THR_STACK_USER 0x100 /* 0xFF reserved for */ int flags; void *stackaddr_attr; size_t stacksize_attr; size_t guardsize_attr; #define pthread_attr_end_copy cpuset cpuset_t *cpuset; size_t cpusetsize; }; struct wake_addr { struct wake_addr *link; unsigned int value; char pad[12]; }; struct sleepqueue { TAILQ_HEAD(, pthread) sq_blocked; SLIST_HEAD(, sleepqueue) sq_freeq; LIST_ENTRY(sleepqueue) sq_hash; SLIST_ENTRY(sleepqueue) sq_flink; void *sq_wchan; int sq_type; }; /* * Thread creation state attributes. */ #define THR_CREATE_RUNNING 0 #define THR_CREATE_SUSPENDED 1 /* * Miscellaneous definitions. */ #define THR_STACK_DEFAULT (sizeof(void *) / 4 * 1024 * 1024) /* * Maximum size of initial thread's stack. This perhaps deserves to be larger * than the stacks of other threads, since many applications are likely to run * almost entirely on this stack. */ #define THR_STACK_INITIAL (THR_STACK_DEFAULT * 2) /* * Define priorities returned by kernel. */ #define THR_MIN_PRIORITY (_thr_priorities[SCHED_OTHER-1].pri_min) #define THR_MAX_PRIORITY (_thr_priorities[SCHED_OTHER-1].pri_max) #define THR_DEF_PRIORITY (_thr_priorities[SCHED_OTHER-1].pri_default) #define THR_MIN_RR_PRIORITY (_thr_priorities[SCHED_RR-1].pri_min) #define THR_MAX_RR_PRIORITY (_thr_priorities[SCHED_RR-1].pri_max) #define THR_DEF_RR_PRIORITY (_thr_priorities[SCHED_RR-1].pri_default) /* XXX The SCHED_FIFO should have same priority range as SCHED_RR */ #define THR_MIN_FIFO_PRIORITY (_thr_priorities[SCHED_FIFO_1].pri_min) #define THR_MAX_FIFO_PRIORITY (_thr_priorities[SCHED_FIFO-1].pri_max) #define THR_DEF_FIFO_PRIORITY (_thr_priorities[SCHED_FIFO-1].pri_default) struct pthread_prio { int pri_min; int pri_max; int pri_default; }; struct pthread_rwlockattr { int pshared; }; struct pthread_rwlock { struct urwlock lock; struct pthread *owner; }; /* * Thread states. */ enum pthread_state { PS_RUNNING, PS_DEAD }; struct pthread_specific_elem { const void *data; int seqno; }; struct pthread_key { volatile int allocated; int seqno; void (*destructor)(void *); }; /* * lwpid_t is 32bit but kernel thr API exports tid as long type * in very earily date. */ #define TID(thread) ((uint32_t) ((thread)->tid)) /* * Thread structure. */ struct pthread { /* Kernel thread id. */ long tid; #define TID_TERMINATED 1 /* * Lock for accesses to this thread structure. */ struct umutex lock; /* Internal condition variable cycle number. */ uint32_t cycle; /* How many low level locks the thread held. */ int locklevel; /* * Set to non-zero when this thread has entered a critical * region. We allow for recursive entries into critical regions. */ int critical_count; /* Signal blocked counter. */ int sigblock; /* Queue entry for list of all threads. */ TAILQ_ENTRY(pthread) tle; /* link for all threads in process */ /* Queue entry for GC lists. */ TAILQ_ENTRY(pthread) gcle; /* Hash queue entry. */ LIST_ENTRY(pthread) hle; /* Sleep queue entry */ TAILQ_ENTRY(pthread) wle; /* Threads reference count. */ int refcount; /* * Thread start routine, argument, stack pointer and thread * attributes. */ void *(*start_routine)(void *); void *arg; struct pthread_attr attr; #define SHOULD_CANCEL(thr) \ ((thr)->cancel_pending && (thr)->cancel_enable && \ (thr)->no_cancel == 0) /* Cancellation is enabled */ int cancel_enable; /* Cancellation request is pending */ int cancel_pending; /* Thread is at cancellation point */ int cancel_point; /* Cancellation is temporarily disabled */ int no_cancel; /* Asynchronouse cancellation is enabled */ int cancel_async; /* Cancellation is in progress */ int cancelling; /* Thread temporary signal mask. */ sigset_t sigmask; /* Thread should unblock SIGCANCEL. */ int unblock_sigcancel; /* In sigsuspend state */ int in_sigsuspend; /* deferred signal info */ siginfo_t deferred_siginfo; /* signal mask to restore. */ sigset_t deferred_sigmask; /* the sigaction should be used for deferred signal. */ struct sigaction deferred_sigact; /* Force new thread to exit. */ int force_exit; /* Thread state: */ enum pthread_state state; /* * Error variable used instead of errno. The function __error() * returns a pointer to this. */ int error; /* * The joiner is the thread that is joining to this thread. The * join status keeps track of a join operation to another thread. */ struct pthread *joiner; /* Miscellaneous flags; only set with scheduling lock held. */ int flags; #define THR_FLAGS_PRIVATE 0x0001 #define THR_FLAGS_NEED_SUSPEND 0x0002 /* thread should be suspended */ #define THR_FLAGS_SUSPENDED 0x0004 /* thread is suspended */ #define THR_FLAGS_DETACHED 0x0008 /* thread is detached */ /* Thread list flags; only set with thread list lock held. */ int tlflags; #define TLFLAGS_GC_SAFE 0x0001 /* thread safe for cleaning */ #define TLFLAGS_IN_TDLIST 0x0002 /* thread in all thread list */ #define TLFLAGS_IN_GCLIST 0x0004 /* thread in gc list */ /* Queue of currently owned NORMAL or PRIO_INHERIT type mutexes. */ struct mutex_queue mutexq; /* Queue of all owned PRIO_PROTECT mutexes. */ struct mutex_queue pp_mutexq; void *ret; struct pthread_specific_elem *specific; int specific_data_count; /* Number rwlocks rdlocks held. */ int rdlock_count; /* * Current locks bitmap for rtld. */ int rtld_bits; /* Thread control block */ struct tcb *tcb; /* Cleanup handlers Link List */ struct pthread_cleanup *cleanup; #ifdef _PTHREAD_FORCED_UNWIND struct _Unwind_Exception ex; void *unwind_stackend; int unwind_disabled; #endif /* * Magic value to help recognize a valid thread structure * from an invalid one: */ #define THR_MAGIC ((u_int32_t) 0xd09ba115) u_int32_t magic; /* Enable event reporting */ int report_events; /* Event mask */ int event_mask; /* Event */ td_event_msg_t event_buf; struct wake_addr *wake_addr; #define WAKE_ADDR(td) ((td)->wake_addr) /* Sleep queue */ struct sleepqueue *sleepqueue; /* Wait channel */ void *wchan; /* Referenced mutex. */ struct pthread_mutex *mutex_obj; /* Thread will sleep. */ int will_sleep; /* Number of threads deferred. */ int nwaiter_defer; /* Deferred threads from pthread_cond_signal. */ unsigned int *defer_waiters[MAX_DEFER_WAITERS]; }; #define THR_SHOULD_GC(thrd) \ ((thrd)->refcount == 0 && (thrd)->state == PS_DEAD && \ ((thrd)->flags & THR_FLAGS_DETACHED) != 0) #define THR_IN_CRITICAL(thrd) \ (((thrd)->locklevel > 0) || \ ((thrd)->critical_count > 0)) #define THR_CRITICAL_ENTER(thrd) \ (thrd)->critical_count++ #define THR_CRITICAL_LEAVE(thrd) \ do { \ (thrd)->critical_count--; \ _thr_ast(thrd); \ } while (0) #define THR_UMUTEX_TRYLOCK(thrd, lck) \ _thr_umutex_trylock((lck), TID(thrd)) #define THR_UMUTEX_LOCK(thrd, lck) \ _thr_umutex_lock((lck), TID(thrd)) #define THR_UMUTEX_TIMEDLOCK(thrd, lck, timo) \ _thr_umutex_timedlock((lck), TID(thrd), (timo)) #define THR_UMUTEX_UNLOCK(thrd, lck) \ _thr_umutex_unlock((lck), TID(thrd)) #define THR_LOCK_ACQUIRE(thrd, lck) \ do { \ (thrd)->locklevel++; \ _thr_umutex_lock(lck, TID(thrd)); \ } while (0) #define THR_LOCK_ACQUIRE_SPIN(thrd, lck) \ do { \ (thrd)->locklevel++; \ _thr_umutex_lock_spin(lck, TID(thrd)); \ } while (0) #ifdef _PTHREADS_INVARIANTS #define THR_ASSERT_LOCKLEVEL(thrd) \ do { \ if (__predict_false((thrd)->locklevel <= 0)) \ _thr_assert_lock_level(); \ } while (0) #else #define THR_ASSERT_LOCKLEVEL(thrd) #endif #define THR_LOCK_RELEASE(thrd, lck) \ do { \ THR_ASSERT_LOCKLEVEL(thrd); \ _thr_umutex_unlock((lck), TID(thrd)); \ (thrd)->locklevel--; \ _thr_ast(thrd); \ } while (0) #define THR_LOCK(curthrd) THR_LOCK_ACQUIRE(curthrd, &(curthrd)->lock) #define THR_UNLOCK(curthrd) THR_LOCK_RELEASE(curthrd, &(curthrd)->lock) #define THR_THREAD_LOCK(curthrd, thr) THR_LOCK_ACQUIRE(curthrd, &(thr)->lock) #define THR_THREAD_UNLOCK(curthrd, thr) THR_LOCK_RELEASE(curthrd, &(thr)->lock) #define THREAD_LIST_RDLOCK(curthrd) \ do { \ (curthrd)->locklevel++; \ _thr_rwl_rdlock(&_thr_list_lock); \ } while (0) #define THREAD_LIST_WRLOCK(curthrd) \ do { \ (curthrd)->locklevel++; \ _thr_rwl_wrlock(&_thr_list_lock); \ } while (0) #define THREAD_LIST_UNLOCK(curthrd) \ do { \ _thr_rwl_unlock(&_thr_list_lock); \ (curthrd)->locklevel--; \ _thr_ast(curthrd); \ } while (0) /* * Macros to insert/remove threads to the all thread list and * the gc list. */ #define THR_LIST_ADD(thrd) do { \ if (((thrd)->tlflags & TLFLAGS_IN_TDLIST) == 0) { \ TAILQ_INSERT_HEAD(&_thread_list, thrd, tle); \ _thr_hash_add(thrd); \ (thrd)->tlflags |= TLFLAGS_IN_TDLIST; \ } \ } while (0) #define THR_LIST_REMOVE(thrd) do { \ if (((thrd)->tlflags & TLFLAGS_IN_TDLIST) != 0) { \ TAILQ_REMOVE(&_thread_list, thrd, tle); \ _thr_hash_remove(thrd); \ (thrd)->tlflags &= ~TLFLAGS_IN_TDLIST; \ } \ } while (0) #define THR_GCLIST_ADD(thrd) do { \ if (((thrd)->tlflags & TLFLAGS_IN_GCLIST) == 0) { \ TAILQ_INSERT_HEAD(&_thread_gc_list, thrd, gcle);\ (thrd)->tlflags |= TLFLAGS_IN_GCLIST; \ _gc_count++; \ } \ } while (0) #define THR_GCLIST_REMOVE(thrd) do { \ if (((thrd)->tlflags & TLFLAGS_IN_GCLIST) != 0) { \ TAILQ_REMOVE(&_thread_gc_list, thrd, gcle); \ (thrd)->tlflags &= ~TLFLAGS_IN_GCLIST; \ _gc_count--; \ } \ } while (0) #define THR_REF_ADD(curthread, pthread) { \ THR_CRITICAL_ENTER(curthread); \ pthread->refcount++; \ } while (0) #define THR_REF_DEL(curthread, pthread) { \ pthread->refcount--; \ THR_CRITICAL_LEAVE(curthread); \ } while (0) #define GC_NEEDED() (_gc_count >= 5) #define SHOULD_REPORT_EVENT(curthr, e) \ (curthr->report_events && \ (((curthr)->event_mask | _thread_event_mask ) & e) != 0) extern int __isthreaded; /* * Global variables for the pthread kernel. */ extern char *_usrstack __hidden; extern struct pthread *_thr_initial __hidden; /* For debugger */ extern int _libthr_debug; extern int _thread_event_mask; extern struct pthread *_thread_last_event; /* List of all threads: */ extern pthreadlist _thread_list; /* List of threads needing GC: */ extern pthreadlist _thread_gc_list __hidden; extern int _thread_active_threads; extern atfork_head _thr_atfork_list __hidden; extern struct urwlock _thr_atfork_lock __hidden; /* Default thread attributes: */ extern struct pthread_attr _pthread_attr_default __hidden; /* Default mutex attributes: */ extern struct pthread_mutex_attr _pthread_mutexattr_default __hidden; extern struct pthread_mutex_attr _pthread_mutexattr_adaptive_default __hidden; /* Default condition variable attributes: */ extern struct pthread_cond_attr _pthread_condattr_default __hidden; extern struct pthread_prio _thr_priorities[] __hidden; extern pid_t _thr_pid __hidden; extern int _thr_is_smp __hidden; extern size_t _thr_guard_default __hidden; extern size_t _thr_stack_default __hidden; extern size_t _thr_stack_initial __hidden; extern int _thr_page_size __hidden; extern int _thr_spinloops __hidden; extern int _thr_yieldloops __hidden; /* Garbage thread count. */ extern int _gc_count __hidden; extern struct umutex _mutex_static_lock __hidden; extern struct umutex _cond_static_lock __hidden; extern struct umutex _rwlock_static_lock __hidden; extern struct umutex _keytable_lock __hidden; extern struct urwlock _thr_list_lock __hidden; extern struct umutex _thr_event_lock __hidden; /* * Function prototype definitions. */ __BEGIN_DECLS int _thr_setthreaded(int) __hidden; int _mutex_cv_lock(struct pthread_mutex *, int count) __hidden; int _mutex_cv_unlock(struct pthread_mutex *, int *count) __hidden; int _mutex_cv_attach(struct pthread_mutex *, int count) __hidden; int _mutex_cv_detach(struct pthread_mutex *, int *count) __hidden; int _mutex_owned(struct pthread *, const struct pthread_mutex *) __hidden; int _mutex_reinit(pthread_mutex_t *) __hidden; void _mutex_fork(struct pthread *curthread) __hidden; void _libpthread_init(struct pthread *) __hidden; struct pthread *_thr_alloc(struct pthread *) __hidden; void _thread_exit(const char *, int, const char *) __hidden __dead2; int _thr_ref_add(struct pthread *, struct pthread *, int) __hidden; void _thr_ref_delete(struct pthread *, struct pthread *) __hidden; void _thr_ref_delete_unlocked(struct pthread *, struct pthread *) __hidden; int _thr_find_thread(struct pthread *, struct pthread *, int) __hidden; void _thr_rtld_init(void) __hidden; void _thr_rtld_fini(void) __hidden; void _thr_rtld_postfork_child(void) __hidden; int _thr_stack_alloc(struct pthread_attr *) __hidden; void _thr_stack_free(struct pthread_attr *) __hidden; void _thr_free(struct pthread *, struct pthread *) __hidden; void _thr_gc(struct pthread *) __hidden; void _thread_cleanupspecific(void) __hidden; void _thread_printf(int, const char *, ...) __hidden; void _thr_spinlock_init(void) __hidden; void _thr_cancel_enter(struct pthread *) __hidden; void _thr_cancel_enter2(struct pthread *, int) __hidden; void _thr_cancel_leave(struct pthread *, int) __hidden; void _thr_testcancel(struct pthread *) __hidden; void _thr_signal_block(struct pthread *) __hidden; void _thr_signal_unblock(struct pthread *) __hidden; void _thr_signal_init(void) __hidden; void _thr_signal_deinit(void) __hidden; int _thr_send_sig(struct pthread *, int sig) __hidden; void _thr_list_init(void) __hidden; void _thr_hash_add(struct pthread *) __hidden; void _thr_hash_remove(struct pthread *) __hidden; struct pthread *_thr_hash_find(struct pthread *) __hidden; void _thr_link(struct pthread *, struct pthread *) __hidden; void _thr_unlink(struct pthread *, struct pthread *) __hidden; void _thr_assert_lock_level(void) __hidden __dead2; void _thr_ast(struct pthread *) __hidden; void _thr_once_init(void) __hidden; void _thr_report_creation(struct pthread *curthread, struct pthread *newthread) __hidden; void _thr_report_death(struct pthread *curthread) __hidden; int _thr_getscheduler(lwpid_t, int *, struct sched_param *) __hidden; int _thr_setscheduler(lwpid_t, int, const struct sched_param *) __hidden; void _thr_signal_prefork(void) __hidden; void _thr_signal_postfork(void) __hidden; void _thr_signal_postfork_child(void) __hidden; void _thr_try_gc(struct pthread *, struct pthread *) __hidden; int _rtp_to_schedparam(const struct rtprio *rtp, int *policy, struct sched_param *param) __hidden; int _schedparam_to_rtp(int policy, const struct sched_param *param, struct rtprio *rtp) __hidden; void _thread_bp_create(void); void _thread_bp_death(void); int _sched_yield(void); void _pthread_cleanup_push(void (*)(void *), void *); void _pthread_cleanup_pop(int); void _pthread_exit_mask(void *status, sigset_t *mask) __dead2 __hidden; void _pthread_cancel_enter(int maycancel); void _pthread_cancel_leave(int maycancel); /* #include */ #ifdef _SYS_FCNTL_H_ int __sys_fcntl(int, int, ...); int __sys_open(const char *, int, ...); int __sys_openat(int, const char *, int, ...); #endif /* #include */ #ifdef _SIGNAL_H_ int __sys_kill(pid_t, int); int __sys_sigaction(int, const struct sigaction *, struct sigaction *); int __sys_sigpending(sigset_t *); int __sys_sigprocmask(int, const sigset_t *, sigset_t *); int __sys_sigsuspend(const sigset_t *); int __sys_sigreturn(const ucontext_t *); int __sys_sigaltstack(const struct sigaltstack *, struct sigaltstack *); int __sys_sigwait(const sigset_t *, int *); int __sys_sigtimedwait(const sigset_t *, siginfo_t *, const struct timespec *); int __sys_sigwaitinfo(const sigset_t *set, siginfo_t *info); #endif /* #include */ #ifdef _TIME_H_ int __sys_nanosleep(const struct timespec *, struct timespec *); #endif /* #include */ #ifdef _SYS_UCONTEXT_H_ int __sys_setcontext(const ucontext_t *ucp); int __sys_swapcontext(ucontext_t *oucp, const ucontext_t *ucp); #endif /* #include */ #ifdef _UNISTD_H_ int __sys_close(int); int __sys_fork(void); pid_t __sys_getpid(void); ssize_t __sys_read(int, void *, size_t); ssize_t __sys_write(int, const void *, size_t); void __sys_exit(int); #endif int _umtx_op_err(void *, int op, u_long, void *, void *) __hidden; static inline int _thr_isthreaded(void) { return (__isthreaded != 0); } static inline int _thr_is_inited(void) { return (_thr_initial != NULL); } static inline void _thr_check_init(void) { if (_thr_initial == NULL) _libpthread_init(NULL); } struct wake_addr *_thr_alloc_wake_addr(void); void _thr_release_wake_addr(struct wake_addr *); int _thr_sleep(struct pthread *, int, const struct timespec *); void _thr_wake_addr_init(void) __hidden; static inline void _thr_clear_wake(struct pthread *td) { td->wake_addr->value = 0; } static inline int _thr_is_woken(struct pthread *td) { return td->wake_addr->value != 0; } static inline void _thr_set_wake(unsigned int *waddr) { *waddr = 1; _thr_umtx_wake(waddr, INT_MAX, 0); } void _thr_wake_all(unsigned int *waddrs[], int) __hidden; static inline struct pthread * _sleepq_first(struct sleepqueue *sq) { return TAILQ_FIRST(&sq->sq_blocked); } void _sleepq_init(void) __hidden; struct sleepqueue *_sleepq_alloc(void) __hidden; void _sleepq_free(struct sleepqueue *) __hidden; void _sleepq_lock(void *) __hidden; void _sleepq_unlock(void *) __hidden; struct sleepqueue *_sleepq_lookup(void *) __hidden; void _sleepq_add(void *, struct pthread *) __hidden; int _sleepq_remove(struct sleepqueue *, struct pthread *) __hidden; void _sleepq_drop(struct sleepqueue *, void (*cb)(struct pthread *, void *arg), void *) __hidden; struct dl_phdr_info; void __pthread_cxa_finalize(struct dl_phdr_info *phdr_info); void _thr_tsd_unload(struct dl_phdr_info *phdr_info) __hidden; void _thr_sigact_unload(struct dl_phdr_info *phdr_info) __hidden; +void _thr_stack_fix_protection(struct pthread *thrd); __END_DECLS #endif /* !_THR_PRIVATE_H */ diff --git a/lib/libthr/thread/thr_rtld.c b/lib/libthr/thread/thr_rtld.c index e6af7029c76d..d9dd94d52f13 100644 --- a/lib/libthr/thread/thr_rtld.c +++ b/lib/libthr/thread/thr_rtld.c @@ -1,221 +1,226 @@ /* * Copyright (c) 2006, David Xu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ * */ /* * A lockless rwlock for rtld. */ #include +#include +#include #include #include #include "rtld_lock.h" #include "thr_private.h" #undef errno extern int errno; static int _thr_rtld_clr_flag(int); static void *_thr_rtld_lock_create(void); static void _thr_rtld_lock_destroy(void *); static void _thr_rtld_lock_release(void *); static void _thr_rtld_rlock_acquire(void *); static int _thr_rtld_set_flag(int); static void _thr_rtld_wlock_acquire(void *); struct rtld_lock { struct urwlock lock; char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)]; }; static struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE); static int busy_places; static void * _thr_rtld_lock_create(void) { int locki; struct rtld_lock *l; static const char fail[] = "_thr_rtld_lock_create failed\n"; for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) { if ((busy_places & (1 << locki)) == 0) break; } if (locki == MAX_RTLD_LOCKS) { write(2, fail, sizeof(fail) - 1); return (NULL); } busy_places |= (1 << locki); l = &lock_place[locki]; l->lock.rw_flags = URWLOCK_PREFER_READER; return (l); } static void _thr_rtld_lock_destroy(void *lock) { int locki; size_t i; locki = (struct rtld_lock *)lock - &lock_place[0]; for (i = 0; i < sizeof(struct rtld_lock); ++i) ((char *)lock)[i] = 0; busy_places &= ~(1 << locki); } #define SAVE_ERRNO() { \ if (curthread != _thr_initial) \ errsave = curthread->error; \ else \ errsave = errno; \ } #define RESTORE_ERRNO() { \ if (curthread != _thr_initial) \ curthread->error = errsave; \ else \ errno = errsave; \ } static void _thr_rtld_rlock_acquire(void *lock) { struct pthread *curthread; struct rtld_lock *l; int errsave; curthread = _get_curthread(); SAVE_ERRNO(); l = (struct rtld_lock *)lock; THR_CRITICAL_ENTER(curthread); while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0) ; curthread->rdlock_count++; RESTORE_ERRNO(); } static void _thr_rtld_wlock_acquire(void *lock) { struct pthread *curthread; struct rtld_lock *l; int errsave; curthread = _get_curthread(); SAVE_ERRNO(); l = (struct rtld_lock *)lock; THR_CRITICAL_ENTER(curthread); while (_thr_rwlock_wrlock(&l->lock, NULL) != 0) ; RESTORE_ERRNO(); } static void _thr_rtld_lock_release(void *lock) { struct pthread *curthread; struct rtld_lock *l; int32_t state; int errsave; curthread = _get_curthread(); SAVE_ERRNO(); l = (struct rtld_lock *)lock; state = l->lock.rw_state; if (_thr_rwlock_unlock(&l->lock) == 0) { if ((state & URWLOCK_WRITE_OWNER) == 0) curthread->rdlock_count--; THR_CRITICAL_LEAVE(curthread); } RESTORE_ERRNO(); } static int _thr_rtld_set_flag(int mask __unused) { /* * The caller's code in rtld-elf is broken, it is not signal safe, * just return zero to fool it. */ return (0); } static int _thr_rtld_clr_flag(int mask __unused) { return (0); } void _thr_rtld_init(void) { struct RtldLockInfo li; struct pthread *curthread; long dummy = -1; curthread = _get_curthread(); /* force to resolve _umtx_op PLT */ _umtx_op_err((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0); /* force to resolve errno() PLT */ __error(); /* force to resolve memcpy PLT */ memcpy(&dummy, &dummy, sizeof(dummy)); + mprotect(NULL, 0, 0); + _rtld_get_stack_prot(); + li.lock_create = _thr_rtld_lock_create; li.lock_destroy = _thr_rtld_lock_destroy; li.rlock_acquire = _thr_rtld_rlock_acquire; li.wlock_acquire = _thr_rtld_wlock_acquire; li.lock_release = _thr_rtld_lock_release; li.thread_set_flag = _thr_rtld_set_flag; li.thread_clr_flag = _thr_rtld_clr_flag; li.at_fork = NULL; /* mask signals, also force to resolve __sys_sigprocmask PLT */ _thr_signal_block(curthread); _rtld_thread_init(&li); _thr_signal_unblock(curthread); } void _thr_rtld_fini(void) { struct pthread *curthread; curthread = _get_curthread(); _thr_signal_block(curthread); _rtld_thread_init(NULL); _thr_signal_unblock(curthread); } diff --git a/lib/libthr/thread/thr_stack.c b/lib/libthr/thread/thr_stack.c index 76849530159a..cb390cc06d87 100644 --- a/lib/libthr/thread/thr_stack.c +++ b/lib/libthr/thread/thr_stack.c @@ -1,256 +1,289 @@ /* * Copyright (c) 2001 Daniel Eischen * Copyright (c) 2000-2001 Jason Evans * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include +#include #include "thr_private.h" /* Spare thread stack. */ struct stack { LIST_ENTRY(stack) qe; /* Stack queue linkage. */ size_t stacksize; /* Stack size (rounded up). */ size_t guardsize; /* Guard size. */ void *stackaddr; /* Stack address. */ }; /* * Default sized (stack and guard) spare stack queue. Stacks are cached * to avoid additional complexity managing mmap()ed stack regions. Spare * stacks are used in LIFO order to increase cache locality. */ static LIST_HEAD(, stack) dstackq = LIST_HEAD_INITIALIZER(dstackq); /* * Miscellaneous sized (non-default stack and/or guard) spare stack queue. * Stacks are cached to avoid additional complexity managing mmap()ed * stack regions. This list is unordered, since ordering on both stack * size and guard size would be more trouble than it's worth. Stacks are * allocated from this cache on a first size match basis. */ static LIST_HEAD(, stack) mstackq = LIST_HEAD_INITIALIZER(mstackq); /** * Base address of the last stack allocated (including its red zone, if * there is one). Stacks are allocated contiguously, starting beyond the * top of the main stack. When a new stack is created, a red zone is * typically created (actually, the red zone is mapped with PROT_NONE) above * the top of the stack, such that the stack will not be able to grow all * the way to the bottom of the next stack. This isn't fool-proof. It is * possible for a stack to grow by a large amount, such that it grows into * the next stack, and as long as the memory within the red zone is never * accessed, nothing will prevent one thread stack from trouncing all over * the next. * * low memory * . . . . . . . . . . . . . . . . . . * | | * | stack 3 | start of 3rd thread stack * +-----------------------------------+ * | | * | Red Zone (guard page) | red zone for 2nd thread * | | * +-----------------------------------+ * | stack 2 - _thr_stack_default | top of 2nd thread stack * | | * | | * | | * | | * | stack 2 | * +-----------------------------------+ <-- start of 2nd thread stack * | | * | Red Zone | red zone for 1st thread * | | * +-----------------------------------+ * | stack 1 - _thr_stack_default | top of 1st thread stack * | | * | | * | | * | | * | stack 1 | * +-----------------------------------+ <-- start of 1st thread stack * | | (initial value of last_stack) * | Red Zone | * | | red zone for main thread * +-----------------------------------+ * | USRSTACK - _thr_stack_initial | top of main thread stack * | | ^ * | | | * | | | * | | | stack growth * | | * +-----------------------------------+ <-- start of main thread stack * (USRSTACK) * high memory * */ static char *last_stack = NULL; /* * Round size up to the nearest multiple of * _thr_page_size. */ static inline size_t round_up(size_t size) { if (size % _thr_page_size != 0) size = ((size / _thr_page_size) + 1) * _thr_page_size; return size; } +void +_thr_stack_fix_protection(struct pthread *thrd) +{ + + mprotect((char *)thrd->attr.stackaddr_attr + + round_up(thrd->attr.guardsize_attr), + round_up(thrd->attr.stacksize_attr), + _rtld_get_stack_prot()); +} + +void __pthread_map_stacks_exec(void); +void +__pthread_map_stacks_exec(void) +{ + struct pthread *curthread, *thrd; + struct stack *st; + + curthread = _get_curthread(); + THREAD_LIST_RDLOCK(curthread); + LIST_FOREACH(st, &mstackq, qe) + mprotect((char *)st->stackaddr + st->guardsize, st->stacksize, + _rtld_get_stack_prot()); + LIST_FOREACH(st, &dstackq, qe) + mprotect((char *)st->stackaddr + st->guardsize, st->stacksize, + _rtld_get_stack_prot()); + TAILQ_FOREACH(thrd, &_thread_gc_list, gcle) + _thr_stack_fix_protection(thrd); + TAILQ_FOREACH(thrd, &_thread_list, tle) + _thr_stack_fix_protection(thrd); + THREAD_LIST_UNLOCK(curthread); +} + int _thr_stack_alloc(struct pthread_attr *attr) { struct pthread *curthread = _get_curthread(); struct stack *spare_stack; size_t stacksize; size_t guardsize; char *stackaddr; /* * Round up stack size to nearest multiple of _thr_page_size so * that mmap() * will work. If the stack size is not an even * multiple, we end up initializing things such that there is * unused space above the beginning of the stack, so the stack * sits snugly against its guard. */ stacksize = round_up(attr->stacksize_attr); guardsize = round_up(attr->guardsize_attr); attr->stackaddr_attr = NULL; attr->flags &= ~THR_STACK_USER; /* * Use the garbage collector lock for synchronization of the * spare stack lists and allocations from usrstack. */ THREAD_LIST_WRLOCK(curthread); /* * If the stack and guard sizes are default, try to allocate a stack * from the default-size stack cache: */ if ((stacksize == THR_STACK_DEFAULT) && (guardsize == _thr_guard_default)) { if ((spare_stack = LIST_FIRST(&dstackq)) != NULL) { /* Use the spare stack. */ LIST_REMOVE(spare_stack, qe); attr->stackaddr_attr = spare_stack->stackaddr; } } /* * The user specified a non-default stack and/or guard size, so try to * allocate a stack from the non-default size stack cache, using the * rounded up stack size (stack_size) in the search: */ else { LIST_FOREACH(spare_stack, &mstackq, qe) { if (spare_stack->stacksize == stacksize && spare_stack->guardsize == guardsize) { LIST_REMOVE(spare_stack, qe); attr->stackaddr_attr = spare_stack->stackaddr; break; } } } if (attr->stackaddr_attr != NULL) { /* A cached stack was found. Release the lock. */ THREAD_LIST_UNLOCK(curthread); } else { /* Allocate a stack from usrstack. */ if (last_stack == NULL) last_stack = _usrstack - _thr_stack_initial - _thr_guard_default; /* Allocate a new stack. */ stackaddr = last_stack - stacksize - guardsize; /* * Even if stack allocation fails, we don't want to try to * use this location again, so unconditionally decrement * last_stack. Under normal operating conditions, the most * likely reason for an mmap() error is a stack overflow of * the adjacent thread stack. */ last_stack -= (stacksize + guardsize); /* Release the lock before mmap'ing it. */ THREAD_LIST_UNLOCK(curthread); /* Map the stack and guard page together, and split guard page from allocated space: */ if ((stackaddr = mmap(stackaddr, stacksize+guardsize, - PROT_READ | PROT_WRITE, MAP_STACK, + _rtld_get_stack_prot(), MAP_STACK, -1, 0)) != MAP_FAILED && (guardsize == 0 || mprotect(stackaddr, guardsize, PROT_NONE) == 0)) { stackaddr += guardsize; } else { if (stackaddr != MAP_FAILED) munmap(stackaddr, stacksize + guardsize); stackaddr = NULL; } attr->stackaddr_attr = stackaddr; } if (attr->stackaddr_attr != NULL) return (0); else return (-1); } /* This function must be called with _thread_list_lock held. */ void _thr_stack_free(struct pthread_attr *attr) { struct stack *spare_stack; if ((attr != NULL) && ((attr->flags & THR_STACK_USER) == 0) && (attr->stackaddr_attr != NULL)) { spare_stack = (struct stack *) ((char *)attr->stackaddr_attr + attr->stacksize_attr - sizeof(struct stack)); spare_stack->stacksize = round_up(attr->stacksize_attr); spare_stack->guardsize = round_up(attr->guardsize_attr); spare_stack->stackaddr = attr->stackaddr_attr; if (spare_stack->stacksize == THR_STACK_DEFAULT && spare_stack->guardsize == _thr_guard_default) { /* Default stack/guard size. */ LIST_INSERT_HEAD(&dstackq, spare_stack, qe); } else { /* Non-default stack/guard size. */ LIST_INSERT_HEAD(&mstackq, spare_stack, qe); } attr->stackaddr_attr = NULL; } }