Index: head/lib/libthr/Makefile =================================================================== --- head/lib/libthr/Makefile +++ head/lib/libthr/Makefile @@ -27,6 +27,7 @@ CFLAGS+=-Winline CFLAGS.thr_stack.c+= -Wno-cast-align +CFLAGS.malloc.c+= -Wno-cast-align .include .if !(${COMPILER_TYPE} == "gcc" && ${COMPILER_VERSION} < 40300) CFLAGS.thr_symbols.c+= -Wno-missing-variable-declarations @@ -50,12 +51,14 @@ PRECIOUSLIB= .PATH: ${.CURDIR}/arch/${MACHINE_CPUARCH}/${MACHINE_CPUARCH} +.PATH: ${SRCTOP}/libexec/rtld-elf .if exists(${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc) .include "${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc" .endif .include "${.CURDIR}/sys/Makefile.inc" .include "${.CURDIR}/thread/Makefile.inc" +SRCS+= malloc.c .if ${MK_INSTALLLIB} != "no" SYMLINKS+=lib${LIB}.a ${LIBDIR}/libpthread.a Index: head/lib/libthr/thread/Makefile.inc =================================================================== --- head/lib/libthr/thread/Makefile.inc +++ head/lib/libthr/thread/Makefile.inc @@ -31,6 +31,7 @@ thr_kern.c \ thr_kill.c \ thr_main_np.c \ + thr_malloc.c \ thr_multi_np.c \ thr_mutex.c \ thr_mutexattr.c \ Index: head/lib/libthr/thread/thr_fork.c =================================================================== --- head/lib/libthr/thread/thr_fork.c +++ head/lib/libthr/thread/thr_fork.c @@ -170,6 +170,7 @@ */ if (_thr_isthreaded() != 0) { was_threaded = 1; + __thr_malloc_prefork(curthread); _malloc_prefork(); __thr_pshared_atfork_pre(); _rtld_atfork_pre(rtld_locks); @@ -197,6 +198,10 @@ */ curthread->tlflags &= ~TLFLAGS_IN_TDLIST; + /* before thr_self() */ + if (was_threaded) + __thr_malloc_postfork(curthread); + /* child is a new kernel thread. */ thr_self(&curthread->tid); @@ -241,6 +246,7 @@ _thr_signal_postfork(); if (was_threaded) { + __thr_malloc_postfork(curthread); _rtld_atfork_post(rtld_locks); __thr_pshared_atfork_post(); _malloc_postfork(); Index: head/lib/libthr/thread/thr_init.c =================================================================== --- head/lib/libthr/thread/thr_init.c +++ head/lib/libthr/thread/thr_init.c @@ -461,6 +461,7 @@ */ if (init_once == 0) { __thr_pshared_init(); + __thr_malloc_init(); /* Find the stack top */ mib[0] = CTL_KERN; mib[1] = KERN_USRSTACK; Index: head/lib/libthr/thread/thr_malloc.c =================================================================== --- head/lib/libthr/thread/thr_malloc.c +++ head/lib/libthr/thread/thr_malloc.c @@ -0,0 +1,137 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include "thr_private.h" + +int npagesizes; +size_t *pagesizes; +static size_t pagesizes_d[2]; +static struct umutex thr_malloc_umtx; + +void +__thr_malloc_init(void) +{ + + npagesizes = getpagesizes(pagesizes_d, nitems(pagesizes_d)); + if (npagesizes == -1) { + npagesizes = 1; + pagesizes_d[0] = PAGE_SIZE; + } + pagesizes = pagesizes_d; + _thr_umutex_init(&thr_malloc_umtx); +} + +static void +thr_malloc_lock(struct pthread *curthread) +{ + + curthread->locklevel++; + _thr_umutex_lock(&thr_malloc_umtx, TID(curthread)); +} + +static void +thr_malloc_unlock(struct pthread *curthread) +{ + + _thr_umutex_unlock(&thr_malloc_umtx, TID(curthread)); + curthread->locklevel--; + _thr_ast(curthread); +} + +void * +__thr_calloc(size_t num, size_t size) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_calloc(num, size); + thr_malloc_unlock(curthread); + return (res); +} + +void +__thr_free(void *cp) +{ + struct pthread *curthread; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + __crt_free(cp); + thr_malloc_unlock(curthread); +} + +void * +__thr_malloc(size_t nbytes) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_malloc(nbytes); + thr_malloc_unlock(curthread); + return (res); +} + +void * +__thr_realloc(void *cp, size_t nbytes) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_realloc(cp, nbytes); + thr_malloc_unlock(curthread); + return (res); +} + +void +__thr_malloc_prefork(struct pthread *curthread) +{ + + _thr_umutex_lock(&thr_malloc_umtx, TID(curthread)); +} + +void +__thr_malloc_postfork(struct pthread *curthread) +{ + + _thr_umutex_unlock(&thr_malloc_umtx, TID(curthread)); +} Index: head/lib/libthr/thread/thr_mutex.c =================================================================== --- head/lib/libthr/thread/thr_mutex.c +++ head/lib/libthr/thread/thr_mutex.c @@ -306,10 +306,11 @@ THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); if (*mutex == THR_MUTEX_INITIALIZER) - ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc); + ret = mutex_init(mutex, &_pthread_mutexattr_default, + __thr_calloc); else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, - calloc); + __thr_calloc); else ret = 0; THR_LOCK_RELEASE(thread, &_mutex_static_lock); @@ -390,7 +391,7 @@ if (mutex_attr == NULL || (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, - calloc)); + __thr_calloc)); } pmtx = __thr_pshared_offpage(__DECONST(void *, mutex), 1); if (pmtx == NULL) @@ -483,7 +484,7 @@ } else { *mutex = THR_MUTEX_DESTROYED; mutex_assert_not_owned(_get_curthread(), m); - free(m); + __thr_free(m); ret = 0; } } Index: head/lib/libthr/thread/thr_private.h =================================================================== --- head/lib/libthr/thread/thr_private.h +++ head/lib/libthr/thread/thr_private.h @@ -1003,6 +1003,14 @@ void __thr_pshared_atfork_pre(void) __hidden; void __thr_pshared_atfork_post(void) __hidden; +void *__thr_calloc(size_t num, size_t size); +void __thr_free(void *cp); +void *__thr_malloc(size_t nbytes); +void *__thr_realloc(void *cp, size_t nbytes); +void __thr_malloc_init(void); +void __thr_malloc_prefork(struct pthread *curthread); +void __thr_malloc_postfork(struct pthread *curthread); + __END_DECLS __NULLABILITY_PRAGMA_POP Index: head/lib/libthr/thread/thr_spec.c =================================================================== --- head/lib/libthr/thread/thr_spec.c +++ head/lib/libthr/thread/thr_spec.c @@ -155,8 +155,7 @@ } } THR_LOCK_RELEASE(curthread, &_keytable_lock); - munmap(curthread->specific, PTHREAD_KEYS_MAX * sizeof(struct - pthread_specific_elem)); + __thr_free(curthread->specific); curthread->specific = NULL; if (curthread->specific_data_count > 0) { stderr_debug("Thread %p has exited with leftover " @@ -179,10 +178,9 @@ pthread = _get_curthread(); if (pthread->specific == NULL) { - tmp = mmap(NULL, PTHREAD_KEYS_MAX * - sizeof(struct pthread_specific_elem), - PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (tmp == MAP_FAILED) + tmp = __thr_calloc(PTHREAD_KEYS_MAX, + sizeof(struct pthread_specific_elem)); + if (tmp == NULL) return (ENOMEM); pthread->specific = tmp; }