Index: lib/libthr/Makefile =================================================================== --- lib/libthr/Makefile +++ lib/libthr/Makefile @@ -15,7 +15,7 @@ LIB=thr SHLIB_MAJOR= 3 NO_WTHREAD_SAFETY=1 -NO_WCAST_ALIGN.gcc=1 # for gcc 4.2 +NO_WCAST_ALIGN=1 # for gcc 4.2 and for rtld malloc.c CFLAGS+=-DPTHREAD_KERNEL CFLAGS+=-I${SRCTOP}/lib/libc/include -I${.CURDIR}/thread \ -I${SRCTOP}/include @@ -50,12 +50,14 @@ PRECIOUSLIB= .PATH: ${.CURDIR}/arch/${MACHINE_CPUARCH}/${MACHINE_CPUARCH} +.PATH: ${SRCTOP}/libexec/rtld-elf .if exists(${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc) .include "${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc" .endif .include "${.CURDIR}/sys/Makefile.inc" .include "${.CURDIR}/thread/Makefile.inc" +SRCS+= malloc.c .if ${MK_INSTALLLIB} != "no" SYMLINKS+=lib${LIB}.a ${LIBDIR}/libpthread.a Index: lib/libthr/thread/Makefile.inc =================================================================== --- lib/libthr/thread/Makefile.inc +++ lib/libthr/thread/Makefile.inc @@ -31,6 +31,7 @@ thr_kern.c \ thr_kill.c \ thr_main_np.c \ + thr_malloc.c \ thr_multi_np.c \ thr_mutex.c \ thr_mutexattr.c \ Index: lib/libthr/thread/thr_fork.c =================================================================== --- lib/libthr/thread/thr_fork.c +++ lib/libthr/thread/thr_fork.c @@ -170,6 +170,7 @@ */ if (_thr_isthreaded() != 0) { was_threaded = 1; + __thr_malloc_prefork(curthread); _malloc_prefork(); __thr_pshared_atfork_pre(); _rtld_atfork_pre(rtld_locks); @@ -197,6 +198,10 @@ */ curthread->tlflags &= ~TLFLAGS_IN_TDLIST; + /* before thr_self() */ + if (was_threaded) + __thr_malloc_postfork(curthread); + /* child is a new kernel thread. */ thr_self(&curthread->tid); @@ -241,6 +246,7 @@ _thr_signal_postfork(); if (was_threaded) { + __thr_malloc_postfork(curthread); _rtld_atfork_post(rtld_locks); __thr_pshared_atfork_post(); _malloc_postfork(); Index: lib/libthr/thread/thr_init.c =================================================================== --- lib/libthr/thread/thr_init.c +++ lib/libthr/thread/thr_init.c @@ -461,6 +461,7 @@ */ if (init_once == 0) { __thr_pshared_init(); + __thr_malloc_init(); /* Find the stack top */ mib[0] = CTL_KERN; mib[1] = KERN_USRSTACK; Index: lib/libthr/thread/thr_malloc.c =================================================================== --- /dev/null +++ lib/libthr/thread/thr_malloc.c @@ -0,0 +1,137 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include "thr_private.h" + +int npagesizes; +size_t *pagesizes; +static size_t pagesizes_d[2]; +static struct umutex thr_malloc_umtx; + +void +__thr_malloc_init(void) +{ + + npagesizes = getpagesizes(pagesizes_d, nitems(pagesizes_d)); + if (npagesizes == -1) { + npagesizes = 1; + pagesizes_d[0] = PAGE_SIZE; + } + pagesizes = pagesizes_d; + _thr_umutex_init(&thr_malloc_umtx); +} + +static void +thr_malloc_lock(struct pthread *curthread) +{ + + curthread->locklevel++; + _thr_umutex_lock(&thr_malloc_umtx, TID(curthread)); +} + +static void +thr_malloc_unlock(struct pthread *curthread) +{ + + _thr_umutex_unlock(&thr_malloc_umtx, TID(curthread)); + curthread->locklevel--; + _thr_ast(curthread); +} + +void * +__thr_calloc(size_t num, size_t size) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_calloc(num, size); + thr_malloc_unlock(curthread); + return (res); +} + +void +__thr_free(void *cp) +{ + struct pthread *curthread; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + __crt_free(cp); + thr_malloc_unlock(curthread); +} + +void * +__thr_malloc(size_t nbytes) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_malloc(nbytes); + thr_malloc_unlock(curthread); + return (res); +} + +void * +__thr_realloc(void *cp, size_t nbytes) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_realloc(cp, nbytes); + thr_malloc_unlock(curthread); + return (res); +} + +void +__thr_malloc_prefork(struct pthread *curthread) +{ + + _thr_umutex_lock(&thr_malloc_umtx, TID(curthread)); +} + +void +__thr_malloc_postfork(struct pthread *curthread) +{ + + _thr_umutex_unlock(&thr_malloc_umtx, TID(curthread)); +} Index: lib/libthr/thread/thr_mutex.c =================================================================== --- lib/libthr/thread/thr_mutex.c +++ lib/libthr/thread/thr_mutex.c @@ -306,10 +306,11 @@ THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); if (*mutex == THR_MUTEX_INITIALIZER) - ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc); + ret = mutex_init(mutex, &_pthread_mutexattr_default, + __thr_calloc); else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, - calloc); + __thr_calloc); else ret = 0; THR_LOCK_RELEASE(thread, &_mutex_static_lock); @@ -390,7 +391,7 @@ if (mutex_attr == NULL || (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, - calloc)); + __thr_calloc)); } pmtx = __thr_pshared_offpage(__DECONST(void *, mutex), 1); if (pmtx == NULL) @@ -483,7 +484,7 @@ } else { *mutex = THR_MUTEX_DESTROYED; mutex_assert_not_owned(_get_curthread(), m); - free(m); + __thr_free(m); ret = 0; } } Index: lib/libthr/thread/thr_private.h =================================================================== --- lib/libthr/thread/thr_private.h +++ lib/libthr/thread/thr_private.h @@ -1003,6 +1003,14 @@ void __thr_pshared_atfork_pre(void) __hidden; void __thr_pshared_atfork_post(void) __hidden; +void *__thr_calloc(size_t num, size_t size); +void __thr_free(void *cp); +void *__thr_malloc(size_t nbytes); +void *__thr_realloc(void *cp, size_t nbytes); +void __thr_malloc_init(void); +void __thr_malloc_prefork(struct pthread *curthread); +void __thr_malloc_postfork(struct pthread *curthread); + __END_DECLS __NULLABILITY_PRAGMA_POP Index: lib/libthr/thread/thr_spec.c =================================================================== --- lib/libthr/thread/thr_spec.c +++ lib/libthr/thread/thr_spec.c @@ -155,8 +155,7 @@ } } THR_LOCK_RELEASE(curthread, &_keytable_lock); - munmap(curthread->specific, PTHREAD_KEYS_MAX * sizeof(struct - pthread_specific_elem)); + __thr_free(curthread->specific); curthread->specific = NULL; if (curthread->specific_data_count > 0) { stderr_debug("Thread %p has exited with leftover " @@ -179,10 +178,9 @@ pthread = _get_curthread(); if (pthread->specific == NULL) { - tmp = mmap(NULL, PTHREAD_KEYS_MAX * - sizeof(struct pthread_specific_elem), - PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (tmp == MAP_FAILED) + tmp = __thr_calloc(PTHREAD_KEYS_MAX, + sizeof(struct pthread_specific_elem)); + if (tmp == NULL) return (ENOMEM); pthread->specific = tmp; } Index: libexec/rtld-elf/malloc.c =================================================================== --- libexec/rtld-elf/malloc.c +++ libexec/rtld-elf/malloc.c @@ -153,7 +153,7 @@ */ void * -malloc(size_t nbytes) +__crt_malloc(size_t nbytes) { union overhead *op; int bucket; @@ -236,7 +236,7 @@ } void * -calloc(size_t num, size_t size) +__crt_calloc(size_t num, size_t size) { void *ret; @@ -245,7 +245,7 @@ return (NULL); } - if ((ret = malloc(num * size)) != NULL) + if ((ret = __crt_malloc(num * size)) != NULL) memset(ret, 0, num * size); return (ret); @@ -298,7 +298,7 @@ } void -free(void * cp) +__crt_free(void *cp) { int size; union overhead *op; @@ -339,7 +339,7 @@ static int realloc_srchlen = 4; /* 4 should be plenty, -1 =>'s whole list */ void * -realloc(void *cp, size_t nbytes) +__crt_realloc(void *cp, size_t nbytes) { u_int onb; int i; @@ -348,7 +348,7 @@ int was_alloced = 0; if (cp == NULL) - return (malloc(nbytes)); + return (__crt_malloc(nbytes)); op = (union overhead *)((caddr_t)cp - sizeof (union overhead)); if (op->ov_magic == MAGIC) { was_alloced++; @@ -393,9 +393,9 @@ #endif return(cp); } else - free(cp); + __crt_free(cp); } - if ((res = malloc(nbytes)) == NULL) + if ((res = __crt_malloc(nbytes)) == NULL) return (NULL); if (cp != res) /* common optimization if "compacting" */ bcopy(cp, res, (nbytes < onb) ? nbytes : onb); @@ -467,9 +467,11 @@ caddr_t addr = (caddr_t) (((long)pagepool_start + pagesz - 1) & ~(pagesz - 1)); if (munmap(addr, pagepool_end - addr) != 0) { +#ifdef IN_RTLD rtld_fdprintf(STDERR_FILENO, _BASENAME_RTLD ": " "morepages: cannot munmap %p: %s\n", addr, rtld_strerror(errno)); +#endif } } @@ -478,9 +480,11 @@ if ((pagepool_start = mmap(0, n * pagesz, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, fd, 0)) == (caddr_t)-1) { +#ifdef IN_RTLD rtld_fdprintf(STDERR_FILENO, _BASENAME_RTLD ": morepages: " "cannot mmap anonymous memory: %s\n", rtld_strerror(errno)); +#endif return 0; } pagepool_end = pagepool_start + n * pagesz; Index: libexec/rtld-elf/rtld.h =================================================================== --- libexec/rtld-elf/rtld.h +++ libexec/rtld-elf/rtld.h @@ -409,4 +409,9 @@ void init_pltgot(Obj_Entry *); void allocate_initial_tls(Obj_Entry *); +void *__crt_calloc(size_t num, size_t size); +void __crt_free(void *cp); +void *__crt_malloc(size_t nbytes); +void *__crt_realloc(void *cp, size_t nbytes); + #endif /* } */ Index: libexec/rtld-elf/rtld.c =================================================================== --- libexec/rtld-elf/rtld.c +++ libexec/rtld-elf/rtld.c @@ -66,6 +66,7 @@ #include "paths.h" #include "rtld_tls.h" #include "rtld_printf.h" +#include "rtld_malloc.h" #include "rtld_utrace.h" #include "notes.h" @@ -5637,3 +5638,32 @@ for (i = 0; i < len; i++) ((char *)dest)[i] = 0; } + +/* malloc */ +void * +malloc(size_t nbytes) +{ + + return (__crt_malloc(nbytes)); +} + +void * +calloc(size_t num, size_t size) +{ + + return (__crt_calloc(num, size)); +} + +void +free(void *cp) +{ + + __crt_free(cp); +} + +void * +realloc(void *cp, size_t nbytes) +{ + + return (__crt_realloc(cp, nbytes)); +} Index: libexec/rtld-elf/rtld_malloc.h =================================================================== --- /dev/null +++ libexec/rtld-elf/rtld_malloc.h @@ -0,0 +1,45 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef RTLD_MALLOC_H +#define RTLD_MALLOC_H + +void *__crt_calloc(size_t num, size_t size); +void __crt_free(void *cp); +void *__crt_malloc(size_t nbytes); +void *__crt_realloc(void *cp, size_t nbytes); + +extern int npagesizes; +extern size_t *pagesizes; + +#endif Index: libexec/rtld-elf/xmalloc.c =================================================================== --- libexec/rtld-elf/xmalloc.c +++ libexec/rtld-elf/xmalloc.c @@ -33,13 +33,14 @@ #include #include "rtld.h" #include "rtld_printf.h" +#include "rtld_malloc.h" void * xcalloc(size_t number, size_t size) { void *p; - p = calloc(number, size); + p = __crt_calloc(number, size); if (p == NULL) { rtld_fdputstr(STDERR_FILENO, "Out of memory\n"); _exit(1); @@ -50,12 +51,15 @@ void * xmalloc(size_t size) { - void *p = malloc(size); - if (p == NULL) { - rtld_fdputstr(STDERR_FILENO, "Out of memory\n"); - _exit(1); - } - return p; + + void *p; + + p = __crt_malloc(size); + if (p == NULL) { + rtld_fdputstr(STDERR_FILENO, "Out of memory\n"); + _exit(1); + } + return (p); } char *