Index: sys/amd64/amd64/bpf_jit_machdep.c =================================================================== --- sys/amd64/amd64/bpf_jit_machdep.c +++ sys/amd64/amd64/bpf_jit_machdep.c @@ -602,7 +602,7 @@ *size = stream.cur_ip; #ifdef _KERNEL - stream.ibuf = malloc(*size, M_BPFJIT, M_EXEC | M_NOWAIT); + stream.ibuf = malloc_exec(*size, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else Index: sys/i386/i386/bpf_jit_machdep.c =================================================================== --- sys/i386/i386/bpf_jit_machdep.c +++ sys/i386/i386/bpf_jit_machdep.c @@ -632,7 +632,7 @@ *size = stream.cur_ip; #ifdef _KERNEL - stream.ibuf = malloc(*size, M_BPFJIT, M_EXEC | M_NOWAIT); + stream.ibuf = malloc_exec(*size, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else Index: sys/kern/kern_malloc.c =================================================================== --- sys/kern/kern_malloc.c +++ sys/kern/kern_malloc.c @@ -618,13 +618,14 @@ unsigned long osize = size; #endif + MPASS((flags & M_EXEC) == 0); #ifdef MALLOC_DEBUG va = NULL; if (malloc_dbg(&va, &size, mtp, flags) != 0) return (va); #endif - if (size <= kmem_zmax && (flags & M_EXEC) == 0) { + if (size <= kmem_zmax) { if (size & KMEM_ZMASK) size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; indx = kmemsize[size >> KMEM_ZSHIFT]; @@ -640,10 +641,42 @@ va = malloc_large(&size, DOMAINSET_RR(), flags); malloc_type_allocated(mtp, va == NULL ? 0 : size); } - if (flags & M_WAITOK) - KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); - else if (va == NULL) + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); t_malloc_fail = time_uptime; + } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); +#endif + return ((void *) va); +} + +/* + * Allocate and executable area. + */ +void * +malloc_exec(size_t size, struct malloc_type *mtp, int flags) +{ + caddr_t va; +#if defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + + flags |= M_EXEC; +#ifdef MALLOC_DEBUG + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + va = malloc_large(&size, DOMAINSET_RR(), flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); + t_malloc_fail = time_uptime; + } #ifdef DEBUG_REDZONE if (va != NULL) va = redzone_setup(va, osize); @@ -682,7 +715,7 @@ int flags) { struct vm_domainset_iter di; - caddr_t ret; + caddr_t va; int domain; int indx; @@ -690,32 +723,32 @@ unsigned long osize = size; #endif #ifdef MALLOC_DEBUG - ret= NULL; - if (malloc_dbg(&ret, &size, mtp, flags) != 0) - return (ret); + va = NULL; + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); #endif if (size <= kmem_zmax && (flags & M_EXEC) == 0) { vm_domainset_iter_policy_init(&di, ds, &domain, &flags); do { - ret = malloc_domain(&size, &indx, mtp, domain, flags); - } while (ret == NULL && + va = malloc_domain(&size, &indx, mtp, domain, flags); + } while (va == NULL && vm_domainset_iter_policy(&di, &domain) == 0); - malloc_type_zone_allocated(mtp, ret == NULL ? 0 : size, indx); + malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); } else { /* Policy is handled by kmem. */ - ret = malloc_large(&size, ds, flags); - malloc_type_allocated(mtp, ret == NULL ? 0 : size); + va = malloc_large(&size, ds, flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); } - - if (flags & M_WAITOK) - KASSERT(ret != NULL, ("malloc(M_WAITOK) returned NULL")); - else if (ret == NULL) + if (__predict_false(va == NULL)) { + KASSERT((flags & M_WAITOK) == 0, + ("malloc(M_WAITOK) returned NULL")); t_malloc_fail = time_uptime; + } #ifdef DEBUG_REDZONE - if (ret != NULL) - ret = redzone_setup(ret, osize); + if (va != NULL) + va = redzone_setup(va, osize); #endif - return (ret); + return (va); } void * Index: sys/kern/link_elf.c =================================================================== --- sys/kern/link_elf.c +++ sys/kern/link_elf.c @@ -1129,7 +1129,7 @@ goto out; } #else - mapbase = malloc(mapsize, M_LINKER, M_EXEC | M_WAITOK); + mapbase = malloc_exec(mapsize, M_LINKER, M_WAITOK); #endif ef->address = mapbase; Index: sys/sys/malloc.h =================================================================== --- sys/sys/malloc.h +++ sys/sys/malloc.h @@ -183,6 +183,8 @@ void zfree(void *addr, struct malloc_type *type); void *malloc(size_t size, struct malloc_type *type, int flags) __malloc_like __result_use_check __alloc_size(1); +void *malloc_exec(size_t size, struct malloc_type *type, int flags) __malloc_like + __result_use_check __alloc_size(1); /* * Try to optimize malloc(..., ..., M_ZERO) allocations by doing zeroing in * place if the size is known at compilation time.