diff --git a/sys/compat/linuxkpi/common/include/linux/slab.h b/sys/compat/linuxkpi/common/include/linux/slab.h --- a/sys/compat/linuxkpi/common/include/linux/slab.h +++ b/sys/compat/linuxkpi/common/include/linux/slab.h @@ -42,7 +42,6 @@ MALLOC_DECLARE(M_KMALLOC); #define kmalloc(size, flags) lkpi_kmalloc(size, flags) -#define kvmalloc(size, flags) kmalloc(size, flags) #define kvzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) #define kvcalloc(n, size, flags) kvmalloc_array(n, size, (flags) | __GFP_ZERO) #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO) @@ -93,6 +92,7 @@ extern void *lkpi_kmalloc(size_t size, gfp_t flags); void *lkpi___kmalloc(size_t size, gfp_t flags); +void *lkpi___kmalloc_node(size_t size, gfp_t flags, int node); #define __kmalloc(_s, _f) lkpi___kmalloc(_s, _f) static inline gfp_t @@ -113,23 +113,39 @@ static inline void * kmalloc_node(size_t size, gfp_t flags, int node) { - return (malloc_domainset(size, M_KMALLOC, - linux_get_vm_domain_set(node), linux_check_m_flags(flags))); + return (lkpi___kmalloc_node(size, flags, node)); +} + +static inline void * +kmalloc_array(size_t n, size_t size, gfp_t flags) +{ + if (WOULD_OVERFLOW(n, size)) + panic("%s: %zu * %zu overflowed", __func__, n, size); + + return (kmalloc(size * n, flags)); } static inline void * kcalloc(size_t n, size_t size, gfp_t flags) { flags |= __GFP_ZERO; - return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); + return (kmalloc_array(n, size, linux_check_m_flags(flags))); +} + +static inline void * +kmalloc_array_node(size_t n, size_t size, gfp_t flags, int node) +{ + if (WOULD_OVERFLOW(n, size)) + panic("%s: %zu * %zu overflowed", __func__, n, size); + + return (kmalloc_node(size * n, flags, node)); } static inline void * kcalloc_node(size_t n, size_t size, gfp_t flags, int node) { flags |= __GFP_ZERO; - return (mallocarray_domainset(n, size, M_KMALLOC, - linux_get_vm_domain_set(node), linux_check_m_flags(flags))); + return (kmalloc_array_node(n, size, flags, node)); } static inline void * @@ -151,23 +167,20 @@ return (contigmalloc(size, M_KMALLOC, M_WAITOK, 0, UINT_MAX, 1, 1)); } +/* May return non-contiguous memory. */ static inline void * -kmalloc_array(size_t n, size_t size, gfp_t flags) -{ - return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); -} - -static inline void * -kmalloc_array_node(size_t n, size_t size, gfp_t flags, int node) +kvmalloc(size_t size, gfp_t flags) { - return (mallocarray_domainset(n, size, M_KMALLOC, - linux_get_vm_domain_set(node), linux_check_m_flags(flags))); + return (malloc(size, M_KMALLOC, linux_check_m_flags(flags))); } static inline void * kvmalloc_array(size_t n, size_t size, gfp_t flags) { - return (mallocarray(n, size, M_KMALLOC, linux_check_m_flags(flags))); + if (WOULD_OVERFLOW(n, size)) + panic("%s: %zu * %zu overflowed", __func__, n, size); + + return (kvmalloc(size * n, flags)); } static inline void * @@ -179,9 +192,8 @@ static inline void * krealloc_array(void *ptr, size_t n, size_t size, gfp_t flags) { - if (WOULD_OVERFLOW(n, size)) { + if (WOULD_OVERFLOW(n, size)) return NULL; - } return (realloc(ptr, n * size, M_KMALLOC, linux_check_m_flags(flags))); } diff --git a/sys/compat/linuxkpi/common/src/linux_compat.c b/sys/compat/linuxkpi/common/src/linux_compat.c --- a/sys/compat/linuxkpi/common/src/linux_compat.c +++ b/sys/compat/linuxkpi/common/src/linux_compat.c @@ -2767,8 +2767,8 @@ boot_cpu_data.x86_model = CPUID_TO_MODEL(cpu_id); boot_cpu_data.x86_vendor = x86_vendor; - __cpu_data = mallocarray(mp_maxid + 1, - sizeof(*__cpu_data), M_KMALLOC, M_WAITOK | M_ZERO); + __cpu_data = kmalloc_array(mp_maxid + 1, + sizeof(*__cpu_data), M_WAITOK | M_ZERO); CPU_FOREACH(i) { __cpu_data[i].x86_clflush_size = cpu_clflush_line_size; __cpu_data[i].x86_max_cores = mp_ncpus; @@ -2810,8 +2810,8 @@ * This is used by cpumask_of() (and possibly others in the future) for, * e.g., drivers to pass hints to irq_set_affinity_hint(). */ - static_single_cpu_mask = mallocarray(mp_maxid + 1, - sizeof(static_single_cpu_mask), M_KMALLOC, M_WAITOK | M_ZERO); + static_single_cpu_mask = kmalloc_array(mp_maxid + 1, + sizeof(static_single_cpu_mask), M_WAITOK | M_ZERO); /* * When the number of CPUs reach a threshold, we start to save memory @@ -2830,9 +2830,9 @@ * (_BITSET_BITS / 8)' bytes (for comparison with the * overlapping scheme). */ - static_single_cpu_mask_lcs = mallocarray(mp_ncpus, + static_single_cpu_mask_lcs = kmalloc_array(mp_ncpus, sizeof(*static_single_cpu_mask_lcs), - M_KMALLOC, M_WAITOK | M_ZERO); + M_WAITOK | M_ZERO); sscm_ptr = static_single_cpu_mask_lcs; CPU_FOREACH(i) { diff --git a/sys/compat/linuxkpi/common/src/linux_slab.c b/sys/compat/linuxkpi/common/src/linux_slab.c --- a/sys/compat/linuxkpi/common/src/linux_slab.c +++ b/sys/compat/linuxkpi/common/src/linux_slab.c @@ -207,6 +207,18 @@ free(c, M_KMALLOC); } +void * +lkpi___kmalloc_node(size_t size, gfp_t flags, int node) +{ + if (size <= PAGE_SIZE) + return (malloc_domainset(size, M_KMALLOC, + linux_get_vm_domain_set(node), linux_check_m_flags(flags))); + else + return (contigmalloc_domainset(size, M_KMALLOC, + linux_get_vm_domain_set(node), linux_check_m_flags(flags), + 0, -1UL, PAGE_SIZE, 0)); +} + void * lkpi___kmalloc(size_t size, gfp_t flags) {