diff --git a/share/man/man9/contigmalloc.9 b/share/man/man9/contigmalloc.9 --- a/share/man/man9/contigmalloc.9 +++ b/share/man/man9/contigmalloc.9 @@ -23,7 +23,7 @@ .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE .\" POSSIBILITY OF SUCH DAMAGE. .\" -.Dd October 30, 2018 +.Dd May 23, 2024 .Dt CONTIGMALLOC 9 .Os .Sh NAME @@ -113,6 +113,12 @@ .Fn contigmalloc or .Fn contigmalloc_domainset . +Alternatively +.Xr free 9 +can be called. +This is especially useful when +.Fa size +is not known. .Sh IMPLEMENTATION NOTES The .Fn contigmalloc diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -116,6 +116,12 @@ #define DEBUG_REDZONE_ARG #endif +#define SLAB_COOKIE_SLAB_PTR 0x0 +#define SLAB_COOKIE_MALLOC_LARGE 0x1 +#define SLAB_COOKIE_CONTIG_MALLOC 0x2 +#define SLAB_COOKIE_MASK 0x3 +#define SLAB_COOKIE_SHIFT 2 + /* * When realloc() is called, if the new size is sufficiently smaller than * the old size, realloc() will allocate a new, smaller block to avoid @@ -451,6 +457,30 @@ * If M_NOWAIT is set, this routine will not block and return NULL if * the allocation fails. */ +#ifdef INVARIANTS +static inline bool +contigmalloc_slab(uma_slab_t slab) +{ + uintptr_t cookie; + + cookie = (uintptr_t)slab & SLAB_COOKIE_MASK; + KASSERT(cookie != SLAB_COOKIE_MASK, ("%s: cannot have large_malloc and " + "contigmalloc on slab %p\n", __func__, slab)); + return ((cookie & SLAB_COOKIE_CONTIG_MALLOC) != 0); +} +#endif + +static inline size_t +contigmalloc_size(uma_slab_t slab) +{ + uintptr_t va; + + KASSERT(((uintptr_t)slab & SLAB_COOKIE_MASK) == SLAB_COOKIE_CONTIG_MALLOC, + ("%s: called on non-contigmalloc allocation: %p\n", __func__, slab)); + va = (uintptr_t)slab; + return (va >> SLAB_COOKIE_SHIFT); +} + void * contigmalloc(unsigned long size, struct malloc_type *type, int flags, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, @@ -460,8 +490,12 @@ ret = (void *)kmem_alloc_contig(size, flags, low, high, alignment, boundary, VM_MEMATTR_DEFAULT); - if (ret != NULL) + if (ret != NULL) { + /* Use low bits unused for slab pointers. */ + vsetzoneslab((uintptr_t)ret, NULL, + (void *)((size << SLAB_COOKIE_SHIFT) | SLAB_COOKIE_CONTIG_MALLOC)); malloc_type_allocated(type, round_page(size)); + } return (ret); } @@ -474,8 +508,12 @@ ret = (void *)kmem_alloc_contig_domainset(ds, size, flags, low, high, alignment, boundary, VM_MEMATTR_DEFAULT); - if (ret != NULL) + if (ret != NULL) { + /* Use low bits unused for slab pointers. */ + vsetzoneslab((uintptr_t)ret, NULL, + (void *)((size << SLAB_COOKIE_SHIFT) | SLAB_COOKIE_CONTIG_MALLOC)); malloc_type_allocated(type, round_page(size)); + } return (ret); } @@ -486,14 +524,37 @@ * * This routine may not block. */ -void -contigfree(void *addr, unsigned long size, struct malloc_type *type) +static void +_contigfree(void *addr, unsigned long size, struct malloc_type *type) { kmem_free(addr, size); malloc_type_freed(type, round_page(size)); } +void +contigfree(void *addr, unsigned long size, struct malloc_type *type) +{ +#ifdef INVARIANTS + uma_slab_t slab; + + slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); + if (slab == NULL) + panic("%s: address %p(%p) has not been allocated.\n", + __func__, addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); + + if (!contigmalloc_slab(slab)) + panic("%s: address %p(%p) slab %p was not allocated by contigmalloc.\n", + __func__, addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)), slab); + + if (contigmalloc_size(slab) != size) + panic("%s: address %p(%p) slab %p had different allocation size: %lu != %lu.\n", + __func__, addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)), slab, + contigmalloc_size(slab), size); +#endif + _contigfree(addr, size, type); +} + #ifdef MALLOC_DEBUG static int malloc_dbg(caddr_t *vap, size_t *sizep, struct malloc_type *mtp, @@ -562,22 +623,15 @@ /* * Handle large allocations and frees by using kmem_malloc directly. */ -static inline bool -malloc_large_slab(uma_slab_t slab) -{ - uintptr_t va; - - va = (uintptr_t)slab; - return ((va & 1) != 0); -} - static inline size_t malloc_large_size(uma_slab_t slab) { uintptr_t va; va = (uintptr_t)slab; - return (va >> 1); + KASSERT(((uintptr_t)slab & SLAB_COOKIE_MASK) == SLAB_COOKIE_MALLOC_LARGE, + ("%s: called on non-malloc_large allocation: %p\n", __func__, slab)); + return (va >> SLAB_COOKIE_SHIFT); } static caddr_t __noinline @@ -589,8 +643,9 @@ size = roundup(size, PAGE_SIZE); va = kmem_malloc_domainset(policy, size, flags); if (va != NULL) { - /* The low bit is unused for slab pointers. */ - vsetzoneslab((uintptr_t)va, NULL, (void *)((size << 1) | 1)); + /* Use low bits unused for slab pointers. */ + vsetzoneslab((uintptr_t)va, NULL, + (void *)((size << SLAB_COOKIE_SHIFT) | SLAB_COOKIE_MALLOC_LARGE)); uma_total_inc(size); } malloc_type_allocated(mtp, va == NULL ? 0 : size); @@ -903,6 +958,7 @@ uma_zone_t zone; uma_slab_t slab; u_long size; + int slab_cookie; #ifdef MALLOC_DEBUG if (free_dbg(&addr, mtp) != 0) @@ -917,17 +973,30 @@ panic("free: address %p(%p) has not been allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); - if (__predict_true(!malloc_large_slab(slab))) { + slab_cookie = (uintptr_t)slab & SLAB_COOKIE_MASK; + switch (slab_cookie) { + case SLAB_COOKIE_SLAB_PTR: size = zone->uz_size; #if defined(INVARIANTS) && !defined(KASAN) free_save_type(addr, mtp, size); #endif uma_zfree_arg(zone, addr, slab); - } else { + malloc_type_freed(mtp, size); + break; + case SLAB_COOKIE_MALLOC_LARGE: size = malloc_large_size(slab); free_large(addr, size); + malloc_type_freed(mtp, size); + break; + case SLAB_COOKIE_CONTIG_MALLOC: + size = contigmalloc_size(slab); + _contigfree(addr, size, mtp); + break; + default: + panic("%s: addr %p slab %p with unknown cookie %d\n", __func__, + addr, slab, slab_cookie); + /* NOTREACHED */ } - malloc_type_freed(mtp, size); } /* @@ -943,6 +1012,7 @@ uma_zone_t zone; uma_slab_t slab; u_long size; + int slab_cookie; #ifdef MALLOC_DEBUG if (free_dbg(&addr, mtp) != 0) @@ -957,7 +1027,9 @@ panic("free: address %p(%p) has not been allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); - if (__predict_true(!malloc_large_slab(slab))) { + slab_cookie = (uintptr_t)slab & SLAB_COOKIE_MASK; + switch (slab_cookie) { + case SLAB_COOKIE_SLAB_PTR: size = zone->uz_size; #if defined(INVARIANTS) && !defined(KASAN) free_save_type(addr, mtp, size); @@ -965,13 +1037,26 @@ kasan_mark(addr, size, size, 0); explicit_bzero(addr, size); uma_zfree_arg(zone, addr, slab); - } else { + malloc_type_freed(mtp, size); + break; + case SLAB_COOKIE_MALLOC_LARGE: size = malloc_large_size(slab); kasan_mark(addr, size, size, 0); explicit_bzero(addr, size); free_large(addr, size); + malloc_type_freed(mtp, size); + break; + case SLAB_COOKIE_CONTIG_MALLOC: + size = contigmalloc_size(slab); + /* XXX-BZ kasan? */ + explicit_bzero(addr, size); + _contigfree(addr, size, mtp); + break; + default: + panic("%s: addr %p slab %p with unknown cookie %d\n", __func__, + addr, slab, slab_cookie); + /* NOTREACHED */ } - malloc_type_freed(mtp, size); } /* @@ -985,6 +1070,7 @@ uma_slab_t slab; #endif unsigned long alloc; + int slab_cookie; void *newaddr; KASSERT(mtp->ks_version == M_VERSION, @@ -1016,10 +1102,21 @@ ("realloc: address %p out of range", (void *)addr)); /* Get the size of the original block */ - if (!malloc_large_slab(slab)) + slab_cookie = (uintptr_t)slab & SLAB_COOKIE_MASK; + switch (slab_cookie) { + case SLAB_COOKIE_SLAB_PTR: alloc = zone->uz_size; - else + break; + case SLAB_COOKIE_MALLOC_LARGE: alloc = malloc_large_size(slab); + break; + default: +#ifdef INVARIANTS + panic("%s: called for addr %p of unsupported allocation type; " + "slab %p cookie %d\n", __func__, addr, slab, slab_cookie); +#endif + return (NULL); + } /* Reuse the original block if appropriate */ if (size <= alloc && @@ -1084,6 +1181,7 @@ uma_slab_t slab; #endif u_long size; + int slab_cookie; if (addr == NULL) return (0); @@ -1101,10 +1199,22 @@ panic("malloc_usable_size: address %p(%p) is not allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); - if (!malloc_large_slab(slab)) + slab_cookie = (uintptr_t)slab & SLAB_COOKIE_MASK; + switch (slab_cookie) { + case SLAB_COOKIE_SLAB_PTR: size = zone->uz_size; - else + break; + case SLAB_COOKIE_MALLOC_LARGE: size = malloc_large_size(slab); + break; + default: +#ifdef INVARIANTS + panic("%s: called for addr %p of unsupported allocation type; " + "slab %p cookie %d\n", __func__, addr, slab, slab_cookie); +#endif + size = 0; + break; + } #endif /*