Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -93,13 +93,10 @@ #include #include #include +#include #include -#ifdef DEBUG_MEMGUARD -#include -#endif - /* * This is the zone and keg from which all zones are spawned. */ @@ -1845,6 +1842,11 @@ for (i = 0; i < vm_ndomains; i++) TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); +#ifdef INVARIANTS + if (arg->uminit == trash_init && arg->fini == trash_fini) + zone->uz_flags |= UMA_ZFLAG_TRASH; +#endif + /* * This is a pure cache zone, no kegs. */ @@ -2278,14 +2280,12 @@ args.fini = fini; #ifdef INVARIANTS /* - * If a zone is being created with an empty constructor and - * destructor, pass UMA constructor/destructor which checks for - * memory use after free. + * If a zone is being created with empty init and fini procedures, + * inject UMA procedures which check for memory use after free. + * Note, the ctor and dtor do not need to be empty for this check. */ if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOFREE))) && - ctor == NULL && dtor == NULL && uminit == NULL && fini == NULL) { - args.ctor = trash_ctor; - args.dtor = trash_dtor; + uminit == NULL && fini == NULL) { args.uminit = trash_init; args.fini = trash_fini; } @@ -2418,6 +2418,62 @@ uma_zfree_arg(zone, item, udata); } +static inline int +uma_zctor(uma_zone_t zone, void *mem, void *arg, int flags) +{ + int ret; +#ifdef INVARIANTS + bool skipdbg; + + skipdbg = uma_dbg_zskip(zone, mem); + + if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && + zone->uz_ctor != trash_ctor) + trash_ctor(mem, zone->uz_size, arg, flags); +#endif + ret = 0; + if (zone->uz_ctor != NULL) + ret = zone->uz_ctor(mem, zone->uz_size, arg, flags); + + if (ret == 0 && !is_memguard_addr(mem)) { +#ifdef INVARIANTS + if (!skipdbg) + uma_dbg_alloc(zone, NULL, mem); +#endif + if ((flags & M_ZERO) != 0) + uma_zero_item(mem, zone); + } + + return (ret); +} + +static inline void +uma_zdtor(uma_zone_t zone, void *mem, void *arg, enum zfreeskip skip) +{ +#ifdef INVARIANTS + bool skipdbg; + + skipdbg = uma_dbg_zskip(zone, mem); + + if (skip == SKIP_NONE && !skipdbg && !is_memguard_addr(mem)) { + if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) + uma_dbg_free(zone, arg, mem); + else + uma_dbg_free(zone, NULL, mem); + } +#endif + + if (skip < SKIP_DTOR) { + if (zone->uz_dtor != NULL) + zone->uz_dtor(mem, zone->uz_size, arg); +#ifdef INVARIANTS + if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && + zone->uz_dtor != trash_dtor) + trash_dtor(mem, zone->uz_size, arg); +#endif + } +} + /* See uma.h */ void * uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) @@ -2427,9 +2483,6 @@ uma_cache_t cache; void *item; int cpu, domain, lockfail, maxbucket; -#ifdef INVARIANTS - bool skipdbg; -#endif /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); @@ -2456,9 +2509,7 @@ if (zone->uz_init != NULL && zone->uz_init(item, zone->uz_size, flags) != 0) return (NULL); - if (zone->uz_ctor != NULL && - zone->uz_ctor(item, zone->uz_size, udata, - flags) != 0) { + if (uma_zctor(zone, item, udata, flags) != 0) { zone->uz_fini(item, zone->uz_size); return (NULL); } @@ -2494,25 +2545,12 @@ KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); cache->uc_allocs++; critical_exit(); -#ifdef INVARIANTS - skipdbg = uma_dbg_zskip(zone, item); -#endif - if (zone->uz_ctor != NULL && -#ifdef INVARIANTS - (!skipdbg || zone->uz_ctor != trash_ctor || - zone->uz_dtor != trash_dtor) && -#endif - zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { + + if (uma_zctor(zone, item, udata, flags) != 0) { counter_u64_add(zone->uz_fails, 1); zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); return (NULL); } -#ifdef INVARIANTS - if (!skipdbg) - uma_dbg_alloc(zone, NULL, item); -#endif - if (flags & M_ZERO) - uma_zero_item(item, zone); return (item); } @@ -3001,9 +3039,6 @@ zone_alloc_item_locked(uma_zone_t zone, void *udata, int domain, int flags) { void *item; -#ifdef INVARIANTS - bool skipdbg; -#endif ZONE_LOCK_ASSERT(zone); @@ -3036,9 +3071,6 @@ if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) goto fail; -#ifdef INVARIANTS - skipdbg = uma_dbg_zskip(zone, item); -#endif /* * We have to call both the zone's init (not the keg's init) * and the zone's ctor. This is because the item is going from @@ -3051,21 +3083,10 @@ goto fail; } } - if (zone->uz_ctor != NULL && -#ifdef INVARIANTS - (!skipdbg || zone->uz_ctor != trash_ctor || - zone->uz_dtor != trash_dtor) && -#endif - zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { + if (uma_zctor(zone, item, udata, flags) != 0) { zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); goto fail; } -#ifdef INVARIANTS - if (!skipdbg) - uma_dbg_alloc(zone, NULL, item); -#endif - if (flags & M_ZERO) - uma_zero_item(item, zone); counter_u64_add(zone->uz_allocs, 1); CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, @@ -3097,9 +3118,6 @@ int itemdomain; #endif bool lockfail; -#ifdef INVARIANTS - bool skipdbg; -#endif /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); @@ -3113,30 +3131,15 @@ /* uma_zfree(..., NULL) does nothing, to match free(9). */ if (item == NULL) return; + uma_zdtor(zone, item, udata, SKIP_NONE); #ifdef DEBUG_MEMGUARD if (is_memguard_addr(item)) { - if (zone->uz_dtor != NULL) - zone->uz_dtor(item, zone->uz_size, udata); if (zone->uz_fini != NULL) zone->uz_fini(item, zone->uz_size); memguard_free(item); return; } #endif -#ifdef INVARIANTS - skipdbg = uma_dbg_zskip(zone, item); - if (skipdbg == false) { - if (zone->uz_flags & UMA_ZONE_MALLOC) - uma_dbg_free(zone, udata, item); - else - uma_dbg_free(zone, NULL, item); - } - if (zone->uz_dtor != NULL && (!skipdbg || - zone->uz_dtor != trash_dtor || zone->uz_ctor != trash_ctor)) -#else - if (zone->uz_dtor != NULL) -#endif - zone->uz_dtor(item, zone->uz_size, udata); /* * The race here is acceptable. If we miss it we'll just have to wait @@ -3425,24 +3428,8 @@ static void zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) { -#ifdef INVARIANTS - bool skipdbg; - skipdbg = uma_dbg_zskip(zone, item); - if (skip == SKIP_NONE && !skipdbg) { - if (zone->uz_flags & UMA_ZONE_MALLOC) - uma_dbg_free(zone, udata, item); - else - uma_dbg_free(zone, NULL, item); - } - - if (skip < SKIP_DTOR && zone->uz_dtor != NULL && - (!skipdbg || zone->uz_dtor != trash_dtor || - zone->uz_ctor != trash_ctor)) -#else - if (skip < SKIP_DTOR && zone->uz_dtor != NULL) -#endif - zone->uz_dtor(item, zone->uz_size, udata); + uma_zdtor(zone, item, udata, skip); if (skip < SKIP_FINI && zone->uz_fini) zone->uz_fini(item, zone->uz_size); Index: sys/vm/uma_int.h =================================================================== --- sys/vm/uma_int.h +++ sys/vm/uma_int.h @@ -389,6 +389,7 @@ #define UMA_ZFLAG_RECLAIMING 0x08000000 /* Running zone_reclaim(). */ #define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */ #define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */ +#define UMA_ZFLAG_TRASH 0x40000000 /* Add trash ctor/dtor. */ #define UMA_ZFLAG_CACHEONLY 0x80000000 /* Don't ask VM for buckets. */ #define UMA_ZFLAG_INHERIT \