Index: sys/vm/uma.h =================================================================== --- sys/vm/uma.h +++ sys/vm/uma.h @@ -458,6 +458,7 @@ #define UMA_RECLAIM_DRAIN 1 /* release bucket cache */ #define UMA_RECLAIM_DRAIN_CPU 2 /* release bucket and per-CPU caches */ #define UMA_RECLAIM_TRIM 3 /* trim bucket cache to WSS */ +#define UMA_RECLAIM_DRAIN_ALL 4 /* as much as possible, no shrink */ void uma_reclaim(int req); void uma_reclaim_domain(int req, int domain); void uma_zone_reclaim(uma_zone_t, int req); Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -1434,11 +1434,12 @@ } static void -cache_drain_safe_cpu(uma_zone_t zone, void *unused) +cache_drain_safe_cpu(uma_zone_t zone, void *reqp) { uma_cache_t cache; uma_bucket_t b1, b2, b3; int domain; + const int req = *(int *)reqp; if (zone->uz_flags & UMA_ZFLAG_INTERNAL) return; @@ -1450,10 +1451,12 @@ b1 = cache_bucket_unload_alloc(cache); /* - * Don't flush SMR zone buckets. This leaves the zone without a - * bucket and forces every free to synchronize(). + * Don't flush SMR zone buckets, unless explicitly asked to. + * This leaves the zone without a bucket and forces next free + * to smr_synchronize(). */ - if ((zone->uz_flags & UMA_ZONE_SMR) == 0) { + if ((zone->uz_flags & UMA_ZONE_SMR) == 0 || + req == UMA_RECLAIM_DRAIN_ALL) { b2 = cache_bucket_unload_free(cache); b3 = cache_bucket_unload_cross(cache); } @@ -1461,9 +1464,14 @@ if (b1 != NULL) zone_free_bucket(zone, b1, NULL, domain, false); - if (b2 != NULL) + if (b2 != NULL) { + if ((zone->uz_flags & UMA_ZONE_SMR) != 0) + b2->ub_seq = smr_advance(zone->uz_smr); zone_free_bucket(zone, b2, NULL, domain, false); + } if (b3 != NULL) { + if ((zone->uz_flags & UMA_ZONE_SMR) != 0) + b3->ub_seq = smr_advance(zone->uz_smr); /* Adjust the domain so it goes to zone_free_cross. */ domain = (domain + 1) % vm_ndomains; zone_free_bucket(zone, b3, NULL, domain, false); @@ -1478,7 +1486,7 @@ * Zone lock must not be held on call this function. */ static void -pcpu_cache_drain_safe(uma_zone_t zone) +pcpu_cache_drain_safe(uma_zone_t zone, int req) { int cpu; @@ -1488,9 +1496,9 @@ thread_unlock(curthread); if (zone) - cache_drain_safe_cpu(zone, NULL); + cache_drain_safe_cpu(zone, &req); else - zone_foreach(cache_drain_safe_cpu, NULL); + zone_foreach(cache_drain_safe_cpu, &req); } thread_lock(curthread); sched_unbind(curthread); @@ -5218,7 +5226,7 @@ case UMA_RECLAIM_DRAIN_CPU: zone_foreach(zone_drain, arg); zone_foreach(cache_shrink, NULL); - pcpu_cache_drain_safe(NULL); + pcpu_cache_drain_safe(NULL, req); zone_foreach(zone_drain, arg); break; default: @@ -5286,7 +5294,8 @@ break; case UMA_RECLAIM_DRAIN_CPU: cache_shrink(zone, NULL); - pcpu_cache_drain_safe(zone); + case UMA_RECLAIM_DRAIN_ALL: + pcpu_cache_drain_safe(zone, req); zone_drain(zone, arg); break; default: