Index: sys/vm/uma_core.c =================================================================== --- sys/vm/uma_core.c +++ sys/vm/uma_core.c @@ -1095,10 +1095,17 @@ target = drain ? 0 : lmax(zdom->uzd_wss, zdom->uzd_nitems - zdom->uzd_imin); while (zdom->uzd_nitems > target) { - bucket = TAILQ_LAST(&zdom->uzd_buckets, uma_bucketlist); - if (bucket == NULL) - break; - tofree = bucket->ub_cnt; + if (zdom->uzd_cross != NULL) { + bucket = zdom->uzd_cross; + zdom->uzd_cross = NULL; + tofree = 0; + } else { + bucket = TAILQ_LAST(&zdom->uzd_buckets, + uma_bucketlist); + if (bucket == NULL) + break; + tofree = bucket->ub_cnt; + } TAILQ_REMOVE(&zdom->uzd_buckets, bucket, ub_link); zdom->uzd_nitems -= tofree; @@ -2263,6 +2270,7 @@ zone_foreach(zone_count, &cnt); zone->uz_namecnt = cnt.count; ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS)); + ZONE_CROSS_LOCK_INIT(zone); for (i = 0; i < vm_ndomains; i++) TAILQ_INIT(&zone->uz_domain[i].uzd_buckets); @@ -2448,6 +2456,7 @@ counter_u64_free(zone->uz_fails); free(zone->uz_ctlname, M_UMA); ZONE_LOCK_FINI(zone); + ZONE_CROSS_LOCK_FINI(zone); } /* @@ -3724,6 +3733,76 @@ zone_free_item(zone, item, udata, SKIP_DTOR); } +#ifdef UMA_XDOMAIN +/* + * sort crossdomain free buckets to domain correct buckets and cache + * them. + */ +static void +zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) +{ + struct uma_bucketlist fullbuckets; + uma_zone_domain_t zdom; + uma_bucket_t b; + void *item; + int domain; + + CTR3(KTR_UMA, + "uma_zfree: zone %s(%p) draining cross bucket %p", + zone->uz_name, zone, bucket); + + TAILQ_INIT(&fullbuckets); + + /* + * To avoid having ndomain * ndomain buckets for sorting we have a + * lock on the current crossfree bucket. A full matrix with + * per-domain locking could be used if necessary. + */ + ZONE_CROSS_LOCK(zone); + while (bucket->ub_cnt > 0) { + item = bucket->ub_bucket[bucket->ub_cnt - 1]; + domain = _vm_phys_domain(pmap_kextract((vm_offset_t)item)); + zdom = &zone->uz_domain[domain]; + if (zdom->uzd_cross == NULL) { + zdom->uzd_cross = bucket_alloc(zone, udata, M_NOWAIT); + if (zdom->uzd_cross == NULL) + break; + } + zdom->uzd_cross->ub_bucket[zdom->uzd_cross->ub_cnt++] = item; + if (zdom->uzd_cross->ub_cnt == zdom->uzd_cross->ub_entries) { + TAILQ_INSERT_HEAD(&fullbuckets, zdom->uzd_cross, + ub_link); + zdom->uzd_cross = NULL; + } + bucket->ub_cnt--; + } + ZONE_CROSS_UNLOCK(zone); + if (!TAILQ_EMPTY(&fullbuckets)) { + ZONE_LOCK(zone); + while ((b = TAILQ_FIRST(&fullbuckets)) != NULL) { + TAILQ_REMOVE(&fullbuckets, b, ub_link); + if (zone->uz_bkt_count >= zone->uz_bkt_max) { + ZONE_UNLOCK(zone); + bucket_drain(zone, bucket); + bucket_free(zone, bucket, udata); + ZONE_LOCK(zone); + } else { + domain = _vm_phys_domain( + pmap_kextract((vm_offset_t) + bucket->ub_bucket[0])); + zdom = &zone->uz_domain[domain]; + zone_put_bucket(zone, zdom, b, true); + } + } + ZONE_UNLOCK(zone); + } + if (bucket->ub_cnt != 0) + bucket_drain(zone, bucket); + bucket_free(zone, bucket, udata); + return; +} +#endif + static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, int domain, int itemdomain) @@ -3735,17 +3814,14 @@ * Buckets coming from the wrong domain will be entirely for the * only other domain on two domain systems. In this case we can * simply cache them. Otherwise we need to sort them back to - * correct domains by freeing the contents to the slab layer. + * correct domains. */ if (domain != itemdomain && vm_ndomains > 2) { - CTR3(KTR_UMA, - "uma_zfree: zone %s(%p) draining cross bucket %p", - zone->uz_name, zone, bucket); - bucket_drain(zone, bucket); - bucket_free(zone, bucket, udata); + zone_free_cross(zone, bucket, udata); return; } #endif + /* * Attempt to save the bucket in the zone's domain bucket cache. * Index: sys/vm/uma_int.h =================================================================== --- sys/vm/uma_int.h +++ sys/vm/uma_int.h @@ -399,6 +399,7 @@ struct uma_zone_domain { struct uma_bucketlist uzd_buckets; /* full buckets */ + uma_bucket_t uzd_cross; /* Fills from cross buckets. */ long uzd_nitems; /* total item count */ long uzd_imax; /* maximum item count this period */ long uzd_imin; /* minimum item count this period */ @@ -449,6 +450,8 @@ struct task uz_maxaction; /* Task to run when at limit */ uint16_t uz_bucket_size_min; /* Min number of items in bucket */ + struct mtx_padalign uz_cross_lock; /* Cross domain free lock */ + /* Offset 256+, stats and misc. */ counter_u64_t uz_allocs; /* Total number of allocations */ counter_u64_t uz_frees; /* Total number of frees */ @@ -575,6 +578,12 @@ #define ZONE_LOCK_FINI(z) mtx_destroy(&(z)->uz_lock) #define ZONE_LOCK_ASSERT(z) mtx_assert(&(z)->uz_lock, MA_OWNED) +#define ZONE_CROSS_LOCK_INIT(z) \ + mtx_init(&(z)->uz_cross_lock, "UMA Cross", NULL, MTX_DEF) +#define ZONE_CROSS_LOCK(z) mtx_lock(&(z)->uz_cross_lock) +#define ZONE_CROSS_UNLOCK(z) mtx_unlock(&(z)->uz_cross_lock) +#define ZONE_CROSS_LOCK_FINI(z) mtx_destroy(&(z)->uz_cross_lock) + /* * Find a slab within a hash table. This is used for OFFPAGE zones to lookup * the slab structure.