Page MenuHomeFreeBSD
Files F142738501

D16667.id46531.diff
No OneTemporary
Actions

D16667.id46531.diff
View Options

Index: sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c
===================================================================
--- sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c
+++ sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c
@@ -238,14 +238,14 @@
kmem_cache_reap_soon(kmem_cache_t *cache)
{
#ifndef KMEM_DEBUG
- zone_drain(cache->kc_zone);
+ uma_zreclaim(cache->kc_zone, UMA_RECLAIM_DRAIN);
#endif
}
void
kmem_reap(void)
{
- uma_reclaim();
+ uma_reclaim(UMA_RECLAIM_TRIM);
}
#else
void
Index: sys/kern/kern_mbuf.c
===================================================================
--- sys/kern/kern_mbuf.c
+++ sys/kern/kern_mbuf.c
@@ -680,7 +680,7 @@
* mbuf free.
*/
if (uma_zone_exhausted_nolock(zone_clust))
- zone_drain(zone_pack);
+ uma_zreclaim(zone_pack, UMA_RECLAIM_DRAIN);
}
/*
@@ -924,7 +924,7 @@
* we might be able to loosen a few clusters up on the drain.
*/
if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
- zone_drain(zone_pack);
+ uma_zreclaim(zone_pack, UMA_RECLAIM_DRAIN);
uma_zalloc_arg(zone_clust, m, how);
}
MBUF_PROBE2(m__clget, m, how);
Index: sys/kern/subr_vmem.c
===================================================================
--- sys/kern/subr_vmem.c
+++ sys/kern/subr_vmem.c
@@ -589,7 +589,7 @@
qcache_idx_max = vm->vm_qcache_max >> vm->vm_quantum_shift;
for (i = 0; i < qcache_idx_max; i++)
- zone_drain(vm->vm_qcache[i].qc_cache);
+ uma_zreclaim(vm->vm_qcache[i].qc_cache, UMA_RECLAIM_DRAIN);
}
#ifndef UMA_MD_SMALL_ALLOC
Index: sys/kern/vfs_subr.c
===================================================================
--- sys/kern/vfs_subr.c
+++ sys/kern/vfs_subr.c
@@ -1205,7 +1205,7 @@
}
mtx_unlock(&mountlist_mtx);
if (onumvnodes > desiredvnodes && numvnodes <= desiredvnodes)
- uma_reclaim();
+ uma_reclaim(UMA_RECLAIM_DRAIN);
if (done == 0) {
if (force == 0 || force == 1) {
force = 2;
Index: sys/vm/uma.h
===================================================================
--- sys/vm/uma.h
+++ sys/vm/uma.h
@@ -50,8 +50,6 @@
/* Opaque type used as a handle to the zone */
typedef struct uma_zone * uma_zone_t;
-void zone_drain(uma_zone_t);
-
/*
* Item constructor
*
@@ -450,17 +448,18 @@
typedef void (*uma_free)(void *item, vm_size_t size, uint8_t pflag);
/*
- * Reclaims unused memory for all zones
+ * Reclaims unused memory
*
* Arguments:
- * None
+ * req Reclamation request type.
* Returns:
* None
- *
- * This should only be called by the page out daemon.
*/
-
-void uma_reclaim(void);
+#define UMA_RECLAIM_DRAIN 1 /* release bucket cache */
+#define UMA_RECLAIM_DRAIN_CPU 2 /* release bucket and per-CPU caches */
+#define UMA_RECLAIM_TRIM 3 /* trim bucket cache to WSS */
+void uma_reclaim(int req);
+void uma_zreclaim(uma_zone_t, int req);
/*
* Sets the alignment mask to be used for all zones requesting cache
Index: sys/vm/uma_core.c
===================================================================
--- sys/vm/uma_core.c
+++ sys/vm/uma_core.c
@@ -236,7 +236,7 @@
static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int);
static void cache_drain(uma_zone_t);
static void bucket_drain(uma_zone_t, uma_bucket_t);
-static void bucket_cache_drain(uma_zone_t zone);
+static void bucket_cache_reclaim(uma_zone_t zone, bool);
static int keg_ctor(void *, int, void *, int);
static void keg_dtor(void *, int, void *);
static int zone_ctor(void *, int, void *, int);
@@ -453,7 +453,7 @@
struct uma_bucket_zone *ubz;
for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
- zone_drain(ubz->ubz_zone);
+ uma_zreclaim(ubz->ubz_zone, UMA_RECLAIM_DRAIN);
}
static uma_bucket_t
@@ -762,7 +762,7 @@
* XXX: It would good to be able to assert that the zone is being
* torn down to prevent improper use of cache_drain().
*
- * XXX: We lock the zone before passing into bucket_cache_drain() as
+ * XXX: We lock the zone before passing into bucket_cache_reclaim() as
* it is used elsewhere. Should the tear-down path be made special
* there in some form?
*/
@@ -777,7 +777,7 @@
cache->uc_allocbucket = cache->uc_freebucket = NULL;
}
ZONE_LOCK(zone);
- bucket_cache_drain(zone);
+ bucket_cache_reclaim(zone, true);
ZONE_UNLOCK(zone);
}
@@ -843,7 +843,7 @@
* Zone lock must not be held on call this function.
*/
static void
-cache_drain_safe(uma_zone_t zone)
+pcpu_cache_drain_safe(uma_zone_t zone)
{
int cpu;
@@ -871,22 +871,24 @@
}
/*
- * Drain the cached buckets from a zone. Expects a locked zone on entry.
+ * Reclaim cached buckets from a zone. All buckets are reclaimed if the caller
+ * requested a drain, otherwise the per-domain caches are trimmed to either
+ * estimated working set size.
*/
static void
-bucket_cache_drain(uma_zone_t zone)
+bucket_cache_reclaim(uma_zone_t zone, bool drain)
{
uma_zone_domain_t zdom;
uma_bucket_t bucket;
- int i;
+ int i, target;
- /*
- * Drain the bucket queues and free the buckets.
- */
for (i = 0; i < vm_ndomains; i++) {
zdom = &zone->uz_domain[i];
- while ((bucket = zone_try_fetch_bucket(zone, zdom, false)) !=
- NULL) {
+ target = drain ? 0 : zdom->uzd_wss;
+ while (zdom->uzd_nitems > target) {
+ bucket = zone_try_fetch_bucket(zone, zdom, false);
+ if (bucket == NULL)
+ break;
ZONE_UNLOCK(zone);
bucket_drain(zone, bucket);
bucket_free(zone, bucket, NULL);
@@ -895,8 +897,8 @@
}
/*
- * Shrink further bucket sizes. Price of single zone lock collision
- * is probably lower then price of global cache drain.
+ * Shrink the zone bucket size to ensure that the per-CPU caches
+ * don't grow too large.
*/
if (zone->uz_count > zone->uz_count_min)
zone->uz_count--;
@@ -994,7 +996,7 @@
}
static void
-zone_drain_wait(uma_zone_t zone, int waitok)
+zone_reclaim(uma_zone_t zone, int waitok, bool drain)
{
/*
@@ -1004,13 +1006,13 @@
* when it wakes up.
*/
ZONE_LOCK(zone);
- while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
+ while (zone->uz_flags & UMA_ZFLAG_RECLAIMING) {
if (waitok == M_NOWAIT)
goto out;
msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
}
- zone->uz_flags |= UMA_ZFLAG_DRAINING;
- bucket_cache_drain(zone);
+ zone->uz_flags |= UMA_ZFLAG_RECLAIMING;
+ bucket_cache_reclaim(zone, drain);
ZONE_UNLOCK(zone);
/*
* The DRAINING flag protects us from being freed while
@@ -1019,17 +1021,24 @@
*/
zone_foreach_keg(zone, &keg_drain);
ZONE_LOCK(zone);
- zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
+ zone->uz_flags &= ~UMA_ZFLAG_RECLAIMING;
wakeup(zone);
out:
ZONE_UNLOCK(zone);
}
-void
+static void
zone_drain(uma_zone_t zone)
{
- zone_drain_wait(zone, M_NOWAIT);
+ zone_reclaim(zone, M_NOWAIT, true);
+}
+
+static void
+zone_trim(uma_zone_t zone)
+{
+
+ zone_reclaim(zone, M_NOWAIT, false);
}
/*
@@ -1911,7 +1920,7 @@
* released and then refilled before we
* remove it... we dont care for now
*/
- zone_drain_wait(zone, M_WAITOK);
+ zone_reclaim(zone, M_WAITOK, true);
/*
* Unlink all of our kegs.
*/
@@ -3642,17 +3651,28 @@
}
/* See uma.h */
-static void
-uma_reclaim_locked(bool kmem_danger)
+void
+uma_reclaim(int req)
{
CTR0(KTR_UMA, "UMA: vm asked us to release pages!");
- sx_assert(&uma_drain_lock, SA_XLOCKED);
+ sx_xlock(&uma_drain_lock);
bucket_enable();
- zone_foreach(zone_drain);
- if (vm_page_count_min() || kmem_danger) {
- cache_drain_safe(NULL);
+
+ switch (req) {
+ case UMA_RECLAIM_TRIM:
+ zone_foreach(zone_trim);
+ break;
+ case UMA_RECLAIM_DRAIN:
+ case UMA_RECLAIM_DRAIN_CPU:
zone_foreach(zone_drain);
+ if (req == UMA_RECLAIM_DRAIN_CPU) {
+ pcpu_cache_drain_safe(NULL);
+ zone_foreach(zone_drain);
+ }
+ break;
+ default:
+ panic("unhandled reclamation request %d", req);
}
/*
@@ -3662,14 +3682,6 @@
*/
zone_drain(slabzone);
bucket_zone_drain();
-}
-
-void
-uma_reclaim(void)
-{
-
- sx_xlock(&uma_drain_lock);
- uma_reclaim_locked(false);
sx_xunlock(&uma_drain_lock);
}
@@ -3694,15 +3706,34 @@
hz);
sx_xunlock(&uma_drain_lock);
EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM);
- sx_xlock(&uma_drain_lock);
- uma_reclaim_locked(true);
+ uma_reclaim(UMA_RECLAIM_DRAIN_CPU);
atomic_store_int(&uma_reclaim_needed, 0);
- sx_xunlock(&uma_drain_lock);
/* Don't fire more than once per-second. */
pause("umarclslp", hz);
}
}
+/* See uma.h */
+void
+uma_zreclaim(uma_zone_t zone, int req)
+{
+
+ switch (req) {
+ case UMA_RECLAIM_TRIM:
+ zone_trim(zone);
+ break;
+ case UMA_RECLAIM_DRAIN:
+ zone_drain(zone);
+ break;
+ case UMA_RECLAIM_DRAIN_CPU:
+ pcpu_cache_drain_safe(zone);
+ zone_drain(zone);
+ break;
+ default:
+ panic("unhandled reclamation request %d", req);
+ }
+}
+
/* See uma.h */
int
uma_zone_exhausted(uma_zone_t zone)
Index: sys/vm/uma_int.h
===================================================================
--- sys/vm/uma_int.h
+++ sys/vm/uma_int.h
@@ -374,7 +374,7 @@
* These flags must not overlap with the UMA_ZONE flags specified in uma.h.
*/
#define UMA_ZFLAG_MULTI 0x04000000 /* Multiple kegs in the zone. */
-#define UMA_ZFLAG_DRAINING 0x08000000 /* Running zone_drain. */
+#define UMA_ZFLAG_RECLAIMING 0x08000000 /* Running zone_reclaim(). */
#define UMA_ZFLAG_BUCKET 0x10000000 /* Bucket zone. */
#define UMA_ZFLAG_INTERNAL 0x20000000 /* No offpage no PCPU. */
#define UMA_ZFLAG_FULL 0x40000000 /* Reached uz_maxpages */
Index: sys/vm/vm_pageout.c
===================================================================
--- sys/vm/vm_pageout.c
+++ sys/vm/vm_pageout.c
@@ -1874,9 +1874,12 @@
/*
* We do this explicitly after the caches have been
- * drained above.
+ * drained above. If we have a severe page shortage on
+ * our hands, completely drain all UMA zones. Otherwise,
+ * just prune the caches.
*/
- uma_reclaim();
+ uma_reclaim(vm_page_count_min() ? UMA_RECLAIM_DRAIN_CPU :
+ UMA_RECLAIM_TRIM);
return (true);
}
return (false);

File Metadata

Mime Type
text/plain
Expires
Fri, Jan 23, 10:41 PM (11 h, 40 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
27890338
Default Alt Text
D16667.id46531.diff (9 KB)

Event Timeline