Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c +++ sys/vm/vm_page.c @@ -185,6 +185,12 @@ VM_ALLOC_NORMAL | VM_ALLOC_WIRED); } +struct pgcache_arg { + uint32_t pool : 4; + uint32_t domain : 28; +}; +CTASSERT(1 << 4 > VM_NFREEPOOL); + /* * The cache page zone is initialized later since we need to be able to allocate * pages before UMA is fully initialized. @@ -193,21 +199,28 @@ vm_page_init_cache_zones(void *dummy __unused) { struct vm_domain *vmd; - int i; + struct vm_pgcache *pgcache; + int domain, pool; + + for (domain = 0; domain < vm_ndomains; domain++) { + vmd = VM_DOMAIN(domain); - for (i = 0; i < vm_ndomains; i++) { - vmd = VM_DOMAIN(i); /* - * Don't allow the page cache to take up more than .25% of + * Don't allow the page caches to take up more than .25% of * memory. */ - if (vmd->vmd_page_count / 400 < 256 * mp_ncpus) + if (vmd->vmd_page_count / 400 < 256 * mp_ncpus * VM_NFREEPOOL) continue; - vmd->vmd_pgcache = uma_zcache_create("vm pgcache", - sizeof(struct vm_page), NULL, NULL, NULL, NULL, - vm_page_zone_import, vm_page_zone_release, vmd, - UMA_ZONE_MAXBUCKET | UMA_ZONE_VM); - (void )uma_zone_set_maxcache(vmd->vmd_pgcache, 0); + for (pool = 0; pool < VM_NFREEPOOL; pool++) { + pgcache = &vmd->vmd_pgcache[pool]; + pgcache->domain = domain; + pgcache->pool = pool; + pgcache->zone = uma_zcache_create("vm pgcache", + sizeof(struct vm_page), NULL, NULL, NULL, NULL, + vm_page_zone_import, vm_page_zone_release, pgcache, + UMA_ZONE_MAXBUCKET | UMA_ZONE_VM); + (void)uma_zone_set_maxcache(pgcache->zone, 0); + } } } SYSINIT(vm_page2, SI_SUB_VM_CONF, SI_ORDER_ANY, vm_page_init_cache_zones, NULL); @@ -1788,7 +1801,7 @@ { struct vm_domain *vmd; vm_page_t m; - int flags; + int flags, pool; KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) && (object != NULL || (req & VM_ALLOC_SBUSY) == 0) && @@ -1805,6 +1818,7 @@ flags = 0; m = NULL; + pool = (object != NULL) ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT; again: #if VM_NRESERVLEVEL > 0 /* @@ -1819,8 +1833,8 @@ } #endif vmd = VM_DOMAIN(domain); - if (object != NULL && vmd->vmd_pgcache != NULL) { - m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT); + if (vmd->vmd_pgcache[pool].zone != NULL) { + m = uma_zalloc(vmd->vmd_pgcache[pool].zone, M_NOWAIT); if (m != NULL) { flags |= PG_PCPU_CACHE; goto found; @@ -1831,8 +1845,7 @@ * If not, allocate it from the free page queues. */ vm_domain_free_lock(vmd); - m = vm_phys_alloc_pages(domain, object != NULL ? - VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0); + m = vm_phys_alloc_pages(domain, pool, 0); vm_domain_free_unlock(vmd); if (m == NULL) { vm_domain_freecnt_inc(vmd, 1); @@ -2223,15 +2236,17 @@ vm_page_zone_import(void *arg, void **store, int cnt, int domain, int flags) { struct vm_domain *vmd; + struct vm_pgcache *pgcache; int i; - vmd = arg; + pgcache = arg; + vmd = VM_DOMAIN(pgcache->domain); /* Only import if we can bring in a full bucket. */ if (cnt == 1 || !vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt)) return (0); domain = vmd->vmd_domain; vm_domain_free_lock(vmd); - i = vm_phys_alloc_npages(domain, VM_FREEPOOL_DEFAULT, cnt, + i = vm_phys_alloc_npages(domain, pgcache->pool, cnt, (vm_page_t *)store); vm_domain_free_unlock(vmd); if (cnt != i) @@ -2244,10 +2259,12 @@ vm_page_zone_release(void *arg, void **store, int cnt) { struct vm_domain *vmd; + struct vm_pgcache *pgcache; vm_page_t m; int i; - vmd = arg; + pgcache = arg; + vmd = VM_DOMAIN(pgcache->domain); vm_domain_free_lock(vmd); for (i = 0; i < cnt; i++) { m = (vm_page_t)store[i]; @@ -3480,13 +3497,15 @@ vm_page_free_toq(vm_page_t m) { struct vm_domain *vmd; + uma_zone_t zone; if (!vm_page_free_prep(m)) return; vmd = vm_pagequeue_domain(m); - if ((m->flags & PG_PCPU_CACHE) != 0 && vmd->vmd_pgcache != NULL) { - uma_zfree(vmd->vmd_pgcache, m); + zone = vmd->vmd_pgcache[m->pool].zone; + if ((m->flags & PG_PCPU_CACHE) != 0 && zone != NULL) { + uma_zfree(zone, m); return; } vm_domain_free_lock(vmd); Index: sys/vm/vm_pagequeue.h =================================================================== --- sys/vm/vm_pagequeue.h +++ sys/vm/vm_pagequeue.h @@ -103,7 +103,11 @@ struct vm_pagequeue vmd_pagequeues[PQ_COUNT]; struct mtx_padalign vmd_free_mtx; struct mtx_padalign vmd_pageout_mtx; - uma_zone_t vmd_pgcache; /* (c) page free cache. */ + struct vm_pgcache { + int domain; + int pool; + uma_zone_t zone; + } vmd_pgcache[VM_NFREEPOOL]; struct vmem *vmd_kernel_arena; /* (c) per-domain kva R/W arena. */ struct vmem *vmd_kernel_rwx_arena; /* (c) per-domain kva R/W/X arena. */ u_int vmd_domain; /* (c) Domain number. */