Index: sys/ddb/db_ps.c =================================================================== --- sys/ddb/db_ps.c +++ sys/ddb/db_ps.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -505,7 +504,6 @@ { struct proc *p; struct thread *td; - struct kstack_cache_entry *ks_ce; vm_offset_t saddr; if (have_addr) @@ -524,13 +522,4 @@ } } } - - for (ks_ce = kstack_cache; ks_ce != NULL; - ks_ce = ks_ce->next_ks_entry) { - if ((vm_offset_t)ks_ce <= saddr && saddr < (vm_offset_t)ks_ce + - PAGE_SIZE * kstack_pages) { - db_printf("Cached stack %p\n", ks_ce); - return; - } - } } Index: sys/sys/_kstack_cache.h =================================================================== --- sys/sys/_kstack_cache.h +++ /dev/null @@ -1,49 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-3-Clause - * - * Copyright (c) 2009 Konstantin Belousov - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _SYS__KSTACK_CACHE_H -#define _SYS__KSTACK_CACHE_H - -struct kstack_cache_entry { - struct vm_object *ksobj; - struct kstack_cache_entry *next_ks_entry; -}; - -extern struct kstack_cache_entry *kstack_cache; - -#ifndef KSTACK_MAX_PAGES -#define KSTACK_MAX_PAGES 32 -#endif - -#endif - - Index: sys/vm/vm_glue.c =================================================================== --- sys/vm/vm_glue.c +++ sys/vm/vm_glue.c @@ -84,12 +84,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include @@ -282,52 +282,39 @@ pmap_sync_icache(map->pmap, va, sz); } -struct kstack_cache_entry *kstack_cache; +static uma_zone_t kstack_cache; static int kstack_cache_size = 128; -static int kstacks, kstack_domain_iter; -static struct mtx kstack_cache_mtx; -MTX_SYSINIT(kstack_cache, &kstack_cache_mtx, "kstkch", MTX_DEF); +static int kstack_domain_iter; -SYSCTL_INT(_vm, OID_AUTO, kstack_cache_size, CTLFLAG_RW, &kstack_cache_size, 0, - ""); -SYSCTL_INT(_vm, OID_AUTO, kstacks, CTLFLAG_RD, &kstacks, 0, - ""); +static int +sysctl_kstack_cache_size(SYSCTL_HANDLER_ARGS) +{ + int error, newsize; + + newsize = kstack_cache_size; + error = sysctl_handle_int(oidp, &newsize, 0, req); + if (error == 0 && req->newptr && newsize != kstack_cache_size) + kstack_cache_size = + uma_zone_set_maxcache(kstack_cache, newsize); + return (error); +} +SYSCTL_PROC(_vm, OID_AUTO, kstack_cache_size, CTLTYPE_INT|CTLFLAG_RW, + &kstack_cache_size, 0, sysctl_kstack_cache_size, "IU", + "Maximum number of cached kernel stacks"); /* * Create the kernel stack (including pcb for i386) for a new thread. * This routine directly affects the fork perf for a process and * create performance for a thread. */ -int -vm_thread_new(struct thread *td, int pages) +static vm_offset_t +vm_thread_stack_create(struct domainset *ds, vm_object_t *ksobjp, int pages) { + vm_page_t ma[KSTACK_MAX_PAGES]; vm_object_t ksobj; vm_offset_t ks; - vm_page_t ma[KSTACK_MAX_PAGES]; - struct kstack_cache_entry *ks_ce; int i; - /* Bounds check */ - if (pages <= 1) - pages = kstack_pages; - else if (pages > KSTACK_MAX_PAGES) - pages = KSTACK_MAX_PAGES; - - if (pages == kstack_pages && kstack_cache != NULL) { - mtx_lock(&kstack_cache_mtx); - if (kstack_cache != NULL) { - ks_ce = kstack_cache; - kstack_cache = ks_ce->next_ks_entry; - mtx_unlock(&kstack_cache_mtx); - - td->td_kstack_obj = ks_ce->ksobj; - td->td_kstack = (vm_offset_t)ks_ce; - td->td_kstack_pages = kstack_pages; - return (1); - } - mtx_unlock(&kstack_cache_mtx); - } - /* * Allocate an object for the kstack. */ @@ -354,30 +341,17 @@ vm_object_deallocate(ksobj); return (0); } - - /* - * Ensure that kstack objects can draw pages from any memory - * domain. Otherwise a local memory shortage can block a process - * swap-in. - */ if (vm_ndomains > 1) { - ksobj->domain.dr_policy = DOMAINSET_RR(); + ksobj->domain.dr_policy = ds; ksobj->domain.dr_iter = atomic_fetchadd_int(&kstack_domain_iter, 1); } - atomic_add_int(&kstacks, 1); if (KSTACK_GUARD_PAGES != 0) { pmap_qremove(ks, KSTACK_GUARD_PAGES); ks += KSTACK_GUARD_PAGES * PAGE_SIZE; } - td->td_kstack_obj = ksobj; - td->td_kstack = ks; - /* - * Knowing the number of pages allocated is useful when you - * want to deallocate them. - */ - td->td_kstack_pages = pages; + /* * For the length of the stack, link in a real page of ram for each * page of stack. @@ -389,7 +363,9 @@ ma[i]->valid = VM_PAGE_BITS_ALL; VM_OBJECT_WUNLOCK(ksobj); pmap_qenter(ks, ma, pages); - return (1); + *ksobjp = ksobj; + + return (ks); } static void @@ -398,7 +374,6 @@ vm_page_t m; int i; - atomic_add_int(&kstacks, -1); pmap_qremove(ks, pages); VM_OBJECT_WLOCK(ksobj); for (i = 0; i < pages; i++) { @@ -416,6 +391,45 @@ (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); } +/* + * Allocate the kernel stack for a new thread. + */ +int +vm_thread_new(struct thread *td, int pages) +{ + vm_object_t ksobj; + vm_offset_t ks; + + /* Bounds check */ + if (pages <= 1) + pages = kstack_pages; + else if (pages > KSTACK_MAX_PAGES) + pages = KSTACK_MAX_PAGES; + + ks = 0; + ksobj = NULL; + if (pages == kstack_pages && kstack_cache != NULL) { + ks = (vm_offset_t)uma_zalloc(kstack_cache, M_NOWAIT); + if (ks != 0) + ksobj = PHYS_TO_VM_PAGE(pmap_kextract(ks))->object; + } + + /* + * Ensure that kstack objects can draw pages from any memory + * domain. Otherwise a local memory shortage can block a process + * swap-in. + */ + if (ks == 0) + ks = vm_thread_stack_create(DOMAINSET_PREF(PCPU_GET(domain)), + &ksobj, pages); + if (ks == 0) + return (0); + td->td_kstack_obj = ksobj; + td->td_kstack = ks; + td->td_kstack_pages = pages; + return (1); +} + /* * Dispose of a thread's kernel stack. */ @@ -424,7 +438,6 @@ { vm_object_t ksobj; vm_offset_t ks; - struct kstack_cache_entry *ks_ce; int pages; pages = td->td_kstack_pages; @@ -432,43 +445,49 @@ ks = td->td_kstack; td->td_kstack = 0; td->td_kstack_pages = 0; - if (pages == kstack_pages && kstacks <= kstack_cache_size) { - ks_ce = (struct kstack_cache_entry *)ks; - ks_ce->ksobj = ksobj; - mtx_lock(&kstack_cache_mtx); - ks_ce->next_ks_entry = kstack_cache; - kstack_cache = ks_ce; - mtx_unlock(&kstack_cache_mtx); - return; - } - vm_thread_stack_dispose(ksobj, ks, pages); + if (pages == kstack_pages) + uma_zfree(kstack_cache, (void *)ks); + else + vm_thread_stack_dispose(ksobj, ks, pages); } -static void -vm_thread_stack_lowmem(void *nulll) +static int +kstack_import(void *arg, void **store, int cnt, int domain, int flags) { - struct kstack_cache_entry *ks_ce, *ks_ce1; + vm_object_t ksobj; + int i; - mtx_lock(&kstack_cache_mtx); - ks_ce = kstack_cache; - kstack_cache = NULL; - mtx_unlock(&kstack_cache_mtx); + for (i = 0; i < cnt; i++) { + store[i] = (void *)vm_thread_stack_create( + DOMAINSET_PREF(domain), &ksobj, kstack_pages); + if (store[i] == NULL) + break; + } + return (i); +} - while (ks_ce != NULL) { - ks_ce1 = ks_ce; - ks_ce = ks_ce->next_ks_entry; +static void +kstack_release(void *arg, void **store, int cnt) +{ + vm_offset_t ks; + int i; - vm_thread_stack_dispose(ks_ce1->ksobj, (vm_offset_t)ks_ce1, - kstack_pages); + for (i = 0; i < cnt; i++) { + ks = (vm_offset_t)store[i]; + vm_thread_stack_dispose( + PHYS_TO_VM_PAGE(pmap_kextract(ks))->object, + ks, kstack_pages); } } static void -kstack_cache_init(void *nulll) +kstack_cache_init(void *null) { - - EVENTHANDLER_REGISTER(vm_lowmem, vm_thread_stack_lowmem, NULL, - EVENTHANDLER_PRI_ANY); + kstack_cache = uma_zcache_create("kstack_cache", + kstack_pages * PAGE_SIZE, NULL, NULL, NULL, NULL, + kstack_import, kstack_release, NULL, + UMA_ZONE_NUMA|UMA_ZONE_MINBUCKET); + uma_zone_set_maxcache(kstack_cache, kstack_cache_size); } SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL); Index: sys/vm/vm_param.h =================================================================== --- sys/vm/vm_param.h +++ sys/vm/vm_param.h @@ -122,6 +122,10 @@ #endif /* !SMP */ #endif /* !PA_LOCK_COUNT */ +#ifndef KSTACK_MAX_PAGES +#define KSTACK_MAX_PAGES 32 +#endif + #ifndef ASSEMBLER #ifdef _KERNEL #define num_pages(x) \ Index: sys/vm/vm_swapout.c =================================================================== --- sys/vm/vm_swapout.c +++ sys/vm/vm_swapout.c @@ -85,7 +85,6 @@ #include #include #include -#include #include #include #include