Page MenuHomeFreeBSD
Files F150434689

D24473.id70803.diff
No OneTemporary
Actions

D24473.id70803.diff
View Options

Index: sys/kern/kern_thread.c
===================================================================
--- sys/kern/kern_thread.c
+++ sys/kern/kern_thread.c
@@ -84,7 +84,7 @@
"struct thread KBI td_pflags");
_Static_assert(offsetof(struct thread, td_frame) == 0x498,
"struct thread KBI td_frame");
-_Static_assert(offsetof(struct thread, td_emuldata) == 0x6b0,
+_Static_assert(offsetof(struct thread, td_emuldata) == 0x6a0,
"struct thread KBI td_emuldata");
_Static_assert(offsetof(struct proc, p_flag) == 0xb0,
"struct proc KBI p_flag");
Index: sys/sys/proc.h
===================================================================
--- sys/sys/proc.h
+++ sys/sys/proc.h
@@ -348,7 +348,6 @@
/* LP64 hole */
struct callout td_slpcallout; /* (h) Callout for sleep. */
struct trapframe *td_frame; /* (k) */
- struct vm_object *td_kstack_obj;/* (a) Kstack object. */
vm_offset_t td_kstack; /* (a) Kernel VA of kstack. */
int td_kstack_pages; /* (a) Size of the kstack. */
volatile u_int td_critnest; /* (k*) Critical section nest level. */
Index: sys/vm/vm_extern.h
===================================================================
--- sys/vm/vm_extern.h
+++ sys/vm/vm_extern.h
@@ -126,6 +126,8 @@
void vm_imgact_unmap_page(struct sf_buf *sf);
void vm_thread_dispose(struct thread *td);
int vm_thread_new(struct thread *td, int pages);
+void vm_thread_stack_back(struct domainset *ds, vm_offset_t kaddr,
+ vm_page_t ma[], int npages, int req_class);
u_int vm_active_count(void);
u_int vm_inactive_count(void);
u_int vm_laundry_count(void);
Index: sys/vm/vm_glue.c
===================================================================
--- sys/vm/vm_glue.c
+++ sys/vm/vm_glue.c
@@ -264,9 +264,9 @@
pmap_sync_icache(map->pmap, va, sz);
}
+vm_object_t kstack_object;
static uma_zone_t kstack_cache;
static int kstack_cache_size;
-static int kstack_domain_iter;
static int
sysctl_kstack_cache_size(SYSCTL_HANDLER_ARGS)
@@ -285,22 +285,14 @@
/*
* Create the kernel stack (including pcb for i386) for a new thread.
- * This routine directly affects the fork perf for a process and
- * create performance for a thread.
*/
static vm_offset_t
-vm_thread_stack_create(struct domainset *ds, vm_object_t *ksobjp, int pages)
+vm_thread_stack_create(struct domainset *ds, int pages)
{
vm_page_t ma[KSTACK_MAX_PAGES];
- vm_object_t ksobj;
vm_offset_t ks;
int i;
- /*
- * Allocate an object for the kstack.
- */
- ksobj = vm_object_allocate(OBJT_DEFAULT, pages);
-
/*
* Get a kernel virtual address for this thread's kstack.
*/
@@ -319,54 +311,43 @@
#endif
if (ks == 0) {
printf("%s: kstack allocation failed\n", __func__);
- vm_object_deallocate(ksobj);
return (0);
}
- if (vm_ndomains > 1) {
- ksobj->domain.dr_policy = ds;
- ksobj->domain.dr_iter =
- atomic_fetchadd_int(&kstack_domain_iter, 1);
- }
- if (KSTACK_GUARD_PAGES != 0) {
- pmap_qremove(ks, KSTACK_GUARD_PAGES);
- ks += KSTACK_GUARD_PAGES * PAGE_SIZE;
- }
+ for (i = 0; i < KSTACK_GUARD_PAGES; i++)
+ KASSERT(pmap_kextract(ks + i * PAGE_SIZE) == 0,
+ ("%s: guard page at %#lx is mapped",
+ __func__, ks + i * PAGE_SIZE));
+ ks += KSTACK_GUARD_PAGES * PAGE_SIZE;
- /*
- * For the length of the stack, link in a real page of ram for each
- * page of stack.
+ /*
+ * Allocate physical pages to back the stack.
*/
- VM_OBJECT_WLOCK(ksobj);
- (void)vm_page_grab_pages(ksobj, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED,
- ma, pages);
+ vm_thread_stack_back(ds, ks, ma, pages, VM_ALLOC_NORMAL);
for (i = 0; i < pages; i++)
vm_page_valid(ma[i]);
- VM_OBJECT_WUNLOCK(ksobj);
pmap_qenter(ks, ma, pages);
- *ksobjp = ksobj;
return (ks);
}
static void
-vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages)
+vm_thread_stack_dispose(vm_offset_t ks, int pages)
{
vm_page_t m;
int i;
pmap_qremove(ks, pages);
- VM_OBJECT_WLOCK(ksobj);
+ VM_OBJECT_WLOCK(kstack_object);
for (i = 0; i < pages; i++) {
- m = vm_page_lookup(ksobj, i);
+ m = vm_page_lookup(kstack_object, atop(ks) + i);
if (m == NULL)
panic("%s: kstack already missing?", __func__);
vm_page_xbusy_claim(m);
vm_page_unwire_noq(m);
vm_page_free(m);
}
- VM_OBJECT_WUNLOCK(ksobj);
- vm_object_deallocate(ksobj);
+ VM_OBJECT_WUNLOCK(kstack_object);
kva_free(ks - (KSTACK_GUARD_PAGES * PAGE_SIZE),
(pages + KSTACK_GUARD_PAGES) * PAGE_SIZE);
}
@@ -377,7 +358,6 @@
int
vm_thread_new(struct thread *td, int pages)
{
- vm_object_t ksobj;
vm_offset_t ks;
/* Bounds check */
@@ -387,12 +367,8 @@
pages = KSTACK_MAX_PAGES;
ks = 0;
- ksobj = NULL;
- if (pages == kstack_pages && kstack_cache != NULL) {
+ if (pages == kstack_pages && kstack_cache != NULL)
ks = (vm_offset_t)uma_zalloc(kstack_cache, M_NOWAIT);
- if (ks != 0)
- ksobj = PHYS_TO_VM_PAGE(pmap_kextract(ks))->object;
- }
/*
* Ensure that kstack objects can draw pages from any memory
@@ -401,10 +377,9 @@
*/
if (ks == 0)
ks = vm_thread_stack_create(DOMAINSET_PREF(PCPU_GET(domain)),
- &ksobj, pages);
+ pages);
if (ks == 0)
return (0);
- td->td_kstack_obj = ksobj;
td->td_kstack = ks;
td->td_kstack_pages = pages;
return (1);
@@ -416,26 +391,49 @@
void
vm_thread_dispose(struct thread *td)
{
- vm_object_t ksobj;
vm_offset_t ks;
int pages;
pages = td->td_kstack_pages;
- ksobj = td->td_kstack_obj;
ks = td->td_kstack;
td->td_kstack = 0;
td->td_kstack_pages = 0;
if (pages == kstack_pages)
uma_zfree(kstack_cache, (void *)ks);
else
- vm_thread_stack_dispose(ksobj, ks, pages);
+ vm_thread_stack_dispose(ks, pages);
+}
+
+/*
+ * Allocate physical pages, following the specified NUMA policy, to back a
+ * kernel stack.
+ */
+void
+vm_thread_stack_back(struct domainset *ds, vm_offset_t kaddr, vm_page_t ma[],
+ int npages, int req_class)
+{
+ int n;
+
+ VM_OBJECT_WLOCK(kstack_object);
+ for (n = 0; n < npages;) {
+ if (vm_ndomains > 1)
+ kstack_object->domain.dr_policy = ds;
+
+ /*
+ * Use WAITFAIL to force a reset of the domain selection policy
+ * if we had to sleep for pages.
+ */
+ n += vm_page_grab_pages(kstack_object, atop(kaddr) + n,
+ req_class | VM_ALLOC_WIRED | VM_ALLOC_WAITFAIL,
+ &ma[n], npages - n);
+ }
+ VM_OBJECT_WUNLOCK(kstack_object);
}
static int
kstack_import(void *arg, void **store, int cnt, int domain, int flags)
{
struct domainset *ds;
- vm_object_t ksobj;
int i;
if (domain == UMA_ANYDOMAIN)
@@ -444,8 +442,7 @@
ds = DOMAINSET_PREF(domain);
for (i = 0; i < cnt; i++) {
- store[i] = (void *)vm_thread_stack_create(ds, &ksobj,
- kstack_pages);
+ store[i] = (void *)vm_thread_stack_create(ds, kstack_pages);
if (store[i] == NULL)
break;
}
@@ -460,15 +457,15 @@
for (i = 0; i < cnt; i++) {
ks = (vm_offset_t)store[i];
- vm_thread_stack_dispose(
- PHYS_TO_VM_PAGE(pmap_kextract(ks))->object,
- ks, kstack_pages);
+ vm_thread_stack_dispose(ks, kstack_pages);
}
}
static void
kstack_cache_init(void *null)
{
+ kstack_object = vm_object_allocate(OBJT_DEFAULT,
+ atop(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS));
kstack_cache = uma_zcache_create("kstack_cache",
kstack_pages * PAGE_SIZE, NULL, NULL, NULL, NULL,
kstack_import, kstack_release, NULL,
@@ -476,8 +473,7 @@
kstack_cache_size = imax(128, mp_ncpus * 4);
uma_zone_set_maxcache(kstack_cache, kstack_cache_size);
}
-
-SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL);
+SYSINIT(vm_kstacks, SI_SUB_KMEM, SI_ORDER_ANY, kstack_cache_init, NULL);
#ifdef KSTACK_USAGE_PROF
/*
Index: sys/vm/vm_kern.h
===================================================================
--- sys/vm/vm_kern.h
+++ sys/vm/vm_kern.h
@@ -77,5 +77,6 @@
extern u_long vm_kmem_size;
extern u_int exec_map_entries;
extern u_int exec_map_entry_size;
+extern vm_object_t kstack_object;
#endif /* _VM_VM_KERN_H_ */
Index: sys/vm/vm_swapout.c
===================================================================
--- sys/vm/vm_swapout.c
+++ sys/vm/vm_swapout.c
@@ -104,6 +104,7 @@
#include <vm/vm.h>
#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
@@ -526,24 +527,24 @@
static void
vm_thread_swapout(struct thread *td)
{
- vm_object_t ksobj;
vm_page_t m;
+ vm_offset_t kaddr;
int i, pages;
cpu_thread_swapout(td);
+ kaddr = td->td_kstack;
pages = td->td_kstack_pages;
- ksobj = td->td_kstack_obj;
- pmap_qremove(td->td_kstack, pages);
- VM_OBJECT_WLOCK(ksobj);
+ pmap_qremove(kaddr, pages);
+ VM_OBJECT_WLOCK(kstack_object);
for (i = 0; i < pages; i++) {
- m = vm_page_lookup(ksobj, i);
+ m = vm_page_lookup(kstack_object, atop(kaddr) + i);
if (m == NULL)
panic("vm_thread_swapout: kstack already missing?");
vm_page_dirty(m);
vm_page_xunbusy_unchecked(m);
vm_page_unwire(m, PQ_LAUNDRY);
}
- VM_OBJECT_WUNLOCK(ksobj);
+ VM_OBJECT_WUNLOCK(kstack_object);
}
/*
@@ -552,38 +553,36 @@
static void
vm_thread_swapin(struct thread *td, int oom_alloc)
{
- vm_object_t ksobj;
vm_page_t ma[KSTACK_MAX_PAGES];
+ vm_offset_t kaddr;
int a, count, i, j, pages, rv;
+ kaddr = td->td_kstack;
pages = td->td_kstack_pages;
- ksobj = td->td_kstack_obj;
- VM_OBJECT_WLOCK(ksobj);
- (void)vm_page_grab_pages(ksobj, 0, oom_alloc | VM_ALLOC_WIRED, ma,
- pages);
- VM_OBJECT_WUNLOCK(ksobj);
+ vm_thread_stack_back(td->td_domain.dr_policy, kaddr, ma, pages,
+ oom_alloc);
for (i = 0; i < pages;) {
vm_page_assert_xbusied(ma[i]);
if (vm_page_all_valid(ma[i])) {
i++;
continue;
}
- vm_object_pip_add(ksobj, 1);
+ vm_object_pip_add(kstack_object, 1);
for (j = i + 1; j < pages; j++)
if (vm_page_all_valid(ma[j]))
break;
- VM_OBJECT_WLOCK(ksobj);
- rv = vm_pager_has_page(ksobj, ma[i]->pindex, NULL, &a);
- VM_OBJECT_WUNLOCK(ksobj);
+ VM_OBJECT_WLOCK(kstack_object);
+ rv = vm_pager_has_page(kstack_object, ma[i]->pindex, NULL, &a);
+ VM_OBJECT_WUNLOCK(kstack_object);
KASSERT(rv == 1, ("%s: missing page %p", __func__, ma[i]));
count = min(a + 1, j - i);
- rv = vm_pager_get_pages(ksobj, ma + i, count, NULL, NULL);
+ rv = vm_pager_get_pages(kstack_object, ma + i, count, NULL, NULL);
KASSERT(rv == VM_PAGER_OK, ("%s: cannot get kstack for proc %d",
__func__, td->td_proc->p_pid));
- vm_object_pip_wakeup(ksobj);
+ vm_object_pip_wakeup(kstack_object);
i += count;
}
- pmap_qenter(td->td_kstack, ma, pages);
+ pmap_qenter(kaddr, ma, pages);
cpu_thread_swapin(td);
}

File Metadata

Mime Type
text/plain
Expires
Thu, Apr 2, 5:09 AM (5 h, 39 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
30706282
Default Alt Text
D24473.id70803.diff (10 KB)

Event Timeline