Changeset View
Changeset View
Standalone View
Standalone View
sys/vm/vm_glue.c
Show First 20 Lines • Show All 306 Lines • ▼ Show 20 Lines | SYSCTL_INT(_vm, OID_AUTO, kstack_cache_size, CTLFLAG_RW, &kstack_cache_size, 0, | ||||
""); | ""); | ||||
SYSCTL_INT(_vm, OID_AUTO, kstacks, CTLFLAG_RD, &kstacks, 0, | SYSCTL_INT(_vm, OID_AUTO, kstacks, CTLFLAG_RD, &kstacks, 0, | ||||
""); | ""); | ||||
#ifndef KSTACK_MAX_PAGES | #ifndef KSTACK_MAX_PAGES | ||||
#define KSTACK_MAX_PAGES 32 | #define KSTACK_MAX_PAGES 32 | ||||
#endif | #endif | ||||
#if defined(__mips__) | |||||
jmallett: This is a good change to make stack allocation a bit more semantic, flexible, clear, etc. It's… | |||||
static vm_offset_t | |||||
vm_kstack_valloc(int pages) | |||||
{ | |||||
vm_offset_t ks; | |||||
/* | /* | ||||
* We need to align the kstack's mapped address to fit within | |||||
* a single TLB entry. | |||||
*/ | |||||
if (vmem_xalloc(kernel_arena, | |||||
(pages + KSTACK_GUARD_PAGES) * PAGE_SIZE, | |||||
KSTACK_PAGE_SIZE * 2, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, | |||||
M_BESTFIT | M_NOWAIT, &ks)) { | |||||
return (0); | |||||
} | |||||
return (ks); | |||||
} | |||||
#ifdef KSTACK_LARGE_PAGE | |||||
#define KSTACK_OBJT OBJT_PHYS | |||||
static int | |||||
vm_kstack_palloc(vm_object_t ksobj, vm_offset_t ks, int allocflags, int pages, | |||||
vm_page_t ma[]) | |||||
{ | |||||
vm_page_t m, end_m; | |||||
int i; | |||||
KASSERT((ksobj != NULL), ("vm_kstack_palloc: invalid VM object")); | |||||
VM_OBJECT_ASSERT_WLOCKED(ksobj); | |||||
allocflags = (allocflags & ~VM_ALLOC_CLASS_MASK) | VM_ALLOC_NORMAL; | |||||
for (i = 0; i < pages; i++) { | |||||
retrylookup: | |||||
if ((m = vm_page_lookup(ksobj, i)) == NULL) | |||||
break; | |||||
if (vm_page_busied(m)) { | |||||
/* | |||||
* Reference the page before unlocking and | |||||
* sleeping so that the page daemon is less | |||||
* likely to reclaim it. | |||||
*/ | |||||
vm_page_aflag_set(m, PGA_REFERENCED); | |||||
vm_page_lock(m); | |||||
VM_OBJECT_WUNLOCK(ksobj); | |||||
vm_page_busy_sleep(m, "pgrbwt"); | |||||
VM_OBJECT_WLOCK(ksobj); | |||||
goto retrylookup; | |||||
} else { | |||||
if ((allocflags & VM_ALLOC_WIRED) != 0) { | |||||
vm_page_lock(m); | |||||
vm_page_wire(m); | |||||
vm_page_unlock(m); | |||||
} | |||||
ma[i] = m; | |||||
} | |||||
} | |||||
if (i == pages) | |||||
return (i); | |||||
KASSERT((i == 0), ("vm_kstack_palloc: ksobj already has kstack pages")); | |||||
for (;;) { | |||||
m = vm_page_alloc_contig(ksobj, 0, allocflags, | |||||
atop(KSTACK_PAGE_SIZE), 0ul, ~0ul, KSTACK_PAGE_SIZE * 2, 0, | |||||
VM_MEMATTR_DEFAULT); | |||||
if (m != NULL) | |||||
break; | |||||
VM_OBJECT_WUNLOCK(ksobj); | |||||
VM_WAIT; | |||||
VM_OBJECT_WLOCK(ksobj); | |||||
} | |||||
end_m = m + atop(KSTACK_PAGE_SIZE); | |||||
for (i = 0; m < end_m; m++) { | |||||
m->pindex = (vm_pindex_t)i; | |||||
if ((allocflags & VM_ALLOC_NOBUSY) != 0) | |||||
m->valid = VM_PAGE_BITS_ALL; | |||||
ma[i] = m; | |||||
i++; | |||||
} | |||||
return (i); | |||||
} | |||||
#else /* ! KSTACK_LARGE_PAGE */ | |||||
#define KSTACK_OBJT OBJT_DEFAULT | |||||
static int | |||||
vm_kstack_palloc(vm_object_t ksobj, vm_offset_t ks, int allocflags, int pages, | |||||
vm_page_t ma[]) | |||||
{ | |||||
int i; | |||||
KASSERT((ksobj != NULL), ("vm_kstack_palloc: invalid VM object")); | |||||
VM_OBJECT_ASSERT_WLOCKED(ksobj); | |||||
allocflags = (allocflags & ~VM_ALLOC_CLASS_MASK) | VM_ALLOC_NORMAL; | |||||
for (i = 0; i < pages; i++) { | |||||
/* | |||||
* Get a kernel stack page. | |||||
*/ | |||||
ma[i] = vm_page_grab(ksobj, i, allocflags); | |||||
if (allocflags & VM_ALLOC_NOBUSY) | |||||
ma[i]->valid = VM_PAGE_BITS_ALL; | |||||
} | |||||
return (i); | |||||
} | |||||
#endif /* ! KSTACK_LARGE_PAGE */ | |||||
#else /* ! __mips__ */ | |||||
#define KSTACK_OBJT OBJT_DEFAULT | |||||
static vm_offset_t | |||||
vm_kstack_valloc(int pages) | |||||
{ | |||||
vm_offset_t ks; | |||||
ks = kva_alloc((pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); | |||||
return(ks); | |||||
} | |||||
static int | |||||
vm_kstack_palloc(vm_object_t ksobj, vm_offset_t ks, int allocflags, int pages, | |||||
vm_page_t ma[]) | |||||
{ | |||||
int i; | |||||
KASSERT((ksobj != NULL), ("vm_kstack_palloc: invalid VM object")); | |||||
VM_OBJECT_ASSERT_WLOCKED(ksobj); | |||||
allocflags = (allocflags & ~VM_ALLOC_CLASS_MASK) | VM_ALLOC_NORMAL; | |||||
for (i = 0; i < pages; i++) { | |||||
/* | |||||
* Get a kernel stack page. | |||||
*/ | |||||
ma[i] = vm_page_grab(ksobj, i, allocflags); | |||||
if (allocflags & VM_ALLOC_NOBUSY) | |||||
ma[i]->valid = VM_PAGE_BITS_ALL; | |||||
} | |||||
return (i); | |||||
} | |||||
#endif /* ! __mips__ */ | |||||
/* | |||||
* Create the kernel stack (including pcb for i386) for a new thread. | * Create the kernel stack (including pcb for i386) for a new thread. | ||||
* This routine directly affects the fork perf for a process and | * This routine directly affects the fork perf for a process and | ||||
* create performance for a thread. | * create performance for a thread. | ||||
*/ | */ | ||||
int | int | ||||
vm_thread_new(struct thread *td, int pages) | vm_thread_new(struct thread *td, int pages) | ||||
{ | { | ||||
vm_object_t ksobj; | vm_object_t ksobj; | ||||
vm_offset_t ks; | vm_offset_t ks; | ||||
vm_page_t m, ma[KSTACK_MAX_PAGES]; | vm_page_t ma[KSTACK_MAX_PAGES]; | ||||
struct kstack_cache_entry *ks_ce; | struct kstack_cache_entry *ks_ce; | ||||
int i; | |||||
/* Bounds check */ | /* Bounds check */ | ||||
if (pages <= 1) | if (pages <= 1) | ||||
pages = kstack_pages; | pages = kstack_pages; | ||||
else if (pages > KSTACK_MAX_PAGES) | else if (pages > KSTACK_MAX_PAGES) | ||||
pages = KSTACK_MAX_PAGES; | pages = KSTACK_MAX_PAGES; | ||||
if (pages == kstack_pages) { | if (pages == kstack_pages) { | ||||
Show All 9 Lines | if (kstack_cache != NULL) { | ||||
return (1); | return (1); | ||||
} | } | ||||
mtx_unlock(&kstack_cache_mtx); | mtx_unlock(&kstack_cache_mtx); | ||||
} | } | ||||
/* | /* | ||||
* Allocate an object for the kstack. | * Allocate an object for the kstack. | ||||
*/ | */ | ||||
ksobj = vm_object_allocate(OBJT_DEFAULT, pages); | ksobj = vm_object_allocate(KSTACK_OBJT, pages); | ||||
/* | /* | ||||
* Get a kernel virtual address for this thread's kstack. | * Get a kernel virtual address for this thread's kstack. | ||||
*/ | */ | ||||
#if defined(__mips__) | ks = vm_kstack_valloc(pages); | ||||
/* | |||||
* We need to align the kstack's mapped address to fit within | |||||
* a single TLB entry. | |||||
*/ | |||||
if (vmem_xalloc(kernel_arena, (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE, | |||||
PAGE_SIZE * 2, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, | |||||
M_BESTFIT | M_NOWAIT, &ks)) { | |||||
ks = 0; | |||||
} | |||||
#else | |||||
ks = kva_alloc((pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); | |||||
#endif | |||||
if (ks == 0) { | if (ks == 0) { | ||||
printf("vm_thread_new: kstack allocation failed\n"); | printf("vm_thread_new: kstack allocation failed\n"); | ||||
vm_object_deallocate(ksobj); | vm_object_deallocate(ksobj); | ||||
return (0); | return (0); | ||||
} | } | ||||
atomic_add_int(&kstacks, 1); | atomic_add_int(&kstacks, 1); | ||||
if (KSTACK_GUARD_PAGES != 0) { | if (KSTACK_GUARD_PAGES != 0) { | ||||
pmap_qremove(ks, KSTACK_GUARD_PAGES); | pmap_qremove(ks, KSTACK_GUARD_PAGES); | ||||
ks += KSTACK_GUARD_PAGES * PAGE_SIZE; | ks += KSTACK_GUARD_PAGES * PAGE_SIZE; | ||||
} | } | ||||
td->td_kstack_obj = ksobj; | td->td_kstack_obj = ksobj; | ||||
td->td_kstack = ks; | td->td_kstack = ks; | ||||
/* | /* | ||||
* Knowing the number of pages allocated is useful when you | * Knowing the number of pages allocated is useful when you | ||||
* want to deallocate them. | * want to deallocate them. | ||||
*/ | */ | ||||
td->td_kstack_pages = pages; | td->td_kstack_pages = pages; | ||||
/* | |||||
* For the length of the stack, link in a real page of ram for each | |||||
* page of stack. | |||||
*/ | |||||
VM_OBJECT_WLOCK(ksobj); | VM_OBJECT_WLOCK(ksobj); | ||||
for (i = 0; i < pages; i++) { | pages = vm_kstack_palloc(ksobj, ks, (VM_ALLOC_NOBUSY | VM_ALLOC_WIRED), | ||||
/* | pages, ma); | ||||
* Get a kernel stack page. | |||||
*/ | |||||
m = vm_page_grab(ksobj, i, VM_ALLOC_NOBUSY | | |||||
VM_ALLOC_NORMAL | VM_ALLOC_WIRED); | |||||
ma[i] = m; | |||||
m->valid = VM_PAGE_BITS_ALL; | |||||
} | |||||
VM_OBJECT_WUNLOCK(ksobj); | VM_OBJECT_WUNLOCK(ksobj); | ||||
if (pages == 0) { | |||||
printf("vm_thread_new: vm_kstack_palloc() failed\n"); | |||||
return (0); | |||||
} | |||||
pmap_qenter(ks, ma, pages); | pmap_qenter(ks, ma, pages); | ||||
return (1); | return (1); | ||||
} | } | ||||
static void | static void | ||||
vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages) | vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages) | ||||
{ | { | ||||
vm_page_t m; | vm_page_t m; | ||||
▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | |||||
/* | /* | ||||
* Bring the kernel stack for a specified thread back in. | * Bring the kernel stack for a specified thread back in. | ||||
*/ | */ | ||||
static void | static void | ||||
vm_thread_swapin(struct thread *td) | vm_thread_swapin(struct thread *td) | ||||
{ | { | ||||
vm_object_t ksobj; | vm_object_t ksobj; | ||||
vm_page_t ma[KSTACK_MAX_PAGES]; | vm_page_t ma[KSTACK_MAX_PAGES]; | ||||
int pages; | int pages, rv; | ||||
pages = td->td_kstack_pages; | pages = td->td_kstack_pages; | ||||
ksobj = td->td_kstack_obj; | ksobj = td->td_kstack_obj; | ||||
VM_OBJECT_WLOCK(ksobj); | VM_OBJECT_WLOCK(ksobj); | ||||
for (int i = 0; i < pages; i++) | rv = vm_kstack_palloc(ksobj, td->td_kstack, (VM_ALLOC_NORMAL | | ||||
ma[i] = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | | VM_ALLOC_WIRED), pages, ma); | ||||
VM_ALLOC_WIRED); | KASSERT(rv != 0, ("vm_thread_swapin: vm_kstack_palloc() failed")); | ||||
for (int i = 0; i < pages;) { | for (int i = 0; i < pages;) { | ||||
int j, a, count, rv; | int j, a, count, rv; | ||||
vm_page_assert_xbusied(ma[i]); | vm_page_assert_xbusied(ma[i]); | ||||
if (ma[i]->valid == VM_PAGE_BITS_ALL) { | if (ma[i]->valid == VM_PAGE_BITS_ALL) { | ||||
vm_page_xunbusy(ma[i]); | vm_page_xunbusy(ma[i]); | ||||
i++; | i++; | ||||
continue; | continue; | ||||
▲ Show 20 Lines • Show All 517 Lines • Show Last 20 Lines |
This is a good change to make stack allocation a bit more semantic, flexible, clear, etc. It's also very self-contained, and I'd suggest it be reviewed and committed separately.