Differential D27185 Diff 79745 head/sys/kern/kern_thread.c

Changeset View

Standalone View

head/sys/kern/kern_thread.c

Show First 20 Lines • Show All 46 Lines • ▼ Show 20 Lines
#include <sys/sdt.h>		#include <sys/sdt.h>
#include <sys/smp.h>		#include <sys/smp.h>
#include <sys/sched.h>		#include <sys/sched.h>
#include <sys/sleepqueue.h>		#include <sys/sleepqueue.h>
#include <sys/selinfo.h>		#include <sys/selinfo.h>
#include <sys/syscallsubr.h>		#include <sys/syscallsubr.h>
#include <sys/sysent.h>		#include <sys/sysent.h>
#include <sys/turnstile.h>		#include <sys/turnstile.h>
		#include <sys/taskqueue.h>
#include <sys/ktr.h>		#include <sys/ktr.h>
#include <sys/rwlock.h>		#include <sys/rwlock.h>
#include <sys/umtx.h>		#include <sys/umtx.h>
#include <sys/vmmeter.h>		#include <sys/vmmeter.h>
#include <sys/cpuset.h>		#include <sys/cpuset.h>
#ifdef HWPMC_HOOKS		#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>		#include <sys/pmckern.h>
#endif		#endif
#include <sys/priv.h>		#include <sys/priv.h>

#include <security/audit/audit.h>		#include <security/audit/audit.h>

		#include <vm/pmap.h>
#include <vm/vm.h>		#include <vm/vm.h>
#include <vm/vm_extern.h>		#include <vm/vm_extern.h>
#include <vm/uma.h>		#include <vm/uma.h>
		#include <vm/vm_phys.h>
#include <sys/eventhandler.h>		#include <sys/eventhandler.h>

/*		/*
* Asserts below verify the stability of struct thread and struct proc		* Asserts below verify the stability of struct thread and struct proc
* layout, as exposed by KBI to modules. On head, the KBI is allowed		* layout, as exposed by KBI to modules. On head, the KBI is allowed
* to drift, change to the structures must be accompanied by the		* to drift, change to the structures must be accompanied by the
* assert update.		* assert update.
*		*
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines
SDT_PROVIDER_DECLARE(proc);		SDT_PROVIDER_DECLARE(proc);
SDT_PROBE_DEFINE(proc, , , lwp__exit);		SDT_PROBE_DEFINE(proc, , , lwp__exit);

/*		/*
* thread related storage.		* thread related storage.
*/		*/
static uma_zone_t thread_zone;		static uma_zone_t thread_zone;

static __exclusive_cache_line struct thread *thread_zombies;		struct thread_domain_data {
		struct thread *tdd_zombies;
		int tdd_reapticks;
		} __aligned(CACHE_LINE_SIZE);

		static struct thread_domain_data thread_domain_data[MAXMEMDOM];

		static struct task thread_reap_task;
		static struct callout thread_reap_callout;

static void thread_zombie(struct thread *);		static void thread_zombie(struct thread *);
		static void thread_reap_all(void);
		static void thread_reap_task_cb(void *, int);
		static void thread_reap_callout_cb(void *);
static int thread_unsuspend_one(struct thread td, struct proc p,		static int thread_unsuspend_one(struct thread td, struct proc p,
bool boundary);		bool boundary);
static void thread_free_batched(struct thread *td);		static void thread_free_batched(struct thread *td);

static __exclusive_cache_line struct mtx tid_lock;		static __exclusive_cache_line struct mtx tid_lock;
static bitstr_t *tid_bitmap;		static bitstr_t *tid_bitmap;

static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash");		static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash");
Show All 12 Lines
#define TIDHASHLOCK(tid) (&tidhashtbl_lock[(tid) & tidhashlock])		#define TIDHASHLOCK(tid) (&tidhashtbl_lock[(tid) & tidhashlock])

EVENTHANDLER_LIST_DEFINE(thread_ctor);		EVENTHANDLER_LIST_DEFINE(thread_ctor);
EVENTHANDLER_LIST_DEFINE(thread_dtor);		EVENTHANDLER_LIST_DEFINE(thread_dtor);
EVENTHANDLER_LIST_DEFINE(thread_init);		EVENTHANDLER_LIST_DEFINE(thread_init);
EVENTHANDLER_LIST_DEFINE(thread_fini);		EVENTHANDLER_LIST_DEFINE(thread_fini);

static bool		static bool
thread_count_inc(void)		thread_count_inc_try(void)
{		{
static struct timeval lastfail;
static int curfail;
int nthreads_new;		int nthreads_new;

thread_reap();

nthreads_new = atomic_fetchadd_int(&nthreads, 1) + 1;		nthreads_new = atomic_fetchadd_int(&nthreads, 1) + 1;
if (nthreads_new >= maxthread - 100) {		if (nthreads_new >= maxthread - 100) {
if (priv_check_cred(curthread->td_ucred, PRIV_MAXPROC) != 0 \|\|		if (priv_check_cred(curthread->td_ucred, PRIV_MAXPROC) != 0 \|\|
nthreads_new >= maxthread) {		nthreads_new >= maxthread) {
atomic_subtract_int(&nthreads, 1);		atomic_subtract_int(&nthreads, 1);
		return (false);
		}
		}
		return (true);
		}

		static bool
		thread_count_inc(void)
		{
		static struct timeval lastfail;
		static int curfail;

		thread_reap();
		if (thread_count_inc_try()) {
		return (true);
		}

		thread_reap_all();
		if (thread_count_inc_try()) {
		return (true);
		}

if (ppsratecheck(&lastfail, &curfail, 1)) {		if (ppsratecheck(&lastfail, &curfail, 1)) {
printf("maxthread limit exceeded by uid %u "		printf("maxthread limit exceeded by uid %u "
"(pid %d); consider increasing kern.maxthread\n",		"(pid %d); consider increasing kern.maxthread\n",
curthread->td_ucred->cr_ruid, curproc->p_pid);		curthread->td_ucred->cr_ruid, curproc->p_pid);
}		}
return (false);		return (false);
}		}
}
return (true);
}

static void		static void
thread_count_sub(int n)		thread_count_sub(int n)
{		{

atomic_subtract_int(&nthreads, n);		atomic_subtract_int(&nthreads, n);
}		}

▲ Show 20 Lines • Show All 302 Lines • ▼ Show 20 Lines	#endif
tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash);		tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash);
tidhashlock = (tidhash + 1) / 64;		tidhashlock = (tidhash + 1) / 64;
if (tidhashlock > 0)		if (tidhashlock > 0)
tidhashlock--;		tidhashlock--;
tidhashtbl_lock = malloc(sizeof(tidhashtbl_lock) (tidhashlock + 1),		tidhashtbl_lock = malloc(sizeof(tidhashtbl_lock) (tidhashlock + 1),
M_TIDHASH, M_WAITOK \| M_ZERO);		M_TIDHASH, M_WAITOK \| M_ZERO);
for (i = 0; i < tidhashlock + 1; i++)		for (i = 0; i < tidhashlock + 1; i++)
rw_init(&tidhashtbl_lock[i], "tidhash");		rw_init(&tidhashtbl_lock[i], "tidhash");

		TASK_INIT(&thread_reap_task, 0, thread_reap_task_cb, NULL);
		callout_init(&thread_reap_callout, 1);
		callout_reset(&thread_reap_callout, 5 * hz, thread_reap_callout_cb, NULL);
}		}

/*		/*
* Place an unused thread on the zombie list.		* Place an unused thread on the zombie list.
*/		*/
void		void
thread_zombie(struct thread *td)		thread_zombie(struct thread *td)
{		{
		struct thread_domain_data *tdd;
struct thread *ztd;		struct thread *ztd;

ztd = atomic_load_ptr(&thread_zombies);		tdd = &thread_domain_data[vm_phys_domain(vtophys(td))];
		ztd = atomic_load_ptr(&tdd->tdd_zombies);
for (;;) {		for (;;) {
td->td_zombie = ztd;		td->td_zombie = ztd;
if (atomic_fcmpset_rel_ptr((uintptr_t *)&thread_zombies,		if (atomic_fcmpset_rel_ptr((uintptr_t *)&tdd->tdd_zombies,
(uintptr_t *)&ztd, (uintptr_t)td))		(uintptr_t *)&ztd, (uintptr_t)td))
break;		break;
continue;		continue;
}		}
}		}

/*		/*
* Release a thread that has exited after cpu_throw().		* Release a thread that has exited after cpu_throw().
*/		*/
void		void
thread_stash(struct thread *td)		thread_stash(struct thread *td)
{		{
atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);		atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1);
thread_zombie(td);		thread_zombie(td);
}		}

/*		/*
* Reap zombie threads.		* Reap zombies from passed domain.
*/		*/
void		static void
thread_reap(void)		thread_reap_domain(struct thread_domain_data *tdd)
{		{
struct thread itd, ntd;		struct thread itd, ntd;
struct tidbatch tidbatch;		struct tidbatch tidbatch;
struct credbatch credbatch;		struct credbatch credbatch;
int tdcount;		int tdcount;
struct plimit *lim;		struct plimit *lim;
int limcount;		int limcount;

/*		/*
* Reading upfront is pessimal if followed by concurrent atomic_swap,		* Reading upfront is pessimal if followed by concurrent atomic_swap,
* but most of the time the list is empty.		* but most of the time the list is empty.
*/		*/
if (thread_zombies == NULL)		if (tdd->tdd_zombies == NULL)
return;		return;

itd = (struct thread )atomic_swap_ptr((uintptr_t )&thread_zombies,		itd = (struct thread )atomic_swap_ptr((uintptr_t )&tdd->tdd_zombies,
(uintptr_t)NULL);		(uintptr_t)NULL);
if (itd == NULL)		if (itd == NULL)
return;		return;

		/*
		* Multiple CPUs can get here, the race is fine as ticks is only
		* advisory.
		*/
		tdd->tdd_reapticks = ticks;

tidbatch_prep(&tidbatch);		tidbatch_prep(&tidbatch);
credbatch_prep(&credbatch);		credbatch_prep(&credbatch);
tdcount = 0;		tdcount = 0;
lim = NULL;		lim = NULL;
limcount = 0;		limcount = 0;

while (itd != NULL) {		while (itd != NULL) {
ntd = itd->td_zombie;		ntd = itd->td_zombie;
EVENTHANDLER_DIRECT_INVOKE(thread_dtor, itd);		EVENTHANDLER_DIRECT_INVOKE(thread_dtor, itd);
tidbatch_add(&tidbatch, itd);		tidbatch_add(&tidbatch, itd);
credbatch_add(&credbatch, itd);		credbatch_add(&credbatch, itd);
MPASS(itd->td_limit != NULL);		MPASS(itd->td_limit != NULL);
if (lim != itd->td_limit) {		if (lim != itd->td_limit) {
if (limcount != 0) {		if (limcount != 0) {
Show All 16 Lines	thread_reap_domain(struct thread_domain_data *tdd)

tidbatch_final(&tidbatch);		tidbatch_final(&tidbatch);
credbatch_final(&credbatch);		credbatch_final(&credbatch);
if (tdcount != 0) {		if (tdcount != 0) {
thread_count_sub(tdcount);		thread_count_sub(tdcount);
}		}
MPASS(limcount != 0);		MPASS(limcount != 0);
lim_freen(lim, limcount);		lim_freen(lim, limcount);
		}

		/*
		* Reap zombies from all domains.
		*/
		static void
		thread_reap_all(void)
		{
		struct thread_domain_data *tdd;
		int i, domain;

		domain = PCPU_GET(domain);
		for (i = 0; i < vm_ndomains; i++) {
		tdd = &thread_domain_data[(i + domain) % vm_ndomains];
		thread_reap_domain(tdd);
		}
		}

		/*
		* Reap zombies from local domain.
		*/
		void
		thread_reap(void)
		{
		struct thread_domain_data *tdd;
		int domain;

		domain = PCPU_GET(domain);
		tdd = &thread_domain_data[domain];

		thread_reap_domain(tdd);
		}

		static void
		thread_reap_task_cb(void *arg __unused, int pending __unused)
		{

		thread_reap_all();
		}

		static void
		thread_reap_callout_cb(void *arg __unused)
		{
		struct thread_domain_data *tdd;
		int i, cticks, lticks;
		bool wantreap;

		wantreap = false;
		cticks = atomic_load_int(&ticks);
		for (i = 0; i < vm_ndomains; i++) {
		tdd = &thread_domain_data[i];
		lticks = tdd->tdd_reapticks;
		if (tdd->tdd_zombies != NULL &&
		(u_int)(cticks - lticks) > 5 * hz) {
		wantreap = true;
		break;
		}
		}

		if (wantreap)
		taskqueue_enqueue(taskqueue_thread, &thread_reap_task);
		callout_reset(&thread_reap_callout, 5 * hz, thread_reap_callout_cb, NULL);
}		}

/*		/*
* Allocate a thread.		* Allocate a thread.
*/		*/
struct thread *		struct thread *
thread_alloc(int pages)		thread_alloc(int pages)
{		{
▲ Show 20 Lines • Show All 993 Lines • Show Last 20 Lines