diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -40,7 +40,7 @@ #include #include -#include +#include #include #include #include @@ -238,10 +238,7 @@ sysctl_kern_randompid, "I", "Random PID modulus. Special values: 0: disable, 1: choose random value"); -extern bitstr_t proc_id_pidmap; -extern bitstr_t proc_id_grpidmap; -extern bitstr_t proc_id_sessidmap; -extern bitstr_t proc_id_reapmap; +extern struct proc_id_map proc_id_pidmap; /* * Find an unused process ID @@ -253,18 +250,14 @@ fork_findpid(int flags) { pid_t result; - int trypid, random; + int _lastpid, random, trypid; - /* - * Avoid calling arc4random with procid_lock held. - */ random = 0; if (__predict_false(randompid)) random = arc4random() % randompid; - mtx_lock(&procid_lock); - - trypid = lastpid + 1; + _lastpid = lastpid; + trypid = _lastpid; if (flags & RFHIGHPID) { if (trypid < 10) trypid = 10; @@ -275,27 +268,37 @@ if (trypid >= pid_max) trypid = 2; - bit_ffc_at(&proc_id_pidmap, trypid, pid_max, &result); + result = BIT_FFC_AT(pid_max, &proc_id_pidmap, trypid + 1) - 1; if (result == -1) { KASSERT(trypid != 2, ("unexpectedly ran out of IDs")); trypid = 2; goto retry; - } - if (bit_test(&proc_id_grpidmap, result) || - bit_test(&proc_id_sessidmap, result) || - bit_test(&proc_id_reapmap, result)) { - trypid = result + 1; - goto retry; + } else { + /* + * We try to reserve the PID first. If we were able to do that, + * we still need to check if the PID is not set on the other + * bitmaps. We assume that because we were able reserve the PID, + * this bit cannot be set in the other bitmaps anymore. + * It was either set before or it is not set. + */ + if (proc_id_set_cond(PROC_ID_PID, result)) { + /* Lost the race. */ + trypid = result + 1; + goto retry; + } else if (proc_id_test(PROC_ID_GROUP, result) || + proc_id_test(PROC_ID_SESSION, result) || + proc_id_test(PROC_ID_REAP, result)) { + proc_id_clear(PROC_ID_PID, result); + trypid = result + 1; + goto retry; + } } /* * RFHIGHPID does not mess with the lastpid counter during boot. */ if ((flags & RFHIGHPID) == 0) - lastpid = result; - - bit_set(&proc_id_pidmap, result); - mtx_unlock(&procid_lock); + atomic_cmpset_acq_int(&lastpid, _lastpid, result); return (result); } @@ -652,7 +655,7 @@ LIST_INSERT_HEAD(&p2->p_reaper->p_reaplist, p2, p_reapsibling); if (p2->p_reaper == p1 && p1 != initproc) { p2->p_reapsubtree = p2->p_pid; - proc_id_set_cond(PROC_ID_REAP, p2->p_pid); + (void)proc_id_set_cond(PROC_ID_REAP, p2->p_pid); } sx_xunlock(&proctree_lock); diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -37,7 +37,7 @@ #include #include -#include +#include #include #include #include @@ -130,7 +130,6 @@ struct sx __exclusive_cache_line allproc_lock; struct sx __exclusive_cache_line proctree_lock; struct mtx __exclusive_cache_line ppeers_lock; -struct mtx __exclusive_cache_line procid_lock; uma_zone_t proc_zone; uma_zone_t pgrp_zone; @@ -183,7 +182,6 @@ sx_init(&allproc_lock, "allproc"); sx_init(&proctree_lock, "proctree"); mtx_init(&ppeers_lock, "p_peers", NULL, MTX_DEF); - mtx_init(&procid_lock, "procid", NULL, MTX_DEF); pidhashtbl = hashinit(maxproc / 4, M_PROC, &pidhash); pidhashlock = (pidhash + 1) / 64; if (pidhashlock > 0) @@ -317,16 +315,16 @@ * * These bitmaps are used by fork_findpid. */ -bitstr_t bit_decl(proc_id_pidmap, PID_MAX); -bitstr_t bit_decl(proc_id_grpidmap, PID_MAX); -bitstr_t bit_decl(proc_id_sessidmap, PID_MAX); -bitstr_t bit_decl(proc_id_reapmap, PID_MAX); - -static bitstr_t *proc_id_array[] = { - proc_id_pidmap, - proc_id_grpidmap, - proc_id_sessidmap, - proc_id_reapmap, +struct proc_id_map proc_id_pidmap; +struct proc_id_map proc_id_grpidmap; +struct proc_id_map proc_id_sessidmap; +struct proc_id_map proc_id_reapmap; + +static struct proc_id_map *proc_id_array[] = { + &proc_id_pidmap, + &proc_id_grpidmap, + &proc_id_sessidmap, + &proc_id_reapmap, }; void @@ -335,24 +333,18 @@ KASSERT(type >= 0 && type < nitems(proc_id_array), ("invalid type %d\n", type)); - mtx_lock(&procid_lock); - KASSERT(bit_test(proc_id_array[type], id) == 0, - ("bit %d already set in %d\n", id, type)); - bit_set(proc_id_array[type], id); - mtx_unlock(&procid_lock); + if (__predict_false(BIT_TEST_SET_ATOMIC(PID_MAX, id, proc_id_array[type]))) { + panic("%s: bit %d already set in %d", __func__, id, type); + } } -void +bool proc_id_set_cond(int type, pid_t id) { KASSERT(type >= 0 && type < nitems(proc_id_array), ("invalid type %d\n", type)); - if (bit_test(proc_id_array[type], id)) - return; - mtx_lock(&procid_lock); - bit_set(proc_id_array[type], id); - mtx_unlock(&procid_lock); + return (BIT_TEST_SET_ATOMIC(PID_MAX, id, proc_id_array[type])); } void @@ -361,11 +353,18 @@ KASSERT(type >= 0 && type < nitems(proc_id_array), ("invalid type %d\n", type)); - mtx_lock(&procid_lock); - KASSERT(bit_test(proc_id_array[type], id) != 0, - ("bit %d not set in %d\n", id, type)); - bit_clear(proc_id_array[type], id); - mtx_unlock(&procid_lock); + if (__predict_false(!BIT_TEST_CLR_ATOMIC(PID_MAX, id, proc_id_array[type]))) { + panic("%s: bit %d not set in %d", __func__, id, type); + } +} + +bool +proc_id_test(int type, pid_t id) +{ + + KASSERT(type >= 0 && type < nitems(proc_id_array), + ("invalid type %d\n", type)); + return (BIT_ISSET_ATOMIC(PID_MAX, id, proc_id_array[type])); } /* diff --git a/sys/sys/proc.h b/sys/sys/proc.h --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -37,6 +37,7 @@ #ifndef _SYS_PROC_H_ #define _SYS_PROC_H_ +#include #include /* For struct callout. */ #include /* For struct klist. */ #ifdef _KERNEL @@ -1098,7 +1099,6 @@ extern int allproc_gen; extern struct sx proctree_lock; extern struct mtx ppeers_lock; -extern struct mtx procid_lock; extern struct proc proc0; /* Process slot for swapper. */ extern struct thread0_storage thread0_st; /* Primary thread in proc0. */ #define thread0 (thread0_st.t0st_thread) @@ -1351,9 +1351,12 @@ #define PROC_ID_SESSION 2 #define PROC_ID_REAP 3 +BITSET_DEFINE(proc_id_map, PID_MAX); + void proc_id_set(int type, pid_t id); -void proc_id_set_cond(int type, pid_t id); +bool proc_id_set_cond(int type, pid_t id); void proc_id_clear(int type, pid_t id); +bool proc_id_test(int type, pid_t id); EVENTHANDLER_LIST_DECLARE(process_ctor); EVENTHANDLER_LIST_DECLARE(process_dtor);