Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3891,6 +3891,7 @@ kern/subr_counter.c standard kern/subr_devstat.c standard kern/subr_disk.c standard +kern/subr_epoch.c standard kern/subr_eventhandler.c standard kern/subr_fattime.c standard kern/subr_firmware.c optional firmware Index: sys/conf/kern.pre.mk =================================================================== --- sys/conf/kern.pre.mk +++ sys/conf/kern.pre.mk @@ -77,7 +77,7 @@ .endif NOSTDINC= -nostdinc -INCLUDES= ${NOSTDINC} ${INCLMAGIC} -I. -I$S +INCLUDES= ${NOSTDINC} ${INCLMAGIC} -I. -I$S -I$S/contrib/ck/include CFLAGS= ${COPTFLAGS} ${DEBUG} CFLAGS+= ${INCLUDES} -D_KERNEL -DHAVE_KERNEL_OPTION_HEADERS -include opt_global.h Index: sys/kern/kern_malloc.c =================================================================== --- sys/kern/kern_malloc.c +++ sys/kern/kern_malloc.c @@ -514,9 +514,12 @@ } } #endif - if (flags & M_WAITOK) + if (flags & M_WAITOK) { KASSERT(curthread->td_intr_nesting_level == 0, - ("malloc(M_WAITOK) in interrupt context")); + ("malloc(M_WAITOK) in interrupt context")); + KASSERT(curthread->td_epochnest == 0, + ("malloc(M_WAITOK) in epoch context")); + } KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), ("malloc: called with spinlock or critical section held")); Index: sys/kern/kern_synch.c =================================================================== --- sys/kern/kern_synch.c +++ sys/kern/kern_synch.c @@ -147,6 +147,7 @@ ("sleeping without a lock")); KASSERT(ident != NULL, ("_sleep: NULL ident")); KASSERT(TD_IS_RUNNING(td), ("_sleep: curthread not running")); + KASSERT(td->td_epochnest == 0, ("sleeping in an epoch section")); if (priority & PDROP) KASSERT(lock != NULL && lock != &Giant.lock_object, ("PDROP requires a non-Giant lock")); Index: sys/kern/subr_epoch.c =================================================================== --- /dev/null +++ sys/kern/subr_epoch.c @@ -0,0 +1,476 @@ +/*- + * Copyright (c) 2018, Matthew Macy + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Neither the name of Matthew Macy nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); + + +SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information"); +SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW, 0, "epoch stats"); + +/* Stats. */ +static counter_u64_t wait_count; +SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, preemption_waits, CTLFLAG_RW, + &wait_count, "# of times waited due to preemption"); +static counter_u64_t yield_count; +SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, yields, CTLFLAG_RW, + &yield_count, "# of times yielded to other cpu"); + +typedef struct epoch_cb { + void (*ec_callback)(epoch_context_t); + STAILQ_ENTRY(epoch_cb) ec_link; +} *epoch_cb_t; + +typedef struct epoch_record { + ck_epoch_record_t er_record; + TAILQ_HEAD(, thread) er_tdlist; + uint32_t er_cpuid; +} *epoch_record_t; + +struct epoch_pcpu_state { + struct epoch_record eps_record; + volatile int eps_critnest; + volatile int eps_waiters; +} __aligned(CACHE_LINE_SIZE); + +struct epoch { + struct ck_epoch e_epoch; + struct mtx e_lock; + struct grouptask e_gtask; + STAILQ_HEAD(, epoch_cb) e_cblist; + struct epoch_pcpu_state *e_pcpu_dom[MAXMEMDOM]; + struct epoch_pcpu_state *e_pcpu[0]; +}; + +static __read_mostly int domcount[MAXMEMDOM]; +static __read_mostly int domoffsets[MAXMEMDOM]; +static __read_mostly int inited; + +static void epoch_call_task(void *context); +static bool usedomains = true; + +static void +epoch_init(void *arg __unused) +{ + int domain, count; + + count = domain = 0; + domoffsets[0] = 0; + for (domain = 0; domain < vm_ndomains; domain++) { + domcount[domain] = CPU_COUNT(&cpuset_domain[domain]); + if (bootverbose) + printf("domcount[%d] %d\n", domain, domcount[domain]); + } + for (domain = 1; domain < vm_ndomains; domain++) + domoffsets[domain] = domoffsets[domain-1] + domcount[domain-1]; + + for (domain = 0; domain < vm_ndomains; domain++) { + if (domcount[domain] == 0) { + usedomains = false; + break; + } + } + + wait_count = counter_u64_alloc(M_WAITOK); + yield_count = counter_u64_alloc(M_WAITOK); + inited = 1; +} +SYSINIT(epoch, SI_SUB_CPU + 1, SI_ORDER_FIRST, epoch_init, NULL); + +static void +epoch_init_numa(epoch_t epoch) +{ + int domain, cpu_offset; + struct epoch_pcpu_state *eps; + epoch_record_t er; + + for (domain = 0; domain < vm_ndomains; domain++) { + eps = malloc_domain(sizeof(*eps)*domcount[domain], M_EPOCH, + domain, M_ZERO|M_WAITOK); + epoch->e_pcpu_dom[domain] = eps; + cpu_offset = domoffsets[domain]; + for (int i = 0; i < domcount[domain]; i++, eps++) { + epoch->e_pcpu[cpu_offset + i] = eps; + er = &eps->eps_record; + ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL); + TAILQ_INIT(&er->er_tdlist); + er->er_cpuid = cpu_offset + i; + } + } +} + +static void +epoch_init_legacy(epoch_t epoch) +{ + struct epoch_pcpu_state *eps; + epoch_record_t er; + + eps = malloc(sizeof(*eps)*mp_ncpus, M_EPOCH, M_ZERO|M_WAITOK); + epoch->e_pcpu_dom[0] = eps; + for (int i = 0; i < mp_ncpus; i++, eps++) { + epoch->e_pcpu[i] = eps; + er = &eps->eps_record; + ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL); + TAILQ_INIT(&er->er_tdlist); + er->er_cpuid = i; + } +} + +epoch_t +epoch_alloc(void) +{ + epoch_t epoch; + + if (__predict_false(!inited)) + panic("%s called too early in boot", __func__); + epoch = malloc(sizeof(struct epoch) + mp_ncpus*sizeof(void*), + M_EPOCH, M_ZERO|M_WAITOK); + ck_epoch_init(&epoch->e_epoch); + mtx_init(&epoch->e_lock, "epoch cblist", NULL, MTX_DEF); + STAILQ_INIT(&epoch->e_cblist); + taskqgroup_config_gtask_init(epoch, &epoch->e_gtask, epoch_call_task, "epoch call task"); + if (usedomains) + epoch_init_numa(epoch); + else + epoch_init_legacy(epoch); + return (epoch); +} + +void +epoch_free(epoch_t epoch) +{ + int domain; +#ifdef INVARIANTS + struct epoch_pcpu_state *eps; + int cpu; + + CPU_FOREACH(cpu) { + eps = epoch->e_pcpu[cpu]; + MPASS(TAILQ_EMPTY(&eps->eps_record.er_tdlist)); + } +#endif + mtx_destroy(&epoch->e_lock); + taskqgroup_config_gtask_deinit(&epoch->e_gtask); + if (usedomains) + for (domain = 0; domain < vm_ndomains; domain++) + free_domain(epoch->e_pcpu_dom[domain], M_EPOCH); + else + free(epoch->e_pcpu_dom[0], M_EPOCH); + free(epoch, M_EPOCH); +} + +#define INIT_CHECK(epoch) \ + do { \ + if (__predict_false((epoch) == NULL)) \ + return; \ + } while (0) + +void +epoch_enter(epoch_t epoch) +{ + struct epoch_pcpu_state *eps; + struct thread *td; + + INIT_CHECK(epoch); + + td = curthread; + critical_enter(); + eps = epoch->e_pcpu[curcpu]; + td->td_epochnest++; + MPASS(td->td_epochnest < UCHAR_MAX - 2); + if (td->td_epochnest == 1) + TAILQ_INSERT_TAIL(&eps->eps_record.er_tdlist, td, td_epochq); +#ifdef INVARIANTS + if (td->td_epochnest > 1) { + struct thread *curtd; + int found = 0; + + TAILQ_FOREACH(curtd, &eps->eps_record.er_tdlist, td_epochq) + if (curtd == td) + found = 1; + KASSERT(found, ("recursing on a second epoch")); + } +#endif + sched_pin(); + ck_epoch_begin(&eps->eps_record.er_record, NULL); + critical_exit(); +} + +void +epoch_enter_nopreempt(epoch_t epoch) +{ + struct epoch_pcpu_state *eps; + + INIT_CHECK(epoch); + critical_enter(); + eps = epoch->e_pcpu[curcpu]; + curthread->td_epochnest++; + MPASS(curthread->td_epochnest < UCHAR_MAX - 2); + ck_epoch_begin(&eps->eps_record.er_record, NULL); +} + +void +epoch_exit(epoch_t epoch) +{ + struct epoch_pcpu_state *eps; + struct thread *td; + + td = curthread; + INIT_CHECK(epoch); + critical_enter(); + eps = epoch->e_pcpu[curcpu]; + sched_unpin(); + ck_epoch_end(&eps->eps_record.er_record, NULL); + td->td_epochnest--; + if (td->td_epochnest == 0) + TAILQ_REMOVE(&eps->eps_record.er_tdlist, td, td_epochq); + critical_exit(); +} + +void +epoch_exit_nopreempt(epoch_t epoch) +{ + struct epoch_pcpu_state *eps; + + INIT_CHECK(epoch); + MPASS(curthread->td_critnest); + eps = epoch->e_pcpu[curcpu]; + ck_epoch_end(&eps->eps_record.er_record, NULL); + curthread->td_epochnest--; + critical_exit(); +} + +/* + * epoch_block_handler is a callback from the ck code when another thread is + * currently in an epoch section. + */ +static void +epoch_block_handler(struct ck_epoch *global __unused, ck_epoch_record_t *cr, + void *arg __unused) +{ + epoch_record_t record; + struct epoch_pcpu_state *eps; + struct thread *td, *tdwait, *owner; + struct turnstile *ts; + struct lock_object *lock; + + eps = arg; + record = __containerof(cr, struct epoch_record, er_record); + td = curthread; + + counter_u64_add(yield_count, 1); + if (record->er_cpuid != curcpu) { + /* + * Being on the same CPU as that of the record on which + * we need to wait allows us access to the thread + * list associated with that CPU. We can then examine the + * oldest thread in the queue and wait on its turnstile + * until it resumes and so on until a grace period + * elapses. + * + */ + sched_bind(td, record->er_cpuid); + /* + * At this point we need to return to the ck code + * to scan to see if a grace period has elapsed. + * We can't move on to check the thread list, because + * in the meantime new threads may have arrived that + * in fact belong to a different epoch. + */ + return; + } + /* + * Try to find a thread in an epoch section on this CPU + * waiting on a turnstile. + */ + TAILQ_FOREACH(tdwait, &record->er_tdlist, td_epochq) { + if (TD_IS_INHIBITED(tdwait) && TD_ON_LOCK(tdwait) && + ((ts = tdwait->td_blocked) != NULL)) { + /* + * We unlock td to allow turnstile_wait to reacquire the + * the thread lock. Before unlocking it we enter a critical + * section to prevent preemption after we reenable interrupts + * by dropping the thread lock in order to prevent tdwait + * from getting to run. + */ + critical_enter(); + thread_unlock(td); + owner = turnstile_lock(ts, &lock); + /* + * The owner pointer indicates that the lock succeeded. Only + * in case we hold the lock and the turnstile we locked is still + * the one that tdwait is blocked on can we continue. Otherwise + * The turnstile pointer has been changed out from underneath + * us, as in the case where the lock holder has signalled tdwait, + * and we need to continue. + */ + if (owner != NULL && ts == tdwait->td_blocked) { + MPASS(TD_IS_INHIBITED(tdwait) && TD_ON_LOCK(tdwait)); + critical_exit(); + turnstile_wait(ts, owner, tdwait->td_tsqueue); + thread_lock(td); +#ifdef INVARIANTS + MPASS(td->td_turnstile != NULL); + turnstile_assert(td->td_turnstile); +#endif + return; + } else if (owner != NULL) + turnstile_unlock(ts, lock); + thread_lock(td); +#ifdef INVARIANTS + MPASS(td->td_turnstile != NULL); + turnstile_assert(td->td_turnstile); +#endif + critical_exit(); + } + } + /* + * We didn't find any threads actually blocked on a lock + * we have nothing to do except sleep here. Possibly we + * should have a mechanism to atomically sleep on the epoch + * but that could make epoch_exit() more expensive, the path + * whose performance we most care about. + */ + counter_u64_add(wait_count, 1); + thread_unlock(td); + pause("W", 1); + thread_lock(td); +} + +/* + * Taken verbatim from linux_synchronize_rcu + */ +void +epoch_wait(epoch_t epoch) +{ + struct thread *td; + int was_bound; + int old_cpu; + int old_pinned; + u_char old_prio; + + INIT_CHECK(epoch); + + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "epoch_wait() can sleep"); + + td = curthread; + KASSERT(td->td_epochnest == 0, ("epoch_wait() in the middle of an epoch section")); + thread_lock(td); + + DROP_GIANT(); + + old_cpu = PCPU_GET(cpuid); + old_pinned = td->td_pinned; + old_prio = td->td_priority; + was_bound = sched_is_bound(td); + sched_unbind(td); + td->td_pinned = 0; + sched_bind(td, old_cpu); + + ck_epoch_synchronize_wait(&epoch->e_epoch, epoch_block_handler, NULL); + + /* restore CPU binding, if any */ + if (was_bound != 0) { + sched_bind(td, old_cpu); + } else { + /* get thread back to initial CPU, if any */ + if (old_pinned != 0) + sched_bind(td, old_cpu); + sched_unbind(td); + } + /* restore pinned after bind */ + td->td_pinned = old_pinned; + + /* restore thread priority */ + sched_prio(td, old_prio); + thread_unlock(td); + + PICKUP_GIANT(); +} + +void +epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)) +{ + epoch_cb_t cb; + + cb = (void *)ctx; + cb->ec_callback = callback; + mtx_lock(&epoch->e_lock); + STAILQ_INSERT_TAIL(&epoch->e_cblist, cb, ec_link); + GROUPTASK_ENQUEUE(&epoch->e_gtask); + mtx_unlock(&epoch->e_lock); +} + +static void +epoch_call_task(void *context) +{ + epoch_t epoch; + epoch_cb_t cb; + STAILQ_HEAD(, epoch_cb) tmp_head; + + epoch = context; + STAILQ_INIT(&tmp_head); + + mtx_lock(&epoch->e_lock); + STAILQ_CONCAT(&tmp_head, &epoch->e_cblist); + mtx_unlock(&epoch->e_lock); + + epoch_wait(epoch); + + while ((cb = STAILQ_FIRST(&tmp_head)) != NULL) + cb->ec_callback((void*)cb); +} + +int +in_epoch(void) +{ + return (curthread->td_epochnest != 0); +} Index: sys/kern/subr_trap.c =================================================================== --- sys/kern/subr_trap.c +++ sys/kern/subr_trap.c @@ -161,6 +161,8 @@ WITNESS_WARN(WARN_PANIC, NULL, "userret: returning"); KASSERT(td->td_critnest == 0, ("userret: Returning in a critical section")); + KASSERT(td->td_epochnest == 0, + ("userret: Returning in an epoch section")); KASSERT(td->td_locks == 0, ("userret: Returning with %d locks held", td->td_locks)); KASSERT(td->td_rw_rlocks == 0, Index: sys/kern/subr_turnstile.c =================================================================== --- sys/kern/subr_turnstile.c +++ sys/kern/subr_turnstile.c @@ -566,6 +566,45 @@ return (ts); } +struct thread * +turnstile_lock(struct turnstile *ts, struct lock_object **lockp) +{ + struct turnstile_chain *tc; + struct lock_object *lock; + + if ((lock = ts->ts_lockobj) == NULL) + return (NULL); + tc = TC_LOOKUP(lock); + mtx_lock_spin(&tc->tc_lock); + mtx_lock_spin(&ts->ts_lock); + if (__predict_false(lock != ts->ts_lockobj)) { + mtx_unlock_spin(&tc->tc_lock); + mtx_unlock_spin(&ts->ts_lock); + return (NULL); + } + *lockp = lock; + return (ts->ts_owner); +} + +void +turnstile_unlock(struct turnstile *ts, struct lock_object *lock) +{ + struct turnstile_chain *tc; + + mtx_assert(&ts->ts_lock, MA_OWNED); + mtx_unlock_spin(&ts->ts_lock); + if (ts == curthread->td_turnstile) + ts->ts_lockobj = NULL; + tc = TC_LOOKUP(lock); + mtx_unlock_spin(&tc->tc_lock); +} + +void +turnstile_assert(struct turnstile *ts) +{ + MPASS(ts->ts_lockobj == NULL); +} + void turnstile_cancel(struct turnstile *ts) { Index: sys/modules/epoch_test/Makefile =================================================================== --- /dev/null +++ sys/modules/epoch_test/Makefile @@ -0,0 +1,7 @@ +# $FreeBSD$ + +.PATH: ${SRCTOP}/sys/tests/epoch +KMOD= epoch_test +SRCS= epoch_test.c + +.include Index: sys/sys/epoch.h =================================================================== --- /dev/null +++ sys/sys/epoch.h @@ -0,0 +1,49 @@ +/*- + * Copyright (c) 2018, Matthew Macy + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Neither the name of Matthew Macy nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SYS_EPOCH_H_ +#define _SYS_EPOCH_H_ + +struct epoch; +typedef struct epoch *epoch_t; + +struct epoch_context { + void *data[2]; +} __aligned(sizeof(void *)); + +typedef struct epoch_context *epoch_context_t; + +epoch_t epoch_alloc(void); +void epoch_free(epoch_t epoch); +void epoch_enter(epoch_t epoch); +void epoch_exit(epoch_t epoch); +void epoch_enter_nopreempt(epoch_t epoch); +void epoch_exit_nopreempt(epoch_t epoch); +void epoch_wait(epoch_t epoch); +void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)); +int in_epoch(void); + +#endif Index: sys/sys/proc.h =================================================================== --- sys/sys/proc.h +++ sys/sys/proc.h @@ -243,6 +243,7 @@ /* Cleared during fork1() */ #define td_startzero td_flags + u_char td_epochnest; /* (k) Private thread epoch nest counter */ int td_flags; /* (t) TDF_* flags. */ int td_inhibitors; /* (t) Why can not run. */ int td_pflags; /* (k) Private thread (TDP_*) flags. */ @@ -355,6 +356,7 @@ int td_lastcpu; /* (t) Last cpu we were on. */ int td_oncpu; /* (t) Which cpu we are on. */ void *td_lkpi_task; /* LinuxKPI task struct pointer */ + TAILQ_ENTRY(thread) td_epochq; /* (t) Epoch queue. */ }; struct thread0_storage { Index: sys/sys/turnstile.h =================================================================== --- sys/sys/turnstile.h +++ sys/sys/turnstile.h @@ -104,6 +104,8 @@ struct turnstile *turnstile_trywait(struct lock_object *); void turnstile_unpend(struct turnstile *, int); void turnstile_wait(struct turnstile *, struct thread *, int); - +struct thread *turnstile_lock(struct turnstile *, struct lock_object **); +void turnstile_unlock(struct turnstile *, struct lock_object *); +void turnstile_assert(struct turnstile *); #endif /* _KERNEL */ #endif /* _SYS_TURNSTILE_H_ */ Index: sys/tests/epoch/epoch_test.c =================================================================== --- /dev/null +++ sys/tests/epoch/epoch_test.c @@ -0,0 +1,156 @@ +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +struct epoch_test_instance { + int threadid; +}; + +static int inited; +#define ET_EXITING 0x1 +static volatile int state_flags; +static struct mtx state_mtx __aligned(CACHE_LINE_SIZE*2); +MTX_SYSINIT(state_mtx, &state_mtx, "epoch state mutex", MTX_DEF); +static struct mtx mutexA __aligned(CACHE_LINE_SIZE*2); +MTX_SYSINIT(mutexA, &mutexA, "epoch mutexA", MTX_DEF); +static struct mtx mutexB __aligned(CACHE_LINE_SIZE*2); +MTX_SYSINIT(mutexB, &mutexB, "epoch mutexB", MTX_DEF); +epoch_t test_epoch; + +static void +epoch_testcase(struct epoch_test_instance *eti) +{ + int i, startticks; + struct mtx *mtxp; + + startticks = ticks; + i = 0; + if (eti->threadid & 0x1) + mtxp = &mutexA; + else + mtxp = &mutexB; + + while (i < 1000) { + epoch_enter(test_epoch); + mtx_lock(mtxp); + i++; + mtx_unlock(mtxp); + epoch_exit(test_epoch); + epoch_wait(test_epoch); + } + printf("thread: %d took %d ticks to complete 1000 iterations\n", + eti->threadid, ticks - startticks); +} + +static void +testloop(void *arg) { + + mtx_lock(&state_mtx); + while ((state_flags & ET_EXITING) == 0) { + msleep(&state_mtx, &state_mtx, 0, "epoch start wait", 0); + if (state_flags & ET_EXITING) + goto out; + mtx_unlock(&state_mtx); + epoch_testcase(arg); + mtx_lock(&state_mtx); + } + out: + mtx_unlock(&state_mtx); + kthread_exit(); +} + +static struct thread *testthreads[MAXCPU]; +static struct epoch_test_instance etilist[MAXCPU]; + +static int +test_modinit(void) +{ + int i, error; + + test_epoch = epoch_alloc(); + for (i = 0; i < mp_ncpus; i++) { + etilist[i].threadid = i; + error = kthread_add(testloop, &etilist[i], NULL, &testthreads[i], + 0, 0, "epoch_test_%d", i); + if (error) { + printf("%s: kthread_add(epoch_test): error %d", __func__, + error); + } + } + inited = 1; + return (0); +} + +static int +epochtest_execute(SYSCTL_HANDLER_ARGS) +{ + int error, v; + + if (inited == 0) + return (ENOENT); + + v = 0; + error = sysctl_handle_int(oidp, &v, 0, req); + if (error) + return (error); + if (req->newptr == NULL) + return (error); + if (v == 0) + return (0); + wakeup(&state_mtx); + + return (0); +} + +SYSCTL_NODE(_kern, OID_AUTO, epochtest, CTLFLAG_RW, 0, "Epoch Test Framework"); +SYSCTL_PROC(_kern_epochtest, OID_AUTO, runtest, (CTLTYPE_INT | CTLFLAG_RW), + 0, 0, epochtest_execute, "I", "Execute an epoch test"); + +static int +epoch_test_module_event_handler(module_t mod, int what, void *arg __unused) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = test_modinit()) != 0) + return (err); + break; + case MOD_UNLOAD: + mtx_lock(&state_mtx); + state_flags = ET_EXITING; + wakeup(&state_mtx); + mtx_unlock(&state_mtx); + /* yes --- gross */ + pause("epoch unload", 3*hz); + break; + default: + return (EOPNOTSUPP); + } + + return (0); +} + +static moduledata_t epoch_test_moduledata = { + "epoch_test", + epoch_test_module_event_handler, + NULL +}; + +MODULE_VERSION(epoch_test, 1); +DECLARE_MODULE(epoch_test, epoch_test_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY);