Index: sys/amd64/conf/GENERIC =================================================================== --- sys/amd64/conf/GENERIC +++ sys/amd64/conf/GENERIC @@ -73,7 +73,8 @@ options KDTRACE_HOOKS # Kernel DTrace hooks options DDB_CTF # Kernel ELF linker loads CTF data options INCLUDE_CONFIG_FILE # Include this file in kernel - +# Add in the kernel test framework +options KERN_TEST_FRWK # Debugging support. Always need this: options KDB # Enable kernel debugger support. options KDB_TRACE # Print a stack trace for a panic. Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3003,6 +3003,7 @@ kern/kern_syscalls.c standard kern/kern_sysctl.c standard kern/kern_tc.c standard +kern/kern_testfrwk.c standard kern/kern_thr.c standard kern/kern_thread.c standard kern/kern_time.c standard Index: sys/conf/options =================================================================== --- sys/conf/options +++ sys/conf/options @@ -156,6 +156,7 @@ MAC_STATIC opt_mac.h MAC_STUB opt_dontuse.h MAC_TEST opt_dontuse.h +KERN_TEST_FRWK opt_testfw.h MD_ROOT opt_md.h MD_ROOT_FSTYPE opt_md.h MD_ROOT_SIZE opt_md.h Index: sys/kern/kern_testfrwk.c =================================================================== --- sys/kern/kern_testfrwk.c +++ sys/kern/kern_testfrwk.c @@ -0,0 +1,296 @@ +/*- + * Copyright (c) 2015 + * Netflix Incorporated, All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ +#include "opt_testfw.h" +#include +#include +#include "opt_testfw.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef SMP +#include +#endif + +#ifdef KERN_TEST_FRWK +struct kern_test_list { + TAILQ_ENTRY(kern_test_list) next; + char name[TEST_NAME_LEN]; + void (*kerntfunc)(struct kern_test *); +}; + +TAILQ_HEAD(ktestlist, kern_test_list); + +struct kern_test_entry { + TAILQ_ENTRY(kern_test_entry) next; + struct kern_test_list *kt_e; + struct kern_test kt_data; +}; +TAILQ_HEAD(ktestqueue, kern_test_entry); + +MALLOC_DEFINE(M_KTFRWK, "kern_tfrwk", "Kernel Test Framework"); +struct kern_totfrwk { + struct taskqueue *kfrwk_tq; + struct task kfrwk_que; + struct ktestlist kfrwk_testlist; + struct ktestqueue kfrwk_testq; + struct mtx kfrwk_mtx; + int kfrwk_waiting; +}; + +struct kern_totfrwk kfrwk; + +#define KTFRWK_MUTEX_INIT() mtx_init(&kfrwk.kfrwk_mtx, "kern_test_frwk", "tfrwk", MTX_DEF) + +#define KTFRWK_DESTROY() mtx_destroy(&kfrwk.kfrwk_mtx) + +#define KTFRWK_LOCK() mtx_lock(&kfrwk.kfrwk_mtx) + +#define KTFRWK_UNLOCK() mtx_unlock(&kfrwk.kfrwk_mtx) + +static void +kfrwk_task(void *context, int pending) +{ + struct kern_totfrwk *tf; + struct kern_test_entry *wk; + int free_mem=0; + struct kern_test kt_data; + void (*kerntfunc)(struct kern_test *); + + memset(&kt_data, 0, sizeof(kt_data)); + kerntfunc = NULL; + tf = (struct kern_totfrwk *)context; + KTFRWK_LOCK(); + wk = TAILQ_FIRST(&tf->kfrwk_testq); + if (wk) { + wk->kt_data.num_threads--; + tf->kfrwk_waiting--; + if (wk->kt_data.num_threads == 0) { + TAILQ_REMOVE(&tf->kfrwk_testq, wk, next); + free_mem = 1; + } else { + /* Wake one of my colleages up to help too */ + taskqueue_enqueue(tf->kfrwk_tq, &tf->kfrwk_que); + } + memcpy(&kt_data, &wk->kt_data, sizeof(kt_data)); + if (wk->kt_e) { + kerntfunc = wk->kt_e->kerntfunc; + } + } + KTFRWK_UNLOCK(); + if (wk && free_mem) { + free(wk, M_KTFRWK); + } + /* Execute the test */ + if (kerntfunc){ + (*kerntfunc)(&kt_data); + } + /* We are done */ + atomic_add_int(&tf->kfrwk_waiting, 1); +} + +static void +kerntest_frwk_init(void *st) +{ + u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU; + + KTFRWK_MUTEX_INIT(); + TAILQ_INIT(&kfrwk.kfrwk_testq); + TAILQ_INIT(&kfrwk.kfrwk_testlist); + /* Now lets start up a number of tasks to do the work */ + TASK_INIT(&kfrwk.kfrwk_que, 0, kfrwk_task, &kfrwk); + kfrwk.kfrwk_tq = taskqueue_create_fast("sbtls_task", M_NOWAIT, + taskqueue_thread_enqueue, &kfrwk.kfrwk_tq); + if (kfrwk.kfrwk_tq == NULL) { + printf("Can't start taskqueue for Kernel Test Framework\n"); + panic("Taskqueue init fails for kfrwk"); + } + taskqueue_start_threads(&kfrwk.kfrwk_tq, ncpus, PI_NET, "[kt_frwk task]"); + kfrwk.kfrwk_waiting = ncpus; +} + +SYSINIT(kerntest_frwk, SI_SUB_KPROF, SI_ORDER_ANY, kerntest_frwk_init, NULL); + +#endif + +static int kerntest_execute(SYSCTL_HANDLER_ARGS); + +SYSCTL_DECL(_kern); +SYSCTL_NODE(_kern, OID_AUTO, testfrwk, CTLFLAG_RW, 0, "Kernel Test Framework"); + + +SYSCTL_PROC(_kern_testfrwk, OID_AUTO, runtest, (CTLTYPE_STRUCT | CTLFLAG_RW), + 0, 0, kerntest_execute, "IU", "Execute a kernel test"); + +int +kerntest_execute(SYSCTL_HANDLER_ARGS) +{ +#ifndef KERN_TEST_FRWK + return (ENOTSUP); +#else + struct kern_test kt; + struct kern_test_list *li, *te=NULL; + struct kern_test_entry *kte=NULL; + int error = 0; + /* Find the entry if possible */ + error = SYSCTL_IN(req, &kt, sizeof(struct kern_test)); + if (error) { + return(error); + } + if (kt.num_threads <= 0) { + return (EINVAL); + } + KTFRWK_LOCK(); + TAILQ_FOREACH(li, &kfrwk.kfrwk_testlist, next) { + if (strcmp(li->name, kt.name) == 0) { + te = li; + break; + } + } + if (te == NULL) { + error = ENOENT; + goto out; + } + /* Grab some memory */ + kte = malloc(sizeof(struct kern_test_entry), M_KTFRWK, M_WAITOK); + if (kte == NULL) { + error = ENOMEM; + goto out; + } + /* Ok we have a test item to run, can we? */ + if (!TAILQ_EMPTY(&kfrwk.kfrwk_testq)) { + /* We don't know if there is enough threads */ + error = EAGAIN; + free(kte, M_KTFRWK); + goto out; + } + if (kfrwk.kfrwk_waiting < kt.num_threads) { + error = E2BIG; + free(kte, M_KTFRWK); + goto out; + } + /* Ok it looks like we can do it, lets get an entry */ + kte->kt_e = li; + memcpy(&kte->kt_data, &kt, sizeof(kt)); + TAILQ_INSERT_TAIL(&kfrwk.kfrwk_testq, kte, next); + taskqueue_enqueue(kfrwk.kfrwk_tq, &kfrwk.kfrwk_que); +out: + KTFRWK_UNLOCK(); + return(error); +#endif +} + +int +kern_testframework_register(const char *name, + void (*kerntfunc)(struct kern_test *)) +{ +#ifndef KERN_TEST_FRWK + return (ENOTSUP); +#else + int error = 0; + struct kern_test_list *li, *te=NULL; + int len; + + len = strlen(name); + if (len >= TEST_NAME_LEN) { + return (E2BIG); + } + KTFRWK_LOCK(); + /* First does it already exist? */ + TAILQ_FOREACH(li, &kfrwk.kfrwk_testlist, next) { + if (strcmp(li->name, name) == 0) { + error = EALREADY; + goto out; + } + } + /* Ok we can do it, lets add it to the list */ + te = malloc(sizeof(struct kern_test_list), M_KTFRWK, M_WAITOK); + if (te == NULL) { + error = ENOMEM; + goto out; + } + te->kerntfunc = kerntfunc; + strcpy(te->name, name); + TAILQ_INSERT_TAIL(&kfrwk.kfrwk_testlist, te, next); +out: + KTFRWK_UNLOCK(); + return(error); +#endif +} + +int +kern_testframework_deregister(const char *name) +{ +#ifndef KERN_TEST_FRWK + return (ENOTSUP); +#else + struct kern_test_list *li, *te=NULL; + u_int ncpus = mp_ncpus ? mp_ncpus : MAXCPU; + int error = 0; + + KTFRWK_LOCK(); + /* First does it already exist? */ + TAILQ_FOREACH (li, &kfrwk.kfrwk_testlist, next) { + if (strcmp(li->name, name) == 0) { + te = li; + break; + } + } + if (te == NULL) { + /* It is not registered so no problem */ + goto out; + } + if (ncpus != kfrwk.kfrwk_waiting) { + /* We are busy executing something -- can't unload */ + error = EBUSY; + goto out; + } + if (!TAILQ_EMPTY(&kfrwk.kfrwk_testq)) { + /* Something still to execute */ + error = EBUSY; + goto out; + } + /* Ok we can remove the dude safely */ + TAILQ_REMOVE(&kfrwk.kfrwk_testlist, te, next); + memset(te, 0, sizeof(struct kern_test_list)); + free(te, M_KTFRWK); +out: + KTFRWK_UNLOCK(); + return(error); +#endif +} Index: sys/kern/kern_timeout.c =================================================================== --- sys/kern/kern_timeout.c +++ sys/kern/kern_timeout.c @@ -166,26 +166,16 @@ char cc_ktr_event_name[20]; }; -#define cc_exec_curr cc_exec_entity[0].cc_curr -#define cc_exec_next cc_exec_entity[0].cc_next -#define cc_exec_cancel cc_exec_entity[0].cc_cancel -#define cc_exec_waiting cc_exec_entity[0].cc_waiting -#define cc_exec_curr_dir cc_exec_entity[1].cc_curr -#define cc_exec_next_dir cc_exec_entity[1].cc_next -#define cc_exec_cancel_dir cc_exec_entity[1].cc_cancel -#define cc_exec_waiting_dir cc_exec_entity[1].cc_waiting - +#define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr +#define cc_exec_next(cc, dir) cc->cc_exec_entity[dir].cc_next +#define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel +#define cc_exec_waiting(cc, dir) cc->cc_exec_entity[dir].cc_waiting #ifdef SMP -#define cc_migration_func cc_exec_entity[0].ce_migration_func -#define cc_migration_arg cc_exec_entity[0].ce_migration_arg -#define cc_migration_cpu cc_exec_entity[0].ce_migration_cpu -#define cc_migration_time cc_exec_entity[0].ce_migration_time -#define cc_migration_prec cc_exec_entity[0].ce_migration_prec -#define cc_migration_func_dir cc_exec_entity[1].ce_migration_func -#define cc_migration_arg_dir cc_exec_entity[1].ce_migration_arg -#define cc_migration_cpu_dir cc_exec_entity[1].ce_migration_cpu -#define cc_migration_time_dir cc_exec_entity[1].ce_migration_time -#define cc_migration_prec_dir cc_exec_entity[1].ce_migration_prec +#define cc_migration_func(cc, dir) cc->cc_exec_entity[dir].ce_migration_func +#define cc_migration_arg(cc, dir) cc->cc_exec_entity[dir].ce_migration_arg +#define cc_migration_cpu(cc, dir) cc->cc_exec_entity[dir].ce_migration_cpu +#define cc_migration_time(cc, dir) cc->cc_exec_entity[dir].ce_migration_time +#define cc_migration_prec(cc, dir) cc->cc_exec_entity[dir].ce_migration_prec struct callout_cpu cc_cpu[MAXCPU]; #define CPUBLOCK MAXCPU @@ -235,16 +225,16 @@ cc_cce_cleanup(struct callout_cpu *cc, int direct) { - cc->cc_exec_entity[direct].cc_curr = NULL; - cc->cc_exec_entity[direct].cc_next = NULL; - cc->cc_exec_entity[direct].cc_cancel = false; - cc->cc_exec_entity[direct].cc_waiting = false; + cc_exec_curr(cc, direct) = NULL; + cc_exec_next(cc, direct) = NULL; + cc_exec_cancel(cc, direct) = false; + cc_exec_waiting(cc, direct) = false; #ifdef SMP - cc->cc_exec_entity[direct].ce_migration_cpu = CPUBLOCK; - cc->cc_exec_entity[direct].ce_migration_time = 0; - cc->cc_exec_entity[direct].ce_migration_prec = 0; - cc->cc_exec_entity[direct].ce_migration_func = NULL; - cc->cc_exec_entity[direct].ce_migration_arg = NULL; + cc_migration_cpu(cc, direct) = CPUBLOCK; + cc_migration_time(cc, direct) = 0; + cc_migration_prec(cc, direct) = 0; + cc_migration_func(cc, direct) = NULL; + cc_migration_arg(cc, direct) = NULL; #endif } @@ -256,7 +246,7 @@ { #ifdef SMP - return (cc->cc_exec_entity[direct].ce_migration_cpu != CPUBLOCK); + return (cc_migration_cpu(cc, direct) != CPUBLOCK); #else return (0); #endif @@ -492,7 +482,7 @@ #ifdef CALLOUT_PROFILING ++depth_dir; #endif - cc->cc_exec_next_dir = + cc_exec_next(cc, 1) = LIST_NEXT(tmp, c_links.le); cc->cc_bucket = firstb & callwheelmask; LIST_REMOVE(tmp, c_links.le); @@ -501,7 +491,7 @@ &mpcalls_dir, &lockcalls_dir, NULL, #endif 1); - tmp = cc->cc_exec_next_dir; + tmp = cc_exec_next(cc, 1); } else { tmpn = LIST_NEXT(tmp, c_links.le); LIST_REMOVE(tmp, c_links.le); @@ -585,7 +575,7 @@ static void callout_cc_add(struct callout *c, struct callout_cpu *cc, sbintime_t sbt, sbintime_t precision, void (*func)(void *), - void *arg, int cpu, int flags) + void *arg, int cpu, int flags, int direct) { int bucket; @@ -606,7 +596,7 @@ (u_int)(c->c_precision & 0xffffffff)); LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le); if (cc->cc_bucket == bucket) - cc->cc_exec_next_dir = c; + cc_exec_next(cc, direct) = c; #ifndef NO_EVENTTIMERS /* * Inform the eventtimers(4) subsystem there's a new callout @@ -679,8 +669,9 @@ c->c_flags = CALLOUT_LOCAL_ALLOC; else c->c_flags &= ~CALLOUT_PENDING; - cc->cc_exec_entity[direct].cc_curr = c; - cc->cc_exec_entity[direct].cc_cancel = false; + + cc_exec_curr(cc, direct) = c; + cc_exec_cancel(cc, direct) = false; CC_UNLOCK(cc); if (c_lock != NULL) { class->lc_lock(c_lock, lock_status); @@ -688,12 +679,12 @@ * The callout may have been cancelled * while we switched locks. */ - if (cc->cc_exec_entity[direct].cc_cancel) { + if (cc_exec_cancel(cc, direct)) { class->lc_unlock(c_lock); goto skip; } /* The callout cannot be stopped now. */ - cc->cc_exec_entity[direct].cc_cancel = true; + cc_exec_cancel(cc, direct) = true; if (c_lock == &Giant.lock_object) { #ifdef CALLOUT_PROFILING (*gcalls)++; @@ -744,9 +735,9 @@ class->lc_unlock(c_lock); skip: CC_LOCK(cc); - KASSERT(cc->cc_exec_entity[direct].cc_curr == c, ("mishandled cc_curr")); - cc->cc_exec_entity[direct].cc_curr = NULL; - if (cc->cc_exec_entity[direct].cc_waiting) { + KASSERT(cc_exec_curr(cc, direct) == c, ("mishandled cc_curr")); + cc_exec_curr(cc, direct) = NULL; + if (cc_exec_waiting(cc, direct)) { /* * There is someone waiting for the * callout to complete. @@ -762,9 +753,9 @@ */ c->c_flags &= ~CALLOUT_DFRMIGRATION; } - cc->cc_exec_entity[direct].cc_waiting = false; + cc_exec_waiting(cc, direct) = false; CC_UNLOCK(cc); - wakeup(&cc->cc_exec_entity[direct].cc_waiting); + wakeup(&cc_exec_waiting(cc, direct)); CC_LOCK(cc); } else if (cc_cce_migrating(cc, direct)) { KASSERT((c_flags & CALLOUT_LOCAL_ALLOC) == 0, @@ -774,11 +765,11 @@ * If the callout was scheduled for * migration just perform it now. */ - new_cpu = cc->cc_exec_entity[direct].ce_migration_cpu; - new_time = cc->cc_exec_entity[direct].ce_migration_time; - new_prec = cc->cc_exec_entity[direct].ce_migration_prec; - new_func = cc->cc_exec_entity[direct].ce_migration_func; - new_arg = cc->cc_exec_entity[direct].ce_migration_arg; + new_cpu = cc_migration_cpu(cc, direct); + new_time = cc_migration_time(cc, direct); + new_prec = cc_migration_prec(cc, direct); + new_func = cc_migration_func(cc, direct); + new_arg = cc_migration_arg(cc, direct); cc_cce_cleanup(cc, direct); /* @@ -787,7 +778,7 @@ * * As first thing, handle deferred callout stops. */ - if ((c->c_flags & CALLOUT_DFRMIGRATION) == 0) { + if (!callout_migrating(c)) { CTR3(KTR_CALLOUT, "deferred cancelled %p func %p arg %p", c, new_func, new_arg); @@ -795,11 +786,11 @@ return; } c->c_flags &= ~CALLOUT_DFRMIGRATION; - + new_cc = callout_cpu_switch(c, cc, new_cpu); flags = (direct) ? C_DIRECT_EXEC : 0; callout_cc_add(c, new_cc, new_time, new_prec, new_func, - new_arg, new_cpu, flags); + new_arg, new_cpu, flags, direct); CC_UNLOCK(new_cc); CC_LOCK(cc); #else @@ -1007,15 +998,15 @@ KASSERT(!direct || c->c_lock == NULL, ("%s: direct callout %p has lock", __func__, c)); cc = callout_lock(c); - if (cc->cc_exec_entity[direct].cc_curr == c) { + if (cc_exec_curr(cc, direct) == c) { /* * We're being asked to reschedule a callout which is * currently in progress. If there is a lock then we * can cancel the callout if it has not really started. */ - if (c->c_lock != NULL && !cc->cc_exec_entity[direct].cc_cancel) - cancelled = cc->cc_exec_entity[direct].cc_cancel = true; - if (cc->cc_exec_entity[direct].cc_waiting) { + if (c->c_lock != NULL && cc_exec_cancel(cc, direct)) + cancelled = cc_exec_cancel(cc, direct) = true; + if (cc_exec_waiting(cc, direct)) { /* * Someone has called callout_drain to kill this * callout. Don't reschedule. @@ -1026,11 +1017,30 @@ CC_UNLOCK(cc); return (cancelled); } +#ifdef SMP + if (callout_migrating(c)) { + /* + * This only occurs when a second callout_reset_sbt_on + * is made after a previous one moved it into + * deferred migration (below). Note we do *not* change + * the prev_cpu even though the previous target may + * be different. + */ + cc_migration_cpu(cc, direct) = cpu; + cc_migration_time(cc, direct) = to_sbt; + cc_migration_prec(cc, direct) = precision; + cc_migration_func(cc, direct) = ftn; + cc_migration_arg(cc, direct) = arg; + cancelled = 1; + CC_UNLOCK(cc); + return (cancelled); + } +#endif } if (c->c_flags & CALLOUT_PENDING) { if ((c->c_flags & CALLOUT_PROCESSED) == 0) { - if (cc->cc_exec_next_dir == c) - cc->cc_exec_next_dir = LIST_NEXT(c, c_links.le); + if (cc_exec_next(cc, direct) == c) + cc_exec_next(cc, direct) = LIST_NEXT(c, c_links.le); LIST_REMOVE(c, c_links.le); } else TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); @@ -1045,15 +1055,23 @@ * to a more appropriate moment. */ if (c->c_cpu != cpu) { - if (cc->cc_exec_entity[direct].cc_curr == c) { - cc->cc_exec_entity[direct].ce_migration_cpu = cpu; - cc->cc_exec_entity[direct].ce_migration_time - = to_sbt; - cc->cc_exec_entity[direct].ce_migration_prec - = precision; - cc->cc_exec_entity[direct].ce_migration_func = ftn; - cc->cc_exec_entity[direct].ce_migration_arg = arg; - c->c_flags |= CALLOUT_DFRMIGRATION; + if (cc_exec_curr(cc, direct) == c) { + /* + * Pending will have been removed since we + * are actually executing the callout on another + * CPU. That callout should be waiting on the + * lock the caller holds. If we set both active/and/pending + * after we return and the lock on the executing callout + * proceeds, it will then see pending is true and return. + * At the finish out of that our migration will happen + * and the callout will be set in on the new cpu. + */ + cc_migration_cpu(cc, direct) = cpu; + cc_migration_time(cc, direct) = to_sbt; + cc_migration_prec(cc, direct) = precision; + cc_migration_func(cc, direct) = ftn; + cc_migration_arg(cc, direct) = arg; + c->c_flags |= (CALLOUT_DFRMIGRATION | CALLOUT_ACTIVE | CALLOUT_PENDING); CTR6(KTR_CALLOUT, "migration of %p func %p arg %p in %d.%08x to %u deferred", c, c->c_func, c->c_arg, (int)(to_sbt >> 32), @@ -1065,7 +1083,7 @@ } #endif - callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags); + callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags, direct); CTR6(KTR_CALLOUT, "%sscheduled %p func %p arg %p in %d.%08x", cancelled ? "re" : "", c, c->c_func, c->c_arg, (int)(to_sbt >> 32), (u_int)(to_sbt & 0xffffffff)); @@ -1095,6 +1113,7 @@ struct callout_cpu *cc, *old_cc; struct lock_class *class; int direct, sq_locked, use_lock; + int not_on_a_list; if (safe) WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, c->c_lock, @@ -1120,6 +1139,26 @@ again: cc = callout_lock(c); + if ((c->c_flags & (CALLOUT_DFRMIGRATION | CALLOUT_ACTIVE | CALLOUT_PENDING)) == + (CALLOUT_DFRMIGRATION | CALLOUT_ACTIVE | CALLOUT_PENDING)) { + /* + * Special case where this slipped in while we + * were migrating *as* the callout is about to + * execute. The caller probably holds the lock + * the callout wants. + * + * Get rid of the migration first. Then set + * the flag that tells this code *not* to + * try to remove it from any lists (its not + * on one yet). When the callout runs, it + * should do nothing with the callout. + */ + c->c_flags &= ~(CALLOUT_PENDING|CALLOUT_ACTIVE); + not_on_a_list = 1; + } else { + not_on_a_list = 0; + } + /* * If the callout was migrating while the callout cpu lock was * dropped, just drop the sleepqueue lock and check the states @@ -1128,7 +1167,7 @@ if (sq_locked != 0 && cc != old_cc) { #ifdef SMP CC_UNLOCK(cc); - sleepq_release(&old_cc->cc_exec_entity[direct].cc_waiting); + sleepq_release(&cc_exec_waiting(old_cc, direct)); sq_locked = 0; old_cc = NULL; goto again; @@ -1149,13 +1188,12 @@ * If it wasn't on the queue and it isn't the current * callout, then we can't stop it, so just bail. */ - if (cc->cc_exec_entity[direct].cc_curr != c) { + if (cc_exec_curr(cc, direct) != c) { CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p", c, c->c_func, c->c_arg); CC_UNLOCK(cc); if (sq_locked) - sleepq_release( - &cc->cc_exec_entity[direct].cc_waiting); + sleepq_release(&cc_exec_waiting(cc, direct)); return (0); } @@ -1166,7 +1204,7 @@ * just wait for the current invocation to * finish. */ - while (cc->cc_exec_entity[direct].cc_curr == c) { + while (cc_exec_curr(cc, direct) == c) { /* * Use direct calls to sleepqueue interface * instead of cv/msleep in order to avoid @@ -1187,7 +1225,7 @@ if (!sq_locked) { CC_UNLOCK(cc); sleepq_lock( - &cc->cc_exec_entity[direct].cc_waiting); + &cc_exec_waiting(cc, direct)); sq_locked = 1; old_cc = cc; goto again; @@ -1199,15 +1237,15 @@ * will be packed up, just let softclock() * take care of it. */ - cc->cc_exec_entity[direct].cc_waiting = true; + cc_exec_waiting(cc, direct) = true; DROP_GIANT(); CC_UNLOCK(cc); sleepq_add( - &cc->cc_exec_entity[direct].cc_waiting, + &cc_exec_waiting(cc, direct), &cc->cc_lock.lock_object, "codrain", SLEEPQ_SLEEP, 0); sleepq_wait( - &cc->cc_exec_entity[direct].cc_waiting, + &cc_exec_waiting(cc, direct), 0); sq_locked = 0; old_cc = NULL; @@ -1217,7 +1255,8 @@ CC_LOCK(cc); } } else if (use_lock && - !cc->cc_exec_entity[direct].cc_cancel) { + !cc_exec_cancel(cc, direct)) { + /* * The current callout is waiting for its * lock which we hold. Cancel the callout @@ -1225,7 +1264,7 @@ * lock, the callout will be skipped in * softclock(). */ - cc->cc_exec_entity[direct].cc_cancel = true; + cc_exec_cancel(cc, direct) = true; CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p", c, c->c_func, c->c_arg); KASSERT(!cc_cce_migrating(cc, direct), @@ -1233,12 +1272,34 @@ CC_UNLOCK(cc); KASSERT(!sq_locked, ("sleepqueue chain locked")); return (1); - } else if ((c->c_flags & CALLOUT_DFRMIGRATION) != 0) { + } else if (callout_migrating(c)) { + /* + * The callout is currently being serviced + * and the "next" callout is scheduled at + * its completion with a migration. We remove + * the migration flag so it *won't* get rescheduled, + * but we can't stop the one thats running so + * we return 0. + */ c->c_flags &= ~CALLOUT_DFRMIGRATION; +#ifdef SMP + /* + * We can't call cc_cce_cleanup here since + * if we do it will remove .ce_curr and + * its still running. This will prevent a + * reschedule of the callout when the + * execution completes. + */ + cc_migration_cpu(cc, direct) = CPUBLOCK; + cc_migration_time(cc, direct) = 0; + cc_migration_prec(cc, direct) = 0; + cc_migration_func(cc, direct) = NULL; + cc_migration_arg(cc, direct) = NULL; +#endif CTR3(KTR_CALLOUT, "postponing stop %p func %p arg %p", c, c->c_func, c->c_arg); CC_UNLOCK(cc); - return (1); + return (0); } CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p", c, c->c_func, c->c_arg); @@ -1247,20 +1308,21 @@ return (0); } if (sq_locked) - sleepq_release(&cc->cc_exec_entity[direct].cc_waiting); + sleepq_release(&cc_exec_waiting(cc, direct)); c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING); CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p", c, c->c_func, c->c_arg); - if ((c->c_flags & CALLOUT_PROCESSED) == 0) { - if (cc->cc_exec_next_dir == c) - cc->cc_exec_next_dir = LIST_NEXT(c, c_links.le); - LIST_REMOVE(c, c_links.le); - } else - TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); + if (not_on_a_list == 0) { + if ((c->c_flags & CALLOUT_PROCESSED) == 0) { + if (cc_exec_next(cc, direct) == c) + cc_exec_next(cc, direct) = LIST_NEXT(c, c_links.le); + LIST_REMOVE(c, c_links.le); + } else + TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe); + } callout_cc_del(c, cc); - CC_UNLOCK(cc); return (1); } Index: sys/modules/Makefile =================================================================== --- sys/modules/Makefile +++ sys/modules/Makefile @@ -62,6 +62,7 @@ cam \ ${_canbepm} \ ${_canbus} \ + ${_callout_test} \ ${_cardbus} \ ${_carp} \ cas \ @@ -386,6 +387,10 @@ _autofs= autofs .endif +.if ${MK_TESTKERN} != "no" || defined(ALL_MODULES) +_callout_test= callout_test +.endif + .if ${MK_CRYPT} != "no" || defined(ALL_MODULES) .if exists(${.CURDIR}/../opencrypto) _crypto= crypto Index: sys/modules/callout_test/Makefile =================================================================== --- sys/modules/callout_test/Makefile +++ sys/modules/callout_test/Makefile @@ -0,0 +1,15 @@ +# +# $FreeBSD$ +# + +.PATH: ${.CURDIR} + +KMOD= callout_test +SRCS= callout_test.c + +# +# Enable full debugging +# +#CFLAGS += -g + +.include Index: sys/modules/callout_test/callout_test.c =================================================================== --- sys/modules/callout_test/callout_test.c +++ sys/modules/callout_test/callout_test.c @@ -0,0 +1,274 @@ +/*- + * Copyright (c) 2015 Hans Petter Selasky. All rights reserved. + * 2015 Netflix Inc. All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef SMP +#include +#else +#define cpu_spinwait() +#endif + + +static volatile int callouttest_test; +static volatile int callouttest_timo_for_counter; +static struct callout test_co; + +struct mtx ca_mtx_test; +struct mtx ca_print_mtx_test; +uint64_t callout_counting_fun=0; +uint64_t callout_counting_fun_mig=0; +uint64_t callout_counting_fun_ab=0; +uint64_t callout_total_enters=0; +uint64_t callout_sees_freed_mem=0; +sbintime_t callout_x_time; +sbintime_t callout_f_time; +int mem_freed=0; + +#define callout_migrating(c) ((c)->c_flags & CALLOUT_DFRMIGRATION) + +static void +test_callout(void *arg) +{ + struct kern_test *kt; + struct callout_test *u; + + + kt = (struct kern_test *)arg; + u = (struct callout_test *)&kt->test_options; + callout_x_time = sbinuptime(); + callout_total_enters++; + callouttest_timo_for_counter = u->sim_intr_delay; + callouttest_test = 1; + /* simulate fast interrupt affecting timing */ + while (callouttest_test != 2) { + cpu_spinwait(); + callouttest_timo_for_counter--; + if (callouttest_timo_for_counter <= 0) { + break; + } + } + mtx_lock(&ca_mtx_test); + if (callout_pending(&test_co) || + !callout_active(&test_co)) { + callout_counting_fun_ab++; + if (callout_migrating(&test_co)) { + callout_counting_fun_mig++; + } + callout_f_time = sbinuptime(); + callouttest_test = 3; + mtx_unlock(&ca_mtx_test); + return; + } + callout_deactivate(&test_co); + callout_counting_fun++; + if (mem_freed) { + callout_sees_freed_mem++; + } + callout_f_time = sbinuptime(); + callouttest_test = 4; + mtx_unlock(&ca_mtx_test); +} + + + +static void +run_callout_test(struct kern_test *test) +{ + struct callout_test *u; + int test_number; + unsigned int y, rv; + int test_cnt=0; + uint64_t rst_cnt, stop_cnt, drain_cnt; + int dst_cpu; + rst_cnt = stop_cnt = drain_cnt = 0; + /* + * Set this to different things if you want, we + * it the same for predicatbility between runs. + */ + u = (struct callout_test *)&test->test_options; + test_number = test->test_number; + if (test_number < 1) { + outofit: + printf("Test %d unrecognized\n", test_number); + return; + } + if (test_number > 4) { + goto outofit; + } +do_it_again: + mtx_lock(&ca_mtx_test); + callouttest_test = 0; + mem_freed = 0; + callout_reset(&test_co, 1, &test_callout, test); + mtx_unlock(&ca_mtx_test); + while (callouttest_test == 0) { + cpu_spinwait(); + } + callouttest_test = 2; + for (y = 0; y != u->num_loop_iterations; y++) { + if ((test_number == 1) || (test_number == 2)) { + mtx_lock(&ca_mtx_test); + if (test_number == 1) { + dst_cpu = 0; + } else { + /* test 2 */ + dst_cpu = y % mp_ncpus; + } + mem_freed = 0; + callout_reset_on(&test_co, 1, &test_callout, test, dst_cpu); + rst_cnt++; + mtx_unlock(&ca_mtx_test); + } else if ((test_number == 3) || (test_number == 4)) { + rv = random(); + mtx_lock(&ca_mtx_test); + if (test_number == 3) { + dst_cpu = 0; + } else { + /* Test 4 */ + dst_cpu = y % mp_ncpus; + } + if ((rv % 4) == 0) { + mem_freed = 0; + callout_reset_on(&test_co, 1, &test_callout, test, dst_cpu); + rst_cnt++; + } else if ((rv % 4) == 1) { + callout_stop(&test_co); + stop_cnt++; + mem_freed = 1; + } else if ((rv % 4) == 2) { + mtx_unlock(&ca_mtx_test); + callout_drain(&test_co); + mem_freed = 1; + mtx_lock(&ca_mtx_test); + drain_cnt++; + } else if ((rv % 4) == 3) { + mem_freed = 0; + callout_reset_on(&test_co, 1, &test_callout, test, dst_cpu); + rst_cnt++; + callout_reset_on(&test_co, 1, &test_callout, test, ((dst_cpu+1)%mp_ncpus)); + rst_cnt++; + } + mtx_unlock(&ca_mtx_test); + } + } + callout_drain(&test_co); + mem_freed = 1; + drain_cnt++; + mtx_lock(&ca_print_mtx_test); + test_cnt++; + if (test_cnt < u->num_of_times) { + printf("Looping around again for test number:%d\n", + test_cnt); + mtx_unlock(&ca_print_mtx_test); + goto do_it_again; + } + printf("Callout testing complete\ntotentry:%zu trigok:%zu trigab:%zu (mig:%zu sfm:%zu)\n", + callout_total_enters, + callout_counting_fun, + callout_counting_fun_ab, + callout_counting_fun_mig, + callout_sees_freed_mem); + printf("resets:%zu stops:%zu drains:%zu\n", + rst_cnt, + stop_cnt, + drain_cnt); + mtx_unlock(&ca_print_mtx_test); +} + +int callout_test_is_loaded=0; + +static int +callout_test_modevent(module_t mod, int type, void *data) +{ + int err=0; + u_long random_seed; + + switch (type) { + case MOD_LOAD: + err = kern_testframework_register("callout_test", + run_callout_test); + if (err) { + printf("Can't load callout_test err:%d returned\n", + err); + } else { + callout_init(&test_co, CALLOUT_MPSAFE); + mtx_init(&ca_mtx_test, "ca_test_mtx", "ctm", MTX_DEF); + mtx_init(&ca_print_mtx_test, "ca__print_mtx", "cptm", MTX_DEF); + callout_test_is_loaded = 1; + random_seed = 1; + srandom(random_seed); + } + break; + case MOD_QUIESCE: + err = kern_testframework_deregister("callout_test"); + if (err == 0) { + callout_test_is_loaded = 0; + mtx_destroy(&ca_mtx_test); + mtx_destroy(&ca_print_mtx_test); + } + break; + case MOD_UNLOAD: + if (callout_test_is_loaded) { + err = kern_testframework_deregister("callout_test"); + if (err == 0) { + callout_test_is_loaded = 0; + mtx_destroy(&ca_mtx_test); + mtx_destroy(&ca_print_mtx_test); + } + } + break; + default: + return (EOPNOTSUPP); + } + return (err); +} + +static moduledata_t callout_test_mod = { + .name = "callout_test", + .evhand = callout_test_modevent, + .priv = 0 +}; +DECLARE_MODULE(callout_test, callout_test_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); Index: sys/sys/callout.h =================================================================== --- sys/sys/callout.h +++ sys/sys/callout.h @@ -64,6 +64,7 @@ #ifdef _KERNEL #define callout_active(c) ((c)->c_flags & CALLOUT_ACTIVE) +#define callout_migrating(c) ((c)->c_flags & CALLOUT_DFRMIGRATION) #define callout_deactivate(c) ((c)->c_flags &= ~CALLOUT_ACTIVE) #define callout_drain(c) _callout_stop_safe(c, 1) void callout_init(struct callout *, int); Index: sys/sys/callout_test.h =================================================================== --- sys/sys/callout_test.h +++ sys/sys/callout_test.h @@ -0,0 +1,16 @@ +#ifndef __callout_test_h__ +#define __callout_test_h__ + +/* + * The default test RRS was using was: + * sim_intr_delay = 10,000,000 + * num_loop_iterations = 10,000,000- + * num_of_times = 1,000 + */ +struct callout_test { + uint32_t sim_intr_delay; /* Max Number of spin cycles to simulate int delay */ + uint32_t num_loop_iterations; /* Number of loops to do the callout in */ + uint32_t num_of_times; /* Number of times to run entire test */ +}; + +#endif Index: sys/sys/kern_testfrwk.h =================================================================== --- sys/sys/kern_testfrwk.h +++ sys/sys/kern_testfrwk.h @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 2015 + * Netflix Incorporated, All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#ifndef _SYS_KERN_TESTFRWKT_H_ +#define _SYS_KERN_TESTFRWKT_H_ + +#define TEST_NAME_LEN 32 +#define TEST_OPTION_SPACE 256 + +struct kern_test { + char name[TEST_NAME_LEN]; + int test_number; + int num_threads; + uint8_t test_options[TEST_OPTION_SPACE]; +}; + +#ifdef _KERNEL +int kern_testframework_register(const char *name, + void (*kerntfunc)(struct kern_test *)); + +int kern_testframework_deregister(const char *name); +#endif +#endif