Changeset View
Changeset View
Standalone View
Standalone View
sys/kern/kern_timeout.c
Show First 20 Lines • Show All 157 Lines • ▼ Show 20 Lines | |||||
* There is one struct callout_cpu per cpu, holding all relevant | * There is one struct callout_cpu per cpu, holding all relevant | ||||
* state for the callout processing thread on the individual CPU. | * state for the callout processing thread on the individual CPU. | ||||
*/ | */ | ||||
struct callout_cpu { | struct callout_cpu { | ||||
struct mtx_padalign cc_lock; | struct mtx_padalign cc_lock; | ||||
struct cc_exec cc_exec_entity[2]; | struct cc_exec cc_exec_entity[2]; | ||||
struct callout *cc_next; | struct callout *cc_next; | ||||
struct callout *cc_callout; | struct callout *cc_callout; | ||||
struct callout_list *cc_callwheel; | struct callout_list *cc_callwheel_high; | ||||
struct callout_list *cc_callwheel_low; | |||||
struct callout_tailq cc_expireq; | struct callout_tailq cc_expireq; | ||||
struct callout_slist cc_callfree; | struct callout_slist cc_callfree; | ||||
sbintime_t cc_firstevent; | sbintime_t cc_firstevent; | ||||
sbintime_t cc_firstopt; | |||||
sbintime_t cc_lastscan; | sbintime_t cc_lastscan; | ||||
void *cc_cookie; | void *cc_cookie; | ||||
u_int cc_bucket; | #ifdef KTR | ||||
u_int cc_inited; | |||||
char cc_ktr_event_name[20]; | char cc_ktr_event_name[20]; | ||||
#endif | |||||
bool cc_inited; | |||||
bool cc_running; | |||||
}; | }; | ||||
#define callout_migrating(c) ((c)->c_iflags & CALLOUT_DFRMIGRATION) | #define callout_migrating(c) ((c)->c_iflags & CALLOUT_DFRMIGRATION) | ||||
#define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr | #define cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr | ||||
#define cc_exec_drain(cc, dir) cc->cc_exec_entity[dir].cc_drain | #define cc_exec_drain(cc, dir) cc->cc_exec_entity[dir].cc_drain | ||||
#define cc_exec_next(cc) cc->cc_next | #define cc_exec_next(cc) cc->cc_next | ||||
#define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel | #define cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel | ||||
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
callout_cpu_init(struct callout_cpu *cc, int cpu) | callout_cpu_init(struct callout_cpu *cc, int cpu) | ||||
{ | { | ||||
struct callout *c; | struct callout *c; | ||||
int i; | int i; | ||||
mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE); | mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE); | ||||
SLIST_INIT(&cc->cc_callfree); | SLIST_INIT(&cc->cc_callfree); | ||||
cc->cc_inited = 1; | cc->cc_inited = true; | ||||
cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize, | cc->cc_running = false; | ||||
M_CALLOUT, M_WAITOK); | cc->cc_callwheel_high = malloc(sizeof(struct callout_list) * | ||||
for (i = 0; i < callwheelsize; i++) | callwheelsize, M_CALLOUT, M_WAITOK); | ||||
LIST_INIT(&cc->cc_callwheel[i]); | cc->cc_callwheel_low = malloc(sizeof(struct callout_list) * | ||||
callwheelsize, M_CALLOUT, M_WAITOK); | |||||
for (i = 0; i < callwheelsize; i++) { | |||||
LIST_INIT(&cc->cc_callwheel_high[i]); | |||||
LIST_INIT(&cc->cc_callwheel_low[i]); | |||||
} | |||||
TAILQ_INIT(&cc->cc_expireq); | TAILQ_INIT(&cc->cc_expireq); | ||||
cc->cc_firstevent = SBT_MAX; | cc->cc_firstevent = SBT_MAX; | ||||
for (i = 0; i < 2; i++) | for (i = 0; i < 2; i++) | ||||
cc_cce_cleanup(cc, i); | cc_cce_cleanup(cc, i); | ||||
#ifdef KTR | |||||
snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name), | snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name), | ||||
"callwheel cpu %d", cpu); | "callwheel cpu %d", cpu); | ||||
#endif | |||||
if (cc->cc_callout == NULL) /* Only BSP handles timeout(9) */ | if (cc->cc_callout == NULL) /* Only BSP handles timeout(9) */ | ||||
return; | return; | ||||
for (i = 0; i < ncallout; i++) { | for (i = 0; i < ncallout; i++) { | ||||
c = &cc->cc_callout[i]; | c = &cc->cc_callout[i]; | ||||
callout_init(c, 0); | callout_init(c, 0); | ||||
c->c_iflags = CALLOUT_LOCAL_ALLOC; | c->c_iflags = CALLOUT_LOCAL_ALLOC; | ||||
SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); | SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | |||||
static inline u_int | static inline u_int | ||||
callout_hash(sbintime_t sbt) | callout_hash(sbintime_t sbt) | ||||
{ | { | ||||
return (sbt >> (32 - CC_HASH_SHIFT)); | return (sbt >> (32 - CC_HASH_SHIFT)); | ||||
} | } | ||||
#define CC_BUCKET_WIDTH (1LLU << (32 - CC_HASH_SHIFT)) | |||||
#define CC_BUCKET_MASK (CC_BUCKET_WIDTH - 1) | |||||
static inline u_int | static inline u_int | ||||
callout_get_bucket(sbintime_t sbt) | callout_get_bucket(sbintime_t sbt) | ||||
{ | { | ||||
return (callout_hash(sbt) & callwheelmask); | return (callout_hash(sbt) & callwheelmask); | ||||
} | } | ||||
static inline sbintime_t | |||||
callout_trunc_time(sbintime_t sbt) | |||||
{ | |||||
return (sbt & (0xffffffffffffffffLLU << (32 - CC_HASH_SHIFT))); | |||||
} | |||||
#ifdef CALLOUT_PROFILING | |||||
static inline struct callout * | |||||
callout_process_single(struct callout_cpu *cc, struct callout *c, int bucket, | |||||
int *depth_dir, int *mpcalls_dir, *lockcalls_dir) | |||||
#else | |||||
static inline struct callout * | |||||
callout_process_single(struct callout_cpu *cc, struct callout *c, int bucket) | |||||
#endif | |||||
{ | |||||
struct callout *rv; | |||||
CC_LOCK_ASSERT(cc); | |||||
/* | |||||
* Consumer told us the callout may be run | |||||
* directly from hardware interrupt context. | |||||
*/ | |||||
if (c->c_iflags & CALLOUT_DIRECT) { | |||||
#ifdef CALLOUT_PROFILING | |||||
++(*depth_dir); | |||||
#endif | |||||
cc_exec_next(cc) = LIST_NEXT(c, c_links.le); | |||||
LIST_REMOVE(c, c_links.le); | |||||
softclock_call_cc(c, cc, | |||||
#ifdef CALLOUT_PROFILING | |||||
mpcalls_dir, lockcalls_dir, NULL, | |||||
#endif | |||||
1); | |||||
rv = cc_exec_next(cc); | |||||
cc_exec_next(cc) = NULL; | |||||
} else { | |||||
rv = LIST_NEXT(c, c_links.le); | |||||
LIST_REMOVE(c, c_links.le); | |||||
TAILQ_INSERT_TAIL(&cc->cc_expireq, c, c_links.tqe); | |||||
c->c_iflags |= CALLOUT_PROCESSED; | |||||
} | |||||
return (rv); | |||||
} | |||||
void | void | ||||
callout_process(sbintime_t now) | callout_process(sbintime_t now) | ||||
{ | { | ||||
struct callout *tmp, *tmpn; | struct callout *tmp; | ||||
struct callout_cpu *cc; | struct callout_cpu *cc; | ||||
struct callout_list *sc; | struct callout_list *sc; | ||||
sbintime_t first, last, max, tmp_max; | sbintime_t first, max; | ||||
uint32_t lookahead; | uint32_t lookahead; | ||||
u_int firstb, lastb, nowb; | u_int firstb, nowb; | ||||
#ifdef CALLOUT_PROFILING | #ifdef CALLOUT_PROFILING | ||||
int depth_dir = 0, mpcalls_dir = 0, lockcalls_dir = 0; | int depth_dir = 0, mpcalls_dir = 0, lockcalls_dir = 0; | ||||
#endif | #endif | ||||
bool done; | |||||
cc = CC_SELF(); | cc = CC_SELF(); | ||||
mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); | mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); | ||||
/* Compute the buckets of the last scan and present times. */ | /* Compute the buckets of the last scan and present times. */ | ||||
firstb = callout_hash(cc->cc_lastscan); | firstb = callout_hash(cc->cc_lastscan); | ||||
cc->cc_lastscan = now; | first = callout_trunc_time(cc->cc_lastscan); | ||||
cc->cc_lastscan = now + 1; | |||||
nowb = callout_hash(now); | nowb = callout_hash(now); | ||||
/* Compute the last bucket and minimum time of the bucket after it. */ | /* Compute the last bucket and minimum time of the bucket after it. */ | ||||
if (nowb == firstb) | if (nowb == firstb) | ||||
lookahead = (SBT_1S / 16); | lookahead = (SBT_1S / 16); | ||||
else if (nowb - firstb == 1) | else if (nowb - firstb == 1) | ||||
lookahead = (SBT_1S / 8); | lookahead = (SBT_1S / 8); | ||||
else | else | ||||
lookahead = (SBT_1S / 2); | lookahead = (SBT_1S / 2); | ||||
first = last = now; | cc->cc_firstopt = now + (lookahead / 2); | ||||
first += (lookahead / 2); | cc->cc_firstevent = callout_trunc_time(now + lookahead); | ||||
last += lookahead; | max = cc->cc_firstevent - 1; | ||||
last &= (0xffffffffffffffffLLU << (32 - CC_HASH_SHIFT)); | cc->cc_running = true; | ||||
lastb = callout_hash(last) - 1; | |||||
max = last; | |||||
/* | /* | ||||
* Check if we wrapped around the entire wheel from the last scan. | * Deal with a case where we have been asleep so long we | ||||
* In case, we need to scan entirely the wheel for pending callouts. | * need to scan all the buckets. | ||||
* | |||||
* If the space between first and now wraps, we clip first to the | |||||
* minimum value that avoids a wrap, and clip max to the maximum | |||||
* value from the same bucket as now. We then schedule an event to | |||||
* fire no later than the start of the next bucket. | |||||
* | |||||
* However, if there is no wrap between first and now, but only | |||||
* between first and max, we clip max at the maximum value that | |||||
* avoids a wrap. | |||||
*/ | */ | ||||
if (lastb - firstb >= callwheelsize) { | if (cc->cc_firstevent - first > callwheelsize * CC_BUCKET_WIDTH) { | ||||
lastb = firstb + callwheelsize - 1; | if (first + ((callwheelsize - 1) * CC_BUCKET_WIDTH) < | ||||
if (nowb - firstb >= callwheelsize) | callout_trunc_time(now)) { | ||||
nowb = lastb; | /* The space between first and now wraps. */ | ||||
first = callout_trunc_time(now) - | |||||
((callwheelsize - 1) * CC_BUCKET_WIDTH); | |||||
max = callout_trunc_time(now) + CC_BUCKET_MASK; | |||||
cc->cc_firstopt = cc->cc_firstevent = max + 1; | |||||
} else { | |||||
/* The space between first and last wraps. */ | |||||
cc->cc_firstevent = | |||||
first + (callwheelsize * CC_BUCKET_WIDTH); | |||||
max = cc->cc_firstevent - 1; | |||||
if (cc->cc_firstevent < cc->cc_firstopt) | |||||
cc->cc_firstopt = cc->cc_firstevent; | |||||
} | } | ||||
} | |||||
KASSERT(first <= now && now <= max && first < max, | |||||
("%s: time mismatch: first=%jd, now=%jd, max=%jd", | |||||
__func__, (intmax_t)first, (intmax_t)now, (intmax_t)max)); | |||||
/* Iterate callwheel from firstb to nowb and then up to lastb. */ | /* Iterate callwheel from first to now and then up to max. */ | ||||
done = false; | |||||
do { | do { | ||||
sc = &cc->cc_callwheel[firstb & callwheelmask]; | firstb = callout_get_bucket(first); | ||||
KASSERT(first == callout_trunc_time(first), | |||||
("%s: Unexpected first time (%jx)", __func__, | |||||
(intmax_t)first)); | |||||
/* Run high-precision callouts. */ | |||||
sc = &cc->cc_callwheel_high[firstb]; | |||||
tmp = LIST_FIRST(sc); | tmp = LIST_FIRST(sc); | ||||
while (tmp != NULL) { | while (tmp != NULL) { | ||||
/* Run the callout if present time within allowed. */ | /* Run the callout if present time within allowed. */ | ||||
if (tmp->c_time <= now) { | if (tmp->c_time <= now) { | ||||
/* | tmp = callout_process_single(cc, tmp, | ||||
* Consumer told us the callout may be run | firstb | ||||
* directly from hardware interrupt context. | |||||
*/ | |||||
if (tmp->c_iflags & CALLOUT_DIRECT) { | |||||
#ifdef CALLOUT_PROFILING | #ifdef CALLOUT_PROFILING | ||||
++depth_dir; | , &depth_dir, &mpcalls_dir, &lockcalls_dir | ||||
#endif | #endif | ||||
cc_exec_next(cc) = | ); | ||||
LIST_NEXT(tmp, c_links.le); | |||||
cc->cc_bucket = firstb & callwheelmask; | |||||
LIST_REMOVE(tmp, c_links.le); | |||||
softclock_call_cc(tmp, cc, | |||||
#ifdef CALLOUT_PROFILING | |||||
&mpcalls_dir, &lockcalls_dir, NULL, | |||||
#endif | |||||
1); | |||||
tmp = cc_exec_next(cc); | |||||
cc_exec_next(cc) = NULL; | |||||
} else { | |||||
tmpn = LIST_NEXT(tmp, c_links.le); | |||||
LIST_REMOVE(tmp, c_links.le); | |||||
TAILQ_INSERT_TAIL(&cc->cc_expireq, | |||||
tmp, c_links.tqe); | |||||
tmp->c_iflags |= CALLOUT_PROCESSED; | |||||
tmp = tmpn; | |||||
} | |||||
continue; | continue; | ||||
} | } | ||||
/* Skip events from distant future. */ | /* Skip events from distant future. */ | ||||
if (tmp->c_time >= max) | if (tmp->c_time > max) | ||||
goto next; | goto next_callout; | ||||
/* | /* | ||||
* Event minimal time is bigger than present maximal | * Event minimal time is bigger than present maximal | ||||
* time, so it cannot be aggregated. | * time, so it cannot be aggregated. | ||||
*/ | */ | ||||
if (tmp->c_time > last) { | if (tmp->c_time > cc->cc_firstevent) { | ||||
lastb = nowb; | done = true; | ||||
goto next; | goto next_callout; | ||||
} | } | ||||
/* Update first and last time, respecting this event. */ | /* Update first and last time, respecting this event. */ | ||||
if (tmp->c_time < first) | if (tmp->c_time < cc->cc_firstopt) | ||||
first = tmp->c_time; | cc->cc_firstopt = tmp->c_time; | ||||
tmp_max = tmp->c_time + tmp->c_precision; | if (tmp->c_deadline < cc->cc_firstevent) | ||||
if (tmp_max < last) | cc->cc_firstevent = tmp->c_deadline; | ||||
last = tmp_max; | next_callout: | ||||
next: | |||||
tmp = LIST_NEXT(tmp, c_links.le); | tmp = LIST_NEXT(tmp, c_links.le); | ||||
} | } | ||||
/* | |||||
* We only walk the low-precision callout list when it is | |||||
* actually time to execute it. If it isn't time to execute | |||||
* the bucket yet, we'll simply check to see if it has items | |||||
* that need to execute. | |||||
*/ | |||||
sc = &cc->cc_callwheel_low[firstb]; | |||||
if (first > now) { | |||||
/* | |||||
* Rather than walking the items in the bucket, | |||||
* we'll just check that there is something in | |||||
* the bucket. If so, we'll schedule the | |||||
* callout to fire using the bucket's range. | |||||
* | |||||
* If we find a low-precision bucket, we can | |||||
* stop further processing. (Things in later | |||||
* buckets will necessarily have later times | |||||
* than these.) | |||||
*/ | |||||
if (LIST_FIRST(sc) != NULL) { | |||||
if (first < cc->cc_firstopt) | |||||
cc->cc_firstopt = first; | |||||
if (first + CC_BUCKET_MASK < cc->cc_firstevent) | |||||
cc->cc_firstevent = | |||||
first + CC_BUCKET_MASK; | |||||
done = true; | |||||
} | |||||
goto next_bucket; | |||||
} | |||||
/* Run low-precision callouts. */ | |||||
tmp = LIST_FIRST(sc); | |||||
while (tmp != NULL) { | |||||
/* | |||||
* Run the callout if present time within allowed. | |||||
* Note that callouts scheduled very far in the | |||||
* future may not be within range yet. Hence, our | |||||
* need to check the time, even for low-precision | |||||
* timers. | |||||
*/ | |||||
if (tmp->c_time <= now) { | |||||
tmp = callout_process_single(cc, tmp, | |||||
firstb | |||||
#ifdef CALLOUT_PROFILING | |||||
, &depth_dir, &mpcalls_dir, &lockcalls_dir | |||||
#endif | |||||
); | |||||
} else | |||||
tmp = LIST_NEXT(tmp, c_links.le); | |||||
} | |||||
next_bucket: | |||||
/* Proceed with the next bucket. */ | /* Proceed with the next bucket. */ | ||||
firstb++; | first += CC_BUCKET_WIDTH; | ||||
/* | /* | ||||
* Stop if we looked after present time and found | * Stop if we looked after present time and found | ||||
* some event we can't execute at now. | * some event we can't execute at now. | ||||
* Stop if we looked far enough into the future. | * Stop if we looked far enough into the future. | ||||
*/ | */ | ||||
} while (((int)(firstb - lastb)) <= 0); | } while (first <= now || (first < max && !done)); | ||||
cc->cc_firstevent = last; | cc->cc_running = false; | ||||
#ifndef NO_EVENTTIMERS | #ifndef NO_EVENTTIMERS | ||||
cpu_new_callout(curcpu, last, first); | cpu_new_callout(curcpu, cc->cc_firstevent, cc->cc_firstopt); | ||||
#endif | #endif | ||||
#ifdef CALLOUT_PROFILING | #ifdef CALLOUT_PROFILING | ||||
avg_depth_dir += (depth_dir * 1000 - avg_depth_dir) >> 8; | avg_depth_dir += (depth_dir * 1000 - avg_depth_dir) >> 8; | ||||
avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8; | avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8; | ||||
avg_lockcalls_dir += (lockcalls_dir * 1000 - avg_lockcalls_dir) >> 8; | avg_lockcalls_dir += (lockcalls_dir * 1000 - avg_lockcalls_dir) >> 8; | ||||
#endif | #endif | ||||
mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); | mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET); | ||||
/* | /* | ||||
Show All 31 Lines | |||||
static void | static void | ||||
callout_cc_add(struct callout *c, struct callout_cpu *cc, | callout_cc_add(struct callout *c, struct callout_cpu *cc, | ||||
sbintime_t sbt, sbintime_t precision, void (*func)(void *), | sbintime_t sbt, sbintime_t precision, void (*func)(void *), | ||||
void *arg, int cpu, int flags) | void *arg, int cpu, int flags) | ||||
{ | { | ||||
int bucket; | int bucket; | ||||
CC_LOCK_ASSERT(cc); | CC_LOCK_ASSERT(cc); | ||||
if (sbt < cc->cc_lastscan) | |||||
sbt = cc->cc_lastscan; | |||||
c->c_arg = arg; | c->c_arg = arg; | ||||
c->c_iflags |= CALLOUT_PENDING; | c->c_iflags |= CALLOUT_PENDING; | ||||
c->c_iflags &= ~CALLOUT_PROCESSED; | c->c_iflags &= ~CALLOUT_PROCESSED; | ||||
c->c_flags |= CALLOUT_ACTIVE; | c->c_flags |= CALLOUT_ACTIVE; | ||||
if (flags & C_DIRECT_EXEC) | if (flags & C_DIRECT_EXEC) | ||||
c->c_iflags |= CALLOUT_DIRECT; | c->c_iflags |= CALLOUT_DIRECT; | ||||
c->c_func = func; | c->c_func = func; | ||||
/* | |||||
* Set the time and deadline, but ensure they are no sooner | |||||
* than the first callout bucket that callout_process() will | |||||
* hit the next time it runs. | |||||
*/ | |||||
if (sbt >= cc->cc_lastscan) | |||||
c->c_time = sbt; | c->c_time = sbt; | ||||
else | |||||
c->c_time = cc->cc_lastscan; | |||||
if (SBT_MAX - sbt < precision) | |||||
c->c_deadline = SBT_MAX; | |||||
else { | |||||
c->c_deadline = sbt + precision; | |||||
if (c->c_deadline < cc->cc_lastscan) | |||||
c->c_deadline = cc->cc_lastscan; | |||||
} | |||||
c->c_precision = precision; | c->c_precision = precision; | ||||
bucket = callout_get_bucket(c->c_time); | |||||
CTR3(KTR_CALLOUT, "precision set for %p: %d.%08x", | CTR3(KTR_CALLOUT, "precision set for %p: %d.%08x", | ||||
c, (int)(c->c_precision >> 32), | c, (int)(c->c_precision >> 32), | ||||
(u_int)(c->c_precision & 0xffffffff)); | (u_int)(c->c_precision & 0xffffffff)); | ||||
LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le); | /* | ||||
if (cc->cc_bucket == bucket) | * If the precision is high enough that the time through the | ||||
cc_exec_next(cc) = c; | * deadline (time + precision) completely straddle a low-precisions | ||||
* bucket, then use a low-precision bucket. Otherwise, leave it in | |||||
* the high-precision buckets. | |||||
* | |||||
* There are three cases: | |||||
* 1. The time is now bucket, and the deadline is after the | |||||
* current bucket. In this case, the callout can live | |||||
* in the low-precision bucket. | |||||
* 2. The time is in a future bucket, but the entire space of a | |||||
* low-precision bucket is fully enclosed in the runnable range | |||||
* of the callout. In this case, the callout can live in that | |||||
* low-precision bucket. | |||||
* 3. Any other case, the callout must live in a high-precision | |||||
* bucket. | |||||
*/ | |||||
bucket = callout_get_bucket(c->c_time); | |||||
sbt = roundup2(c->c_time, CC_BUCKET_WIDTH); | |||||
if (c->c_time == cc->cc_lastscan && c->c_deadline >= | |||||
callout_trunc_time(cc->cc_lastscan) + CC_BUCKET_MASK) { | |||||
c->c_deadline = callout_trunc_time(cc->cc_lastscan) + | |||||
CC_BUCKET_MASK; | |||||
LIST_INSERT_HEAD(&cc->cc_callwheel_low[bucket], c, c_links.le); | |||||
} else if (c->c_deadline >= sbt + CC_BUCKET_MASK) { | |||||
c->c_time = sbt; | |||||
c->c_deadline = sbt + CC_BUCKET_MASK; | |||||
bucket = callout_get_bucket(c->c_time); | |||||
LIST_INSERT_HEAD(&cc->cc_callwheel_low[bucket], c, c_links.le); | |||||
} else | |||||
LIST_INSERT_HEAD(&cc->cc_callwheel_high[bucket], c, c_links.le); | |||||
#ifndef NO_EVENTTIMERS | #ifndef NO_EVENTTIMERS | ||||
/* | /* | ||||
* Inform the eventtimers(4) subsystem there's a new callout | * If callout_process() is currently running, we only need to | ||||
* feed our timings into its calculations. It will schedule the | |||||
* timer when done. | |||||
* | |||||
* Otherwise, inform the eventtimers(4) subsystem there's a new callout | |||||
* that has been inserted, but only if really required. | * that has been inserted, but only if really required. | ||||
*/ | */ | ||||
if (SBT_MAX - c->c_time < c->c_precision) | if (c->c_time < cc->cc_firstopt) | ||||
c->c_precision = SBT_MAX - c->c_time; | cc->cc_firstopt = c->c_time; | ||||
sbt = c->c_time + c->c_precision; | if (c->c_deadline < cc->cc_firstevent) { | ||||
if (sbt < cc->cc_firstevent) { | cc->cc_firstevent = c->c_deadline; | ||||
cc->cc_firstevent = sbt; | if (!cc->cc_running) | ||||
cpu_new_callout(cpu, sbt, c->c_time); | cpu_new_callout(cpu, cc->cc_firstevent, | ||||
cc->cc_firstopt); | |||||
} | } | ||||
#endif | #endif | ||||
} | } | ||||
static void | static void | ||||
callout_cc_del(struct callout *c, struct callout_cpu *cc) | callout_cc_del(struct callout *c, struct callout_cpu *cc) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 389 Lines • ▼ Show 20 Lines | callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t prec, | ||||
sbintime_t to_sbt, precision; | sbintime_t to_sbt, precision; | ||||
struct callout_cpu *cc; | struct callout_cpu *cc; | ||||
int cancelled, direct; | int cancelled, direct; | ||||
int ignore_cpu=0; | int ignore_cpu=0; | ||||
cancelled = 0; | cancelled = 0; | ||||
if (cpu == -1) { | if (cpu == -1) { | ||||
ignore_cpu = 1; | ignore_cpu = 1; | ||||
} else if ((cpu >= MAXCPU) || | } else if (cpu >= MAXCPU || !(CC_CPU(cpu))->cc_inited) { | ||||
((CC_CPU(cpu))->cc_inited == 0)) { | |||||
/* Invalid CPU spec */ | /* Invalid CPU spec */ | ||||
panic("Invalid CPU in callout %d", cpu); | panic("Invalid CPU in callout %d", cpu); | ||||
} | } | ||||
callout_when(sbt, prec, flags, &to_sbt, &precision); | callout_when(sbt, prec, flags, &to_sbt, &precision); | ||||
/* | /* | ||||
* This flag used to be added by callout_cc_add, but the | * This flag used to be added by callout_cc_add, but the | ||||
* first time you call this we could end up with the | * first time you call this we could end up with the | ||||
▲ Show 20 Lines • Show All 490 Lines • ▼ Show 20 Lines | flssbt(sbintime_t sbt) | ||||
sbt += (uint64_t)sbt >> 1; | sbt += (uint64_t)sbt >> 1; | ||||
if (sizeof(long) >= sizeof(sbintime_t)) | if (sizeof(long) >= sizeof(sbintime_t)) | ||||
return (flsl(sbt)); | return (flsl(sbt)); | ||||
if (sbt >= SBT_1S) | if (sbt >= SBT_1S) | ||||
return (flsl(((uint64_t)sbt) >> 32) + 32); | return (flsl(((uint64_t)sbt) >> 32) + 32); | ||||
return (flsl(sbt)); | return (flsl(sbt)); | ||||
} | } | ||||
static void | |||||
kern_callout_wheel_stats(struct callout_list *sc, int *c, sbintime_t *st, | |||||
sbintime_t *spr, sbintime_t *maxt, sbintime_t *maxpr, sbintime_t *now, | |||||
int *ct, int *cpr) | |||||
{ | |||||
struct callout *tmp; | |||||
sbintime_t t; | |||||
int cnt; | |||||
cnt = 0; | |||||
LIST_FOREACH(tmp, sc, c_links.le) { | |||||
cnt++; | |||||
t = tmp->c_time - *now; | |||||
if (t < 0) | |||||
t = 0; | |||||
*st += t / SBT_1US; | |||||
*spr += tmp->c_precision / SBT_1US; | |||||
if (t > *maxt) | |||||
*maxt = t; | |||||
if (tmp->c_precision > *maxpr) | |||||
*maxpr = tmp->c_precision; | |||||
ct[flssbt(t)]++; | |||||
cpr[flssbt(tmp->c_precision)]++; | |||||
} | |||||
*c += cnt; | |||||
} | |||||
/* | /* | ||||
* Dump immediate statistic snapshot of the scheduled callouts. | * Dump immediate statistic snapshot of the scheduled callouts. | ||||
*/ | */ | ||||
static int | static int | ||||
sysctl_kern_callout_stat(SYSCTL_HANDLER_ARGS) | sysctl_kern_callout_stat(SYSCTL_HANDLER_ARGS) | ||||
{ | { | ||||
struct callout *tmp; | |||||
struct callout_cpu *cc; | struct callout_cpu *cc; | ||||
struct callout_list *sc; | |||||
sbintime_t maxpr, maxt, medpr, medt, now, spr, st, t; | sbintime_t maxpr, maxt, medpr, medt, now, spr, st, t; | ||||
int ct[64], cpr[64], ccpbk[32]; | int ct[64], cpr[64], ccpbk[32]; | ||||
int error, val, i, count, tcum, pcum, maxc, c, medc; | int error, val, i, count, tcum, pcum, maxc, c, medc; | ||||
int chi, clo, ctmp; | |||||
#ifdef SMP | #ifdef SMP | ||||
int cpu; | int cpu; | ||||
#endif | #endif | ||||
val = 0; | val = 0; | ||||
error = sysctl_handle_int(oidp, &val, 0, req); | error = sysctl_handle_int(oidp, &val, 0, req); | ||||
if (error != 0 || req->newptr == NULL) | if (error != 0 || req->newptr == NULL) | ||||
return (error); | return (error); | ||||
count = maxc = 0; | count = maxc = 0; | ||||
st = spr = maxt = maxpr = 0; | st = spr = maxt = maxpr = 0; | ||||
bzero(ccpbk, sizeof(ccpbk)); | bzero(ccpbk, sizeof(ccpbk)); | ||||
bzero(ct, sizeof(ct)); | bzero(ct, sizeof(ct)); | ||||
bzero(cpr, sizeof(cpr)); | bzero(cpr, sizeof(cpr)); | ||||
now = sbinuptime(); | now = sbinuptime(); | ||||
#ifdef SMP | #ifdef SMP | ||||
CPU_FOREACH(cpu) { | CPU_FOREACH(cpu) { | ||||
cc = CC_CPU(cpu); | cc = CC_CPU(cpu); | ||||
#else | #else | ||||
cc = CC_CPU(timeout_cpu); | cc = CC_CPU(timeout_cpu); | ||||
#endif | #endif | ||||
chi = clo = 0; | |||||
CC_LOCK(cc); | CC_LOCK(cc); | ||||
for (i = 0; i < callwheelsize; i++) { | for (i = 0; i < callwheelsize; i++) { | ||||
sc = &cc->cc_callwheel[i]; | |||||
c = 0; | c = 0; | ||||
LIST_FOREACH(tmp, sc, c_links.le) { | kern_callout_wheel_stats(&cc->cc_callwheel_high[i], | ||||
c++; | &c, &st, &spr, &maxt, &maxpr, &now, ct, cpr); | ||||
t = tmp->c_time - now; | chi += c; | ||||
if (t < 0) | ctmp = c; | ||||
t = 0; | kern_callout_wheel_stats(&cc->cc_callwheel_low[i], | ||||
st += t / SBT_1US; | &c, &st, &spr, &maxt, &maxpr, &now, ct, cpr); | ||||
spr += tmp->c_precision / SBT_1US; | clo += c - ctmp; | ||||
if (t > maxt) | |||||
maxt = t; | |||||
if (tmp->c_precision > maxpr) | |||||
maxpr = tmp->c_precision; | |||||
ct[flssbt(t)]++; | |||||
cpr[flssbt(tmp->c_precision)]++; | |||||
} | |||||
if (c > maxc) | if (c > maxc) | ||||
maxc = c; | maxc = c; | ||||
ccpbk[fls(c + c / 2)]++; | ccpbk[fls(c + c / 2)]++; | ||||
count += c; | count += c; | ||||
} | } | ||||
CC_UNLOCK(cc); | CC_UNLOCK(cc); | ||||
#ifdef SMP | #ifdef SMP | ||||
printf("CPU %d callouts: %d/%d/%d\n", cpu, clo, chi, clo + chi); | |||||
} | } | ||||
#endif | #endif | ||||
for (i = 0, tcum = 0; i < 64 && tcum < count / 2; i++) | for (i = 0, tcum = 0; i < 64 && tcum < count / 2; i++) | ||||
tcum += ct[i]; | tcum += ct[i]; | ||||
medt = (i >= 2) ? (((sbintime_t)1) << (i - 2)) : 0; | medt = (i >= 2) ? (((sbintime_t)1) << (i - 2)) : 0; | ||||
for (i = 0, pcum = 0; i < 64 && pcum < count / 2; i++) | for (i = 0, pcum = 0; i < 64 && pcum < count / 2; i++) | ||||
pcum += cpr[i]; | pcum += cpr[i]; | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
static void | static void | ||||
_show_callout(struct callout *c) | _show_callout(struct callout *c) | ||||
{ | { | ||||
db_printf("callout %p\n", c); | db_printf("callout %p\n", c); | ||||
#define C_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, c->e); | #define C_DB_PRINTF(f, e) db_printf(" %s = " f "\n", #e, c->e); | ||||
db_printf(" &c_links = %p\n", &(c->c_links)); | db_printf(" &c_links = %p\n", &(c->c_links)); | ||||
C_DB_PRINTF("%" PRId64, c_time); | C_DB_PRINTF("%" PRId64, c_time); | ||||
C_DB_PRINTF("%" PRId64, c_precision); | C_DB_PRINTF("%" PRId64, c_deadline); | ||||
C_DB_PRINTF("%p", c_arg); | C_DB_PRINTF("%p", c_arg); | ||||
C_DB_PRINTF("%p", c_func); | C_DB_PRINTF("%p", c_func); | ||||
C_DB_PRINTF("%p", c_lock); | C_DB_PRINTF("%p", c_lock); | ||||
C_DB_PRINTF("%#x", c_flags); | C_DB_PRINTF("%#x", c_flags); | ||||
C_DB_PRINTF("%#x", c_iflags); | C_DB_PRINTF("%#x", c_iflags); | ||||
C_DB_PRINTF("%d", c_cpu); | C_DB_PRINTF("%d", c_cpu); | ||||
#undef C_DB_PRINTF | #undef C_DB_PRINTF | ||||
} | } | ||||
Show All 12 Lines |