diff --git a/sys/kern/kern_ffclock.c b/sys/kern/kern_ffclock.c --- a/sys/kern/kern_ffclock.c +++ b/sys/kern/kern_ffclock.c @@ -243,7 +243,7 @@ ffclock_bintime(struct bintime *bt) { - ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); + ffclock_abstime(NULL, bt, NULL, FFCLOCK_MONO | FFCLOCK_LEAPSEC); } void @@ -251,7 +251,7 @@ { struct bintime bt; - ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); + ffclock_abstime(NULL, &bt, NULL, FFCLOCK_MONO | FFCLOCK_LEAPSEC); bintime2timespec(&bt, tsp); } @@ -260,7 +260,7 @@ { struct bintime bt; - ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); + ffclock_abstime(NULL, &bt, NULL, FFCLOCK_MONO | FFCLOCK_LEAPSEC); bintime2timeval(&bt, tvp); } @@ -269,7 +269,7 @@ { ffclock_abstime(NULL, bt, NULL, - FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); + FFCLOCK_MONO | FFCLOCK_LEAPSEC | FFCLOCK_FAST); } void @@ -278,7 +278,7 @@ struct bintime bt; ffclock_abstime(NULL, &bt, NULL, - FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); + FFCLOCK_MONO | FFCLOCK_LEAPSEC | FFCLOCK_FAST); bintime2timespec(&bt, tsp); } @@ -288,7 +288,7 @@ struct bintime bt; ffclock_abstime(NULL, &bt, NULL, - FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); + FFCLOCK_MONO | FFCLOCK_LEAPSEC | FFCLOCK_FAST); bintime2timeval(&bt, tvp); } @@ -296,7 +296,7 @@ ffclock_binuptime(struct bintime *bt) { - ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); + ffclock_abstime(NULL, bt, NULL, FFCLOCK_MONO | FFCLOCK_UPTIME); } void @@ -304,7 +304,7 @@ { struct bintime bt; - ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); + ffclock_abstime(NULL, &bt, NULL, FFCLOCK_MONO | FFCLOCK_UPTIME); bintime2timespec(&bt, tsp); } @@ -313,7 +313,7 @@ { struct bintime bt; - ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); + ffclock_abstime(NULL, &bt, NULL, FFCLOCK_MONO | FFCLOCK_UPTIME); bintime2timeval(&bt, tvp); } @@ -322,7 +322,7 @@ { ffclock_abstime(NULL, bt, NULL, - FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); + FFCLOCK_MONO | FFCLOCK_UPTIME | FFCLOCK_FAST); } void @@ -331,7 +331,7 @@ struct bintime bt; ffclock_abstime(NULL, &bt, NULL, - FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); + FFCLOCK_MONO | FFCLOCK_UPTIME | FFCLOCK_FAST); bintime2timespec(&bt, tsp); } @@ -341,7 +341,7 @@ struct bintime bt; ffclock_abstime(NULL, &bt, NULL, - FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); + FFCLOCK_MONO | FFCLOCK_UPTIME | FFCLOCK_FAST); bintime2timeval(&bt, tvp); } diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -500,10 +500,20 @@ #ifdef FFCLOCK /* - * Support for feedforward synchronization algorithms. This is heavily inspired - * by the timehands mechanism but kept independent from it. *_windup() functions - * have some connection to avoid accessing the timecounter hardware more than - * necessary. + * Support for FFclock, the feedforward family of system clocks. + * The implementation is heavily inspired by the timehands mechanism for the + * traditional feedback-based FBclock. The implementation works in conjunction + * with, yet is quite separate from, the timehands code. The core FFclock + * function, ffclock_windup, is called from within the FBclock's tc_windup to + * avoid duplication of hardware access and tc-tick processing overheads. + * However the FFclock code does not alter or interfere with FBclock operation. + * + * Two forms of the FFclock are supported : + * native [natFFC]: kernel version of the userland FF daemon's clock. + * monotonic [monoFFC]: monotonic version of natFFC (resets aside). + * Each of these clocks correspond to UTC at the time that the FFclock + * daemon is started. Leap seconds subsequent to that time are added in + * separately. */ /* Feedforward clock estimates kept updated by the synchronization daemon. */ @@ -514,13 +524,14 @@ struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */ struct fftimehands { - struct ffclock_estimate cest; - struct bintime tick_time; - struct bintime tick_time_lerp; - ffcounter tick_ffcount; - uint64_t period_lerp; - volatile uint8_t gen; - struct fftimehands *next; + struct ffclock_estimate cest; /* natFFC data */ + struct bintime tick_time; /* natFFC */ + struct bintime tick_time_mono; /* monoFFC */ + struct bintime tick_error; + ffcounter tick_ffcount; + uint64_t period_mono; + volatile uint8_t gen; + struct fftimehands *next; }; #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x)) @@ -543,6 +554,7 @@ ffclock_updated = 0; ffclock_status = FFCLOCK_STA_UNSYNC; + ffclock_boottime.sec = ffclock_boottime.frac = 0; mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF); } @@ -584,11 +596,10 @@ } /* - * Sub-routine to convert a time interval measured in RAW counter units to time - * in seconds stored in bintime format. - * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be - * larger than the max value of u_int (on 32 bit architecture). Loop to consume - * extra cycles. + * Safely convert a time interval measured in raw counter units to time + * in seconds in bintime format. Designed for use when the time interval + * may be over one second, where ffdelta may be (32 bit architectures) + * larger than the max value of u_int argument required by bintime_mul. */ static void ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt) @@ -598,7 +609,7 @@ delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1; bintime_clear(bt); - do { + do { // loop to deal with ffdelta one safe chunk at a time if (ffdelta > delta_max) delta = delta_max; else @@ -611,156 +622,203 @@ } while (ffdelta > 0); } + /* - * Update the fftimehands. - * Push the tick ffcount and time(s) forward based on current clock estimate. - * The conversion from ffcounter to bintime relies on the difference clock - * principle, whose accuracy relies on computing small time intervals. If a new - * clock estimate has been passed by the synchronisation daemon, make it - * current, and compute the linear interpolation for monotonic time if needed. + * Update the fftimehands. The updated tick state is based on the previous tick. + * If there has been no actionable update in the FFclock parameters during the + * current tick (ffclock_updated <= 0), then each of the natFFC and monoFFC + * clocks advance linearly. Otherwise it is based off the updated + * parameters at the time of the update. The native FFclock natFFC will then + * jump, monoFFC will not (except under special conditions). + * The linear interpolation parameters of + * monoFFC ({tick_time,period}_mono) are recomputed for the new tick. + * + * The instant defining the start of the new tick is the delta=tc_delta call + * from tc_windup. This is simply mirrored here in the FF counter `read'. */ static void ffclock_windup(unsigned int delta) { struct ffclock_estimate *cest; struct fftimehands *ffth; - struct bintime bt, gap_lerp; + struct bintime bt, gap; ffcounter ffdelta; uint64_t frac; unsigned int polling; uint8_t forward_jump, ogen; - /* - * Pick the next timehand, copy current ffclock estimates and move tick - * times and counter forward. - */ - forward_jump = 0; + /* Prepare next fftimehand where tick state will be updated. */ ffth = fftimehands->next; ogen = ffth->gen; ffth->gen = 0; cest = &ffth->cest; - memcpy(cest, &fftimehands->cest, sizeof(struct ffclock_estimate)); - ffdelta = (ffcounter)delta; - ffth->period_lerp = fftimehands->period_lerp; - - ffth->tick_time = fftimehands->tick_time; - ffclock_convert_delta(ffdelta, cest->period, &bt); - bintime_add(&ffth->tick_time, &bt); - - ffth->tick_time_lerp = fftimehands->tick_time_lerp; - ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt); - bintime_add(&ffth->tick_time_lerp, &bt); + /* Move FF counter forward to existing tick start. */ + ffdelta = (ffcounter)delta; ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta; /* - * Assess the status of the clock, if the last update is too old, it is - * likely the synchronisation daemon is dead and the clock is free - * running. + * No acceptable update in FFclock parameters to process. Tick + * state update based on copy or simple projection from previous tick. */ - if (ffclock_updated == 0) { - ffdelta = ffth->tick_ffcount - cest->update_ffcount; + if (ffclock_updated <= 0) { + + /* Update natFFC members {cest, tick_time, tick_error} */ + memcpy(cest, &fftimehands->cest,sizeof(struct ffclock_estimate)); + ffth->tick_time = fftimehands->tick_time; ffclock_convert_delta(ffdelta, cest->period, &bt); - if (bt.sec > 2 * FFCLOCK_SKM_SCALE) - ffclock_status |= FFCLOCK_STA_UNSYNC; - } + bintime_add(&ffth->tick_time, &bt); + bintime_mul(&bt, cest->errb_rate * PS_AS_BINFRAC); + bintime_add(&ffth->tick_error, &bt); + + /* Update monoFFC members {period_mono, tick_time_mono} */ + ffth->period_mono = fftimehands->period_mono; + ffth->tick_time_mono = fftimehands->tick_time_mono; + ffclock_convert_delta(ffdelta, ffth->period_mono, &bt); + bintime_add(&ffth->tick_time_mono, &bt); + + /* Check if the clock status should be set to unsynchronized. + * Assessment based on age of last/current update. If the + * daemon's UNSYNC status is too stale, it is over-ridden. + */ + if (ffclock_updated == 0) { + ffdelta = ffth->tick_ffcount - cest->update_ffcount; + ffclock_convert_delta(ffdelta, cest->period, &bt); + if (bt.sec > 2 * FFCLOCK_SKM_SCALE) + ffclock_status |= FFCLOCK_STA_UNSYNC; + } + } + /* - * If available, grab updated clock estimates and make them current. - * Recompute time at this tick using the updated estimates. The clock - * estimates passed the feedforward synchronisation daemon may result - * in time conversion that is not monotonically increasing (just after - * the update). time_lerp is a particular linear interpolation over the - * synchronisation algo polling period that ensures monotonicity for the - * clock ids requesting it. + * An update in FFclock parameters is available in this tick. Generate + * the new tick state based on this, projected from the update time. */ if (ffclock_updated > 0) { + + /* Update natFFC members {cest, tick_time, tick_error} */ memcpy(cest, &ffclock_estimate,sizeof(struct ffclock_estimate)); ffdelta = ffth->tick_ffcount - cest->update_ffcount; ffth->tick_time = cest->update_time; ffclock_convert_delta(ffdelta, cest->period, &bt); bintime_add(&ffth->tick_time, &bt); + bintime_mul(&bt, cest->errb_rate * PS_AS_BINFRAC); + bintime_addx(&bt, cest->errb_abs * NS_AS_BINFRAC); + ffth->tick_error = bt; - /* ffclock_reset sets ffclock_updated to INT8_MAX */ - if (ffclock_updated == INT8_MAX) - ffth->tick_time_lerp = ffth->tick_time; + /* + * Update monoFFC member tick_time_mono, standard case. + * ffclock_updated by ffclock_setto_rtc : re-initialize + * ffclock_updated by daemon : ensure continuity across ticks + */ + if (ffclock_updated == INT8_MAX) // set by ffclock_reset_clock + ffth->tick_time_mono = ffth->tick_time; + else { + ffth->tick_time_mono = fftimehands->tick_time_mono; + ffclock_convert_delta((ffcounter)delta, + fftimehands->period_mono, &bt); + bintime_add(&ffth->tick_time_mono, &bt); + } - if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >)) + /* Record direction of jump between monoFFC and natFFC */ + if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_mono, >)) forward_jump = 1; else forward_jump = 0; - bintime_clear(&gap_lerp); - if (forward_jump) { - gap_lerp = ffth->tick_time; - bintime_sub(&gap_lerp, &ffth->tick_time_lerp); + /* Record magnitude of jump */ + bintime_clear(&gap); + if (forward_jump) { // monoFFC < natFFC + gap = ffth->tick_time; + bintime_sub(&gap, &ffth->tick_time_mono); } else { - gap_lerp = ffth->tick_time_lerp; - bintime_sub(&gap_lerp, &ffth->tick_time); + gap = ffth->tick_time_mono; + bintime_sub(&gap, &ffth->tick_time); } /* - * The reset from the RTC clock may be far from accurate, and - * reducing the gap between real time and interpolated time - * could take a very long time if the interpolated clock insists - * on strict monotonicity. The clock is reset under very strict - * conditions (kernel time is known to be wrong and - * synchronization daemon has been restarted recently. - * ffclock_boottime absorbs the jump to ensure boot time is - * correct and uptime functions stay consistent. + * Update monoFFC member tick_time_mono, exceptional case. + * Break smooth monotonicity by allowing monoFFC to jump to meet + * natFFC. Only occurs under tight conditions to prevent + * a poor monoFFC initialization from taking a very long + * time to approach natFFC. Absorb the jump into + * ffclock_boottime to ensure continuity of uptime functions. + * If the jump is forward, then monoFFC remains monotonic. */ if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) && ((cest->status & FFCLOCK_STA_UNSYNC) == 0) && ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) { - if (forward_jump) - bintime_add(&ffclock_boottime, &gap_lerp); - else - bintime_sub(&ffclock_boottime, &gap_lerp); - ffth->tick_time_lerp = ffth->tick_time; - bintime_clear(&gap_lerp); - } + if (forward_jump) { + printf("ffwindup: forward"); + bintime_add(&ffclock_boottime, &gap); + } else { + printf("ffwindup: backward"); + bintime_sub(&ffclock_boottime, &gap); + } + printf(" jump for monoFFclock of %llu.%03u", + (unsigned long long)gap.sec, + (unsigned int)(gap.frac / MS_AS_BINFRAC) ); + ffth->tick_time_mono = ffth->tick_time; - ffclock_status = cest->status; - ffth->period_lerp = cest->period; + /* Signal nothing to do to period_mono algo below. */ + bintime_clear(&gap); + } /* - * Compute corrected period used for the linear interpolation of - * time. The rate of linear interpolation is capped to 5000PPM - * (5ms/s). + * Update monoFFC member period_mono. The goal of the monoFFC + * algorithm is to reduce the gap between monoFFC and natFFC + * to zero by the next FFclock update. The reduction uses linear + * interpolation via selecting period_mono. + * To ensure rate quality, |period_mono - period| + * is capped at 5000PPM (5ms/s). If there is no gap, the clocks + * will agree throughout the new tick. */ - if (bintime_isset(&gap_lerp)) { + ffth->period_mono = cest->period; // re-initialize + + /* Keep default if no visible gap or no daemon updates yet. */ + if (bintime_isset(&gap)) { + + /* Estimate #seconds to next update. */ ffdelta = cest->update_ffcount; ffdelta -= fftimehands->cest.update_ffcount; ffclock_convert_delta(ffdelta, cest->period, &bt); polling = bt.sec; + + /* Calculate cap */ bt.sec = 0; bt.frac = 5000000 * NS_AS_BINFRAC; bintime_mul(&bt, polling); - if (bintime_cmp(&gap_lerp, &bt, >)) - gap_lerp = bt; + if (bintime_cmp(&gap, &bt, >)) + gap = bt; // gap = min(gap, bt) - /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */ + /* Store the portion of gap per cycle in frac. */ frac = 0; - if (gap_lerp.sec > 0) { + if (gap.sec > 0) { frac -= 1; - frac /= ffdelta / gap_lerp.sec; + frac /= ffdelta / gap.sec; } - frac += gap_lerp.frac / ffdelta; + frac += gap.frac / ffdelta; if (forward_jump) - ffth->period_lerp += frac; + ffth->period_mono += frac; else - ffth->period_lerp -= frac; + ffth->period_mono -= frac; } - ffclock_updated = 0; + ffclock_status = cest->status; // unsets FFCLOCK_STA_UNSYNC + ffclock_updated = 0; // signal latest update done } + + /* Bump generation of new tick, avoiding the reserved 0 value. */ if (++ogen == 0) ogen = 1; ffth->gen = ogen; + fftimehands = ffth; + } + /* * Adjust fftimehands when the timecounter is changed. * This update does not advance the tick itself (hence UTC members remain @@ -812,8 +870,9 @@ cest->status |= FFCLOCK_STA_UNSYNC; ffth->tick_time = fftimehands->tick_time; - ffth->tick_time_lerp = fftimehands->tick_time_lerp; - ffth->period_lerp = cest->period; + ffth->tick_time_mono = fftimehands->tick_time_mono; + ffth->tick_error = fftimehands->tick_error; + ffth->period_mono = cest->period; /* Do not lock but ignore next update from synchronization daemon. */ ffclock_updated--; @@ -839,15 +898,12 @@ struct fftimehands *ffth; uint8_t gen; - /* - * No locking but check generation has not changed. Also need to make - * sure ffdelta is positive, i.e. ffcount > tick_ffcount. - */ + /* No locking but check generation has not changed. */ do { ffth = fftimehands; gen = ffth->gen; - if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) - *bt = ffth->tick_time_lerp; + if ((flags & FFCLOCK_MONO) == FFCLOCK_MONO) + *bt = ffth->tick_time_mono; else *bt = ffth->tick_time; *ffcount = ffth->tick_ffcount; @@ -869,10 +925,7 @@ ffcounter ffdelta; uint8_t gen; - /* - * No locking but check generation has not changed. Also need to make - * sure ffdelta is positive, i.e. ffcount > tick_ffcount. - */ + /* No locking but check generation has not changed. */ do { ffth = fftimehands; gen = ffth->gen; @@ -881,9 +934,9 @@ else ffdelta = ffth->tick_ffcount - ffcount; - if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) { - *bt = ffth->tick_time_lerp; - ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2); + if ((flags & FFCLOCK_MONO) == FFCLOCK_MONO) { + *bt = ffth->tick_time_mono; + ffclock_convert_delta(ffdelta, ffth->period_mono, &bt2); } else { *bt = ffth->tick_time; ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2); @@ -1087,7 +1140,6 @@ struct bintime bt; unsigned int delta, gen; #ifdef FFCLOCK - ffcounter ffcount; struct ffclock_info *ffi; struct fftimehands *ffth; struct ffclock_estimate cest; @@ -1113,9 +1165,9 @@ ffi = &clock_snap->ff_info; ffth = fftimehands; ffi->tick_time = ffth->tick_time; - ffi->tick_time_lerp = ffth->tick_time_lerp; + ffi->tick_time_mono = ffth->tick_time_mono; ffi->period = ffth->cest.period; - ffi->period_lerp = ffth->period_lerp; + ffi->period_mono = ffth->period_mono; clock_snap->ffcount = ffth->tick_ffcount; cest = ffth->cest; #endif @@ -1142,12 +1194,13 @@ ffi->leapsec_adjustment -= cest.leapsec; /* Record feedforward clock status and error. */ - ffi->status = cest.status; - ffcount = clock_snap->ffcount - cest.update_ffcount; - ffclock_convert_delta(ffcount, cest.period, &bt); - bintime_mul(&bt, cest.errb_rate * PS_AS_BINFRAC); - bintime_addx(&bt, cest.errb_abs * NS_AS_BINFRAC); - ffi->error = bt; + ffi->status = cest.status; + ffi->error = ffth->tick_error; + if (!fast) { + ffclock_convert_delta((ffcounter)delta, cest.period, &bt); + bintime_mul(&bt, cest.errb_rate * PS_AS_BINFRAC); + bintime_add(&ffi->error, &bt); + } #endif } @@ -1180,9 +1233,9 @@ break; #ifdef FFCLOCK case SYSCLOCK_FF: - if (flags & FFCLOCK_LERP) { - *bt = cs->ff_info.tick_time_lerp; - period = cs->ff_info.period_lerp; + if (flags & FFCLOCK_MONO) { + *bt = cs->ff_info.tick_time_mono; + period = cs->ff_info.period_mono; } else { *bt = cs->ff_info.tick_time; period = cs->ff_info.period; diff --git a/sys/sys/timeffc.h b/sys/sys/timeffc.h --- a/sys/sys/timeffc.h +++ b/sys/sys/timeffc.h @@ -102,13 +102,13 @@ * {FB|FF}CLOCK_FAST: Do not read the hardware counter, instead using the * value at last tick. The time returned has a resolution * of the kernel tick timer (1/hz [s]). - * FFCLOCK_LERP: Linear interpolation of ffclock time to guarantee + * FFCLOCK_MONO: Linear interpolation of ffclock time to guarantee * monotonic time. * FFCLOCK_LEAPSEC: Include leap seconds. * {FB|FF}CLOCK_UPTIME: Time stamp should be relative to system boot, not epoch. */ #define FFCLOCK_FAST 0x00000001 -#define FFCLOCK_LERP 0x00000002 +#define FFCLOCK_MONO 0x00000002 #define FFCLOCK_LEAPSEC 0x00000004 #define FFCLOCK_UPTIME 0x00000008 #define FFCLOCK_MASK 0x0000ffff @@ -138,10 +138,10 @@ struct ffclock_info { struct bintime error; struct bintime tick_time; - struct bintime tick_time_lerp; + struct bintime tick_time_mono; struct bintime tick_time_diff; uint64_t period; - uint64_t period_lerp; + uint64_t period_mono; int leapsec_adjustment; int status; }; @@ -179,14 +179,14 @@ /* * Retrieve feedforward counter value and time of last kernel tick. This - * accepts the FFCLOCK_LERP flag. + * accepts the FFCLOCK_MONO flag. */ void ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags); /* * Low level routines to convert a counter timestamp into absolute time and a * counter timestamp interval into an interval in seconds. The absolute time - * conversion accepts the FFCLOCK_LERP flag. + * conversion accepts the FFCLOCK_MONO flag. */ void ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags); void ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt); @@ -203,7 +203,7 @@ * timestamp in seconds. The value (in seconds) of the converted timestamp * depends on the flags passed: for a given counter value, different * conversions are possible. Different clock models can be selected by - * combining flags (for example (FFCLOCK_LERP|FFCLOCK_UPTIME) produces + * combining flags (for example (FFCLOCK_MONO|FFCLOCK_UPTIME) produces * linearly interpolated uptime). * ffclock_difftime(): computes a time interval in seconds based on an interval * measured in ffcounter units. This should be the preferred way to measure