diff --git a/sys/kern/kern_ffclock.c b/sys/kern/kern_ffclock.c --- a/sys/kern/kern_ffclock.c +++ b/sys/kern/kern_ffclock.c @@ -231,7 +231,7 @@ sysctl_kern_sysclock_active, "A", "Name of the active system clock which is currently serving time"); -static int sysctl_kern_ffclock_ffcounter_bypass = 0; +int sysctl_kern_ffclock_ffcounter_bypass = 0; SYSCTL_INT(_kern_sysclock_ffclock, OID_AUTO, ffcounter_bypass, CTLFLAG_RW, &sysctl_kern_ffclock_ffcounter_bypass, 0, "Use reliable hardware timecounter as the feedforward counter"); diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -762,38 +762,57 @@ } /* - * Adjust the fftimehands when the timecounter is changed. Stating the obvious, - * the old and new hardware counter cannot be read simultaneously. tc_windup() - * does read the two counters 'back to back', but a few cycles are effectively - * lost, and not accumulated in tick_ffcount. This is a fairly radical - * operation for a feedforward synchronization daemon, and it is its job to not - * pushing irrelevant data to the kernel. Because there is no locking here, - * simply force to ignore pending or next update to give daemon a chance to - * realize the counter has changed. + * Adjust fftimehands when the timecounter is changed. + * This update does not advance the tick itself (hence UTC members remain + * valid), rather values are reinitiated when not relevant for the new counter. + * Because there is no locking here, simply force to ignore pending or next + * update to give daemon a chance to realize the counter has changed. */ static void -ffclock_change_tc(struct timehands *th) +ffclock_change_tc(struct timehands *th, unsigned int ncount) { struct fftimehands *ffth; struct ffclock_estimate *cest; struct timecounter *tc; uint8_t ogen; + ffcounter now; tc = th->th_counter; + + /* Prepare next fftimehand where tick state will be updated. */ ffth = fftimehands->next; ogen = ffth->gen; ffth->gen = 0; - cest = &ffth->cest; + + /* + * Origin setting: reset FFcounter to match start of the current tick. + * If a TSC-derived counter, get correct higher order bits to ensure the + * FFcounter origin matches that of the true counter, rather than the + * time the counter was adopted. If not TSC-derived, the origin will be + * ncount in the past. In all cases, the lower bits of FFcounter and + * th_offset_count will agree. + */ + if ( strcmp(tc->tc_name, "TSC") != 0 ) + ffth->tick_ffcount = (ffcounter)ncount; + else { + now = (ffcounter) rdtsc(); + if (strcmp(tc->tc_name, "TSC-low") == 0) // TSC reads shifted + now >>= (int)(intptr_t)tc->tc_priv; + /* Reconstruct counter value at the time ncount was taken. */ + ffth->tick_ffcount = now - + (ffcounter)((unsigned int)now - ncount); + } + memcpy(cest, &(fftimehands->cest), sizeof(struct ffclock_estimate)); + cest->update_ffcount = ffth->tick_ffcount; cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1; cest->errb_abs = 0; cest->errb_rate = 0; cest->status |= FFCLOCK_STA_UNSYNC; - ffth->tick_ffcount = fftimehands->tick_ffcount; - ffth->tick_time_lerp = fftimehands->tick_time_lerp; ffth->tick_time = fftimehands->tick_time; + ffth->tick_time_lerp = fftimehands->tick_time_lerp; ffth->period_lerp = cest->period; /* Do not lock but ignore next update from synchronization daemon. */ @@ -803,6 +822,12 @@ ogen = 1; ffth->gen = ogen; fftimehands = ffth; + + printf("ffclock_change_tc: new tick_ffcount = %llu = %#llX, with " + "tc %s (%llu Hz)\n", + (unsigned long long)ffth->tick_ffcount, + (unsigned long long)ffth->tick_ffcount, + tc->tc_name, (unsigned long long)tc->tc_frequency); } /* @@ -893,6 +918,7 @@ /* * Access to current ffcounter value. + * If bypass mode on, assume the counter is TSC, and access it directly. */ void ffclock_read_counter(ffcounter *ffcount) @@ -901,6 +927,11 @@ struct fftimehands *ffth; unsigned int gen, delta; + if (sysctl_kern_ffclock_ffcounter_bypass == 1) { + *ffcount = (ffcounter) rdtsc(); + return; + } + /* * ffclock_windup() called from tc_windup(), safe to rely on * th->th_generation only, for correct delta and ffcounter. @@ -1040,9 +1071,13 @@ extern long time_esterror; /* - * Take a raw timestamp (timecounter reading), and then snapshot the sysclock - * data which can be used to compare system clocks and generate timestamps - * of all possible types after the fact. + * Take a raw timestamp (timecounter reading) as soon as possible, then snapshot + * the state of both FB and FF clocks, and the sysclock. The collected state can + * be used to fairly compare alternative system clocks on a precise shared event + * (the raw timestamp) as well as to generate timestamps of all possible types + * at any later time, unaffected by the delay in so doing. If bypass mode is + * on, assume the counter is a TSC variant, and access it directly for reduced + * timestamping latency and overhead. */ void sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) @@ -1053,22 +1088,29 @@ unsigned int delta, gen; #ifdef FFCLOCK ffcounter ffcount; - struct fftimehands *ffth; struct ffclock_info *ffi; + struct fftimehands *ffth; struct ffclock_estimate cest; - - ffi = &clock_snap->ff_info; #endif - fbi = &clock_snap->fb_info; delta = 0; do { th = timehands; gen = atomic_load_acq_int(&th->th_generation); + if (!fast) { +#ifdef FFCLOCK + if (sysctl_kern_ffclock_ffcounter_bypass == 1) + delta = rdtsc32() - th->th_offset_count; + else +#endif + delta = tc_delta(th); + } + fbi = &clock_snap->fb_info; fbi->th_scale = th->th_scale; fbi->tick_time = th->th_offset; #ifdef FFCLOCK + ffi = &clock_snap->ff_info; ffth = fftimehands; ffi->tick_time = ffth->tick_time; ffi->tick_time_lerp = ffth->tick_time_lerp; @@ -1077,8 +1119,6 @@ clock_snap->ffcount = ffth->tick_ffcount; cest = ffth->cest; #endif - if (!fast) - delta = tc_delta(th); atomic_thread_fence_acq(); } while (gen == 0 || gen != th->th_generation); @@ -1461,6 +1501,9 @@ /* Now is a good time to change timecounters. */ if (th->th_counter != tc) { + printf("Changing tc counter : %s = %#X --> %s = %#X \n", + th->th_counter->tc_name, th->th_offset_count, + tc->tc_name, ncount); #ifndef __arm__ if ((tc->tc_flags & TC_FLAGS_C2STOP) != 0) cpu_disable_c2_sleep++; @@ -1473,7 +1516,7 @@ (((uint64_t)tc->tc_counter_mask + 1) / 3)); recalculate_scaling_factor_and_large_delta(th); #ifdef FFCLOCK - ffclock_change_tc(th); + ffclock_change_tc(th, ncount); #endif } diff --git a/sys/sys/timeffc.h b/sys/sys/timeffc.h --- a/sys/sys/timeffc.h +++ b/sys/sys/timeffc.h @@ -68,6 +68,13 @@ /* Declare the kern.sysclock.ffclock sysctl tree. */ SYSCTL_DECL(_kern_sysclock_ffclock); +/* Flag defining if counter bypass mode is desired or not. + * This is only possible if the counter is a TSC with rdtsc defined. + */ +#ifdef FFCLOCK +extern int sysctl_kern_ffclock_ffcounter_bypass; +#endif + /* * Index into the sysclocks array for obtaining the ASCII name of a particular * sysclock.