Index: lib/Makefile =================================================================== --- lib/Makefile +++ lib/Makefile @@ -80,6 +80,7 @@ libsbuf \ libsmb \ libsqlite3 \ + libstats \ libstdbuf \ libstdthreads \ libsysdecode \ Index: lib/libstats/Makefile =================================================================== --- /dev/null +++ lib/libstats/Makefile @@ -0,0 +1,15 @@ +# $FreeBSD$ + +LIB= stats +SHLIBDIR?= /lib +SHLIB_MAJOR= 0 +SRCS= subr_stats.c +WARNS?= 6 + +# To debug, comment WITHOUT_ASSERT_DEBUG= and uncomment CFLAGS:= +WITHOUT_ASSERT_DEBUG= +#CFLAGS:=${CFLAGS:C/-O[0-9]/-O0 -g3/} -DDIAGNOSTIC + +.PATH: ${.CURDIR}/../../sys/kern + +.include Index: share/man/man3/Makefile =================================================================== --- share/man/man3/Makefile +++ share/man/man3/Makefile @@ -18,6 +18,7 @@ queue.3 \ sigevent.3 \ siginfo.3 \ + stats.3 \ stdarg.3 \ sysexits.3 \ tgmath.3 \ @@ -161,6 +162,27 @@ queue.3 TAILQ_PREV.3 \ queue.3 TAILQ_REMOVE.3 \ queue.3 TAILQ_SWAP.3 +MLINKS+= stats.3 stats_tpl_alloc.3 \ + stats.3 stats_tpl_fetch_allocid.3 \ + stats.3 stats_tpl_fetch.3 \ + stats.3 stats_tpl_id2name.3 \ + stats.3 stats_tpl_sample_rates.3 \ + stats.3 stats_tpl_sample_rollthedice.3 \ + stats.3 STATS_VSS_SUM.3 \ + stats.3 STATS_VSS_MAX.3 \ + stats.3 STATS_VSS_MIN.3 \ + stats.3 STATS_VSS_CRHIST32_LIN.3 \ + stats.3 STATS_VSS_CRHIST64_LIN.3 \ + stats.3 stats_tpl_add_voistats.3 \ + stats.3 stats_blob_alloc.3 \ + stats.3 stats_blob_init.3 \ + stats.3 stats_blob_clone.3 \ + stats.3 stats_blob_destroy.3 \ + stats.3 stats_voistat_fetch_dptr.3 \ + stats.3 stats_blob_snapshot.3 \ + stats.3 stats_blob_tostr.3 \ + stats.3 stats_voistatdata_tostr.3 \ + stats.3 stats_blob_visit.3 MLINKS+= stdarg.3 va_arg.3 \ stdarg.3 va_copy.3 \ stdarg.3 va_end.3 \ Index: share/man/man3/stats.3 =================================================================== --- /dev/null +++ share/man/man3/stats.3 @@ -0,0 +1,956 @@ +.\" +.\" Copyright (c) 2016-2018 Netflix, Inc. +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions, and the following disclaimer, +.\" without modification, immediately at the beginning of the file. +.\" 2. The name of the author may not be used to endorse or promote products +.\" derived from this software without specific prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR +.\" ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd July 8, 2018 +.Dt STATS 3 +.Os +.Sh NAME +.Nm stats +.Nd statistics gathering +.Sh LIBRARY +.Lb libstats +.Sh SYNOPSIS +.In sys/stats.h +.Ss Stats Blob Template Management Functions +.Ft int +.Fo stats_tpl_alloc +.Fa "const char *name" +.Fa "uint32_t flags" +.Fc +.Ft int +.Fo stats_tpl_fetch_allocid +.Fa "const char *name" +.Fa "uint32_t hash" +.Fc +.Ft int +.Fo stats_tpl_fetch +.Fa "int tpl_id" +.Fa "struct statsblob_tpl **tpl" +.Fc +.Ft int +.Fo stats_tpl_id2name +.Fa "uint32_t tpl_id" +.Fa "char *buf" +.Fa "size_t len" +.Fc +.Ft int +.Fo stats_tpl_sample_rates +.Fa "SYSCTL_HANDLER_ARGS" +.Fc +.Ft int +.Fo stats_tpl_sample_rollthedice +.Fa "struct stats_tpl_sample_rate *rates" +.Fa "int nrates" +.Fa "void *seed_bytes" +.Fa "size_t seed_len" +.Fc +.Ft struct voistatspec +.Fo STATS_VSS_SUM +.Fc +.Ft struct voistatspec +.Fo STATS_VSS_MAX +.Fc +.Ft struct voistatspec +.Fo STATS_VSS_MIN +.Fc +.Ft struct voistatspec +.Fo STATS_VSS_CRHIST<32|64>_LIN +.Fa "lb" +.Fa "ub" +.Fa "stepinc" +.Fa "vsdflags" +.Fc +.Ft struct voistatspec +.Fo STATS_VSS_CRHIST<32|64>_EXP +.Fa "lb" +.Fa "ub" +.Fa "stepbase" +.Fa "stepexp" +.Fa "vsdflags" +.Fc +.Ft struct voistatspec +.Fo "STATS_VSS_CRHIST<32|64>_LINEXP" +.Fa "lb" +.Fa "ub" +.Fa "nlinsteps" +.Fa "stepbase" +.Fa "vsdflags" +.Fc +.Ft struct voistatspec +.Fo "STATS_VSS_CRHIST<32|64>_USR" +.Fa Sy "HBKTS" Ns Pq Sy "CRBKT" Ns ( Em "lb" ) , "..." Pc , +.Fa "vsdflags" +.Fc +.Ft struct voistatspec +.Fo "STATS_VSS_DRHIST<32|64>_USR" +.Fa Sy "HBKTS" Ns Pq Sy "DRBKT" Ns ( Em "lb" , "ub" ) , "..." Pc , +.Fa "vsdflags" +.Fc +.Ft struct voistatspec +.Fo "STATS_VSS_DVHIST<32|64>_USR" +.Fa Sy "HBKTS" Ns Pq Sy "DVBKT" Ns ( Em "val" ) , "..." Pc , +.Fa "vsdflags" +.Fc +.Ft struct voistatspec +.Fo STATS_VSS_TDGSTCLUST<32|64> +.Fa "nctroids" +.Fa "prec" +.Fc +.Ft int +.Fo stats_tpl_add_voistats +.Fa "uint32_t tpl_id" +.Fa "int32_t voi_id" +.Fa "const char *voi_name" +.Fa "enum vsd_dtype voi_dtype" +.Fa "uint32_t nvss" +.Fa "struct voistatspec *vss" +.Fa "uint32_t flags" +.Fc +.Ss Stats Blob Data Gathering Functions +.Ft int +.Fo stats_voi_update__ +.Fa "struct statsblob *sb" +.Fa "int32_t voi_id" +.Fa " voival" +.Fc +.Ss Stats Blob Utility Functions +.Ft struct statsblob * +.Fo stats_blob_alloc +.Fa "uint32_t tpl_id" +.Fa "uint32_t flags" +.Fc +.Ft int +.Fo stats_blob_init +.Fa "struct statsblob *sb" +.Fa "uint32_t tpl_id" +.Fa "uint32_t flags" +.Fc +.Ft int +.Fo stats_blob_clone +.Fa "struct statsblob **dst" +.Fa "size_t dstmaxsz" +.Fa "struct statsblob *src" +.Fa "uint32_t flags" +.Fc +.Ft void +.Fo stats_blob_destroy +.Fa "struct statsblob *sb" +.Fc +.Ft int +.Fo stats_voistat_fetch_dptr +.Fa "struct statsblob *sb" +.Fa "int32_t voi_id" +.Fa "enum voi_stype stype" +.Fa "enum vsd_dtype *retdtype" +.Fa "struct voistatdata **retvsd" +.Fa "size_t *retvsdsz" +.Fc +.Ft int +.Fo stats_voistat_fetch_ +.Fa "struct statsblob *sb" +.Fa "int32_t voi_id" +.Fa "enum voi_stype stype" +.Fa " *ret" +.Fc +.Ft int +.Fo stats_blob_snapshot +.Fa "struct statsblob **dst" +.Fa "size_t dstmaxsz" +.Fa "struct statsblob *src" +.Fa "uint32_t flags" +.Fc +.Ft int +.Fo stats_blob_tostr +.Fa "struct statsblob *sb" +.Fa "struct sbuf *buf" +.Fa "enum sb_str_fmt fmt" +.Fa "uint32_t flags" +.Fc +.Ft int +.Fo stats_voistatdata_tostr +.Fa "const struct voistatdata *vsd" +.Fa "enum vsd_dtype dtype" +.Fa "enum sb_str_fmt fmt" +.Fa "struct sbuf *buf" +.Fa "int objdump" +.Fc +.Ft typedef int +.Fn "\*(lp*stats_blob_visitcb_t\*(rp" "struct sb_visit *sbv" "void *usrctx" +.Ft int +.Fo stats_blob_visit +.Fa "struct statsblob *sb" +.Fa "stats_blob_visitcb_t func" +.Fa "void *usrctx" +.Fc +.Sh DESCRIPTION +The +.Nm +framework facilitates real-time kernel and user space statistics gathering. +The framework is built around the +.Dq statsblob , +an object embedded within a contiguous memory allocation that is mostly opaque +to consumers and stores all required state. +A +.Dq statsblob +object can itself be embedded within other objects either directly or indirectly +using a pointer. +.Pp +Objects or subsystems for which statistics are to be gathered are initialised +from a template +.Dq statsblob , +which acts as the blueprint for an arbitrary set of +variables of interest (VOIs) and their associated statistics. +Each template defines a schema plus associated metadata, which are kept separate +to minimise the memory footprint of blobs. +.Pp +Data gathering hook functions added at appropriate locations within the code +base of interest feed VOI data into the framework for processing. +.Pp +Each +.Dq statsblob , +consists of a +.Vt struct statsblob +header and opaque internal blob structure per the following diagram: +.Bd -literal -offset indent +--------------------------------------------------------- +| struct | uint8_t | +| statsblob | opaque[] | +--------------------------------------------------------- +.Ed +.Pp +The publicly visible 8-byte header is defined as: +.Bd -literal -offset indent +struct statsblob { + uint8_t abi; + uint8_t endian; + uint16_t flags; + uint16_t maxsz; + uint16_t cursz; + uint8_t opaque[]; +}; +.Ed +.Pp +.Va abi +specifies which API version the blob's opaque internals conform to +.Pq Dv STATS_ABI_V1 is the only version currently defined . +.Va endian +specifies the endianness of the blob's fields +.Po +.Dv SB_LE +for little endian, +.Dv SB_BE +for big endian, or +.Dv SB_UE +for unknown endianness +.Pc . +.Va cursz +specifies the size of the blob, while +.Va maxsz +specifies the size of the underlying memory allocation in which the +blob is embedded. +Both +.Va cursz +and +.Va maxsz +default to units of bytes, unless a flag is set in +.Va flags +that dictates otherwise. +.Pp +Templates are constructed by associating arbitrary VOI IDs with a set of +statistics, where each statistic is specified using a +.Vt struct voistatspec +per the definition below: +.Bd -literal -offset indent +struct voistatspec { + vss_hlpr_fn hlpr; + struct vss_hlpr_info *hlprinfo; + struct voistatdata *iv; + size_t vsdsz; + uint32_t flags; + enum vsd_dtype vs_dtype : 8; + enum voi_stype stype : 8; +}; +.Ed +.Pp +It is generally expected that consumers will not work with +.Vt struct voistatspec +directly, and instead use the +.Fn STATS_VSS_* +helper macros. +.Pp +The +.Nm +framework offers the following statistics for association with VOIs: +.Bl -tag -width ".Dv VS_STYPE_TDGST" +.It Dv VS_STYPE_SUM +The sum of VOI values. +.It Dv VS_STYPE_MAX +The maximum VOI value. +.It Dv VS_STYPE_MIN +The minimum VOI value. +.It Dv VS_STYPE_HIST +A static bucket histogram of VOI values, including a count of +.Dq out-of-band/bucket Dc +values which did not match any bucket. +Histograms can be specified as +.Dq Em C Ns ontinuous Em R Ns ange Dc +.Pq CRHIST Pc , +.Dq Em D Ns iscrete Em R Ns ange Dc +.Pq DRHIST Pc +or +.Dq Em D Ns iscrete Em V Ns alue Dc +.Pq DVHIST Pc , +with 32 or 64 bit bucket counters, depending on the VOI semantics. +.It Dv VS_STYPE_TDGST +A dynamic bucket histogram of VOI values based on the t-digest method +.Po refer to the t-digest paper in the +.Sx SEE ALSO +section below +.Pc . +.El +.Pp +A +.Dq visitor software design pattern Ns +-like scheme is employed to facilitate iterating over a blob's data without +concern for the blob's structure. +The data provided to visitor callback functions is encapsulated in +.Vt struct sb_visit +per the definition below: +.Bd -literal -offset indent +struct sb_visit { + struct voistatdata *vs_data; + uint32_t tplhash; + uint32_t flags; + int16_t voi_id; + int16_t vs_dsz; + enum vsd_dtype voi_dtype : 8; + enum vsd_dtype vs_dtype : 8; + int8_t vs_stype; + uint16_t vs_errs; +}; +.Ed +.Pp +The +.Fn stats_tpl_sample_rates +and +.Fn stats_tpl_sample_rollthedice +functions utilise +.Vt struct stats_tpl_sample_rate +to encapsulate per-template sample rate information per the definition below: +.Bd -literal -offset indent +struct stats_tpl_sample_rate { + int32_t tpl_slot_id; + uint32_t tpl_sample_pct; +}; +.Ed +.Pp +The +.Va tpl_slot_id +member holds the template's slot ID obtained from +.Fn stats_tpl_alloc +or +.Fn stats_tpl_fetch_allocid . +The +.Va tpl_sample_pct +member holds the template's sample rate as an integer percentage in the range +[0,100]. +.Pp +The +.Vt stats_tpl_sr_cb_t +conformant function pointer that is required as the +.Fa arg1 +of +.Fn stats_tpl_sample_rates +is defined as: +.Bd -literal -offset indent +enum stats_tpl_sr_cb_action { + TPL_SR_UNLOCKED_GET, + TPL_SR_RLOCKED_GET, + TPL_SR_RUNLOCK, + TPL_SR_PUT +}; +typedef int (*stats_tpl_sr_cb_t)(enum stats_tpl_sr_cb_action action, + struct stats_tpl_sample_rate **rates, int *nrates, void *ctx); +.Ed +.Pp +It is required that a conformant function: +.Bl -dash +.It +Return an appropriate +.Xr errno 2 +on error, otherwise 0. +.It +When called with +.Qq action == TPL_SR_*_GET , +return the subsystem's rates list ptr and count, locked or unlocked as +requested. +.It +When called with +.Qq action == TPL_SR_RUNLOCK , +unlock the subsystem's rates list ptr and count. +Pair with a prior +.Qq action == TPL_SR_RLOCKED_GET +call. +.It +When called with +.Qq action == TPL_SR_PUT , +update the subsystem's rates list ptr and count to the sysctl processed values +and return the inactive list details in +.Fa rates +and +.Fa nrates +for garbage collection by +.Fn stats_tpl_sample_rates . +.El +.Pp +Where templates need to be referenced via textual means, for example via a MIB +variable, the following string based template spec formats can be used: +.Bl -enum +.It +.Qq Qc Ns +: +.Ns , for example +.Qq TCP_DEFAULT Qc Ns +:1731235399 +.It +.Qq Qc +.Ns , for example +.Qq TCP_DEFAULT Qc +.It +: +.Ns , for example +:1731235399 +.El +.Pp +The first form is the normative spec format generated by the framework, while +the second and third forms are convenience formats primarily for user input. +The use of inverted commas around the template name is optional. +.Ss MIB Variables +The in-kernel +.Nm +framework exposes the following framework-specific variables in the +.Va kern.stats +branch of the +.Xr sysctl 3 +MIB. +.Bl -tag -width "templates" +.It templates +Read-only CSV list of registered templates in normative template spec form. +.El +.Ss Template Management Functions +The +.Fn stats_tpl_alloc +function allocates a new template with the specified unique name and returns its +runtime-stable template slot ID for use with other API functions. +The +.Fa flags +argument is currently unused. +.Pp +The +.Fn stats_tpl_fetch_allocid +function returns the runtime-stable template slot ID of any registered template +matching the specified name and hash. +.Pp +The +.Fn stats_tpl_fetch +function returns the pointer to the registered template object at the specified +template slot ID. +.Pp +The +.Fn stats_tpl_id2name +function returns the name of the registered template object at the specified +template slot ID. +.Pp +The +.Fn stats_tpl_sample_rates +function provides a generic handler for template sample rates management and +reporting via +.Xr sysctl 3 +MIB variables. +Subsystems can use this function to create a subsystem-specific +.Xr SYSCTL_PROC 9 +MIB variable that manages and reports subsystem-specific template sampling +rates. +Subsystems must supply a +.Vt stats_tpl_sr_cb_t +conformant function pointer as the sysctl's +.Fa arg1 , +which is a callback used to interact with the subsystem's stats template sample +rates list. +Subsystems can optionally specify the sysctl's +.Fa arg2 +as non-zero, which causes a zero-initialised allocation of arg2-sized contextual +memory to be heap-allocated and passed in to all subsystem callbacks made during +the operation of +.Fn stats_tpl_sample_rates . +.Pp +The +.Fn stats_tpl_sample_rollthedice +function makes a weighted random template selection from the supplied array of +template sampling rates. +The cumulative percentage of all sampling rates should not exceed 100. +If no seed is supplied, a PRNG is used to generate a true random number so that +every selection is independent. +If a seed is supplied, selection will be made randomly across different seeds, but +deterministically given the same seed. +.Pp +The +.Fn stats_tpl_add_voistats +function is used to add a VOI and associated set of statistics to the registered +template object at the specified template slot ID. +The set of statistics is passed as an array of +.Vt struct voistatspec +which can be initialised using the +.Fn STATS_VSS_* +helper macros or manually for non-standard use cases. +For static +.Fa vss +arrays, the +.Fa nvss +count of array elements can be determined by passing +.Fa vss +to the +.Fn NVSS +macro. +The +.Dv SB_VOI_RELUPDATE +flag can be passed to configure the VOI for use with +.Fn stats_voi_update_rel_ , +which entails maintaining an extra 8b of state in the blob at each update. +.Ss Data Gathering Functions +The +.Fn stats_voi_update_abs_ +and +.Fn stats_voi_update_rel_ +functions both update all the statistics associated with the VOI identified by +.Fa voi_id . +The +.Dq abs +call uses +.Fa voival +as an absolute value, whereas the +.Dq rel +call uses +.Fa voival +as a value relative to that of the previous update function call, by adding it +to the previous value and using the result for the update. +Relative updates are only possible for VOIs that were added to the template with +the +.Dv SB_VOI_RELUPDATE +flag specified to +.Fn stats_tpl_add_voistats . +.Ss Utility Functions +The +.Fn stats_blob_alloc +function allocates and initialises a new blob based on the registered template +object at the specified template slot ID. +.Pp +The +.Fn stats_blob_init +function initialises a new blob in an existing memory allocation based on the +registered template object at the specified template slot ID. +.Pp +The +.Fn stats_blob_clone +function duplicates the +.Fa src +blob into +.Fa dst , +leaving only the +.Va maxsz +field of +.Fa dst +untouched. +The +.Dv SB_CLONE_ALLOCDST +flag can be passed to instruct the function to allocate a new blob of +appropriate size into which to clone +.Fa src , +storing the new pointer in +.Fa *dst . +The +.Dv SB_CLONE_USRDSTNOFAULT +or +.Dv SB_CLONE_USRDST +flags can be set to respectively signal that +.Xr copyout_nofault 9 +or +.Xr copyout 9 +should be used because +.Fa *dst +is a user space address. +.Pp +The +.Fn stats_blob_snapshot +function calls +.Fn stats_blob_clone +to obtain a copy of +.Fa src +and then performs any additional functions required to produce a coherent +blob snapshot. +The flags interpreted by +.Fn stats_blob_clone +also apply to +.Fn stats_blob_snapshot . +Additionally, the +.Dv SB_CLONE_RSTSRC +flag can be used to effect a reset of the +.Fa src +blob's statistics after a snapshot is successfully taken. +.Pp +The +.Fn stats_blob_destroy +function destroys a blob previously created with +.Fn stats_blob_alloc , +.Fn stats_blob_clone +or +.Fn stats_blob_snapshot . +.Pp +The +.Fn stats_blob_visit +function allows the caller to iterate over the contents of a blob. +The callback function +.Fa func +is called for every VOI and statistic in the blob, passing a +.Vt struct sb_visit +and the user context argument +.Fa usrctx +to the callback function. +The +.Fa sbv +passed to the callback function may have one or more of the following flags set +in the +.Va flags +struct member to provide useful metadata about the iteration: +.Dv SB_IT_FIRST_CB , +.Dv SB_IT_LAST_CB , +.Dv SB_IT_FIRST_VOI , +.Dv SB_IT_LAST_VOI , +.Dv SB_IT_FIRST_VOISTAT , +.Dv SB_IT_LAST_VOISTAT , +.Dv SB_IT_NULLVOI +and +.Dv SB_IT_NULLVOISTAT . +Returning a non-zero value from the callback function terminates the iteration. +.Pp +The +.Fn stats_blob_tostr +renders a string representation of a blob into the +.Xr sbuf 9 +.Fa buf . +Currently supported render formats are +.Dv SB_STRFMT_FREEFORM +and +.Dv SB_STRFMT_JSON . +The +.Dv SB_TOSTR_OBJDUMP +flag can be passed to render version specific opaque implementation detail for +debugging or string-to-binary blob reconstruction purposes. +The +.Dv SB_TOSTR_META +flag can be passed to render template metadata into the string representation, +using the blob's template hash to lookup the corresponding template. +.Pp +The +.Fn stats_voistatdata_tostr +renders a string representation of an individual statistic's data into the +.Xr sbuf 9 +.Fa buf . +The same render formats supported by the +.Fn stats_blob_tostr +function can be specified, and the +.Fa objdump +boolean has the same meaning as the +.Dv SB_TOSTR_OBJDUMP +flag. +.Pp +The +.Fn stats_voistat_fetch_dptr +function returns an internal blob pointer to the specified +.Fa stype +statistic data for the VOI +.Fa voi_id . +The +.Fn stats_voistat_fetch_ +functions are convenience wrappers around +.Fn stats_voistat_fetch_dptr +to perform the extraction for simple data types. +.Sh IMPLEMENTATION NOTES +The following notes apply to STATS_ABI_V1 format statsblobs. +.Ss Space-Time Complexity +Blobs are laid out as three distinct memory regions following the header: +.Bd -literal -offset indent +------------------------------------------------------ +| struct | struct | struct | struct | +| statsblobv1 | voi [] | voistat [] | voistatdata [] | +------------------------------------------------------ +.Ed +.Pp +Blobs store VOI and statistic blob state +.Po +8b for +.Vt struct voi +and 8b for +.Vt struct voistat +respectively +.Pc +in sparse arrays, using the +.Fa voi_id +and +.Vt enum voi_stype +as array indices. +This allows O(1) access to any voi/voistat pair in the blob, at the expense of +8b of wasted memory per vacant slot for templates which do not specify +contiguously numbered VOIs and/or statistic types. +Data storage for statistics is only allocated for non-vacant slot pairs. +.Pp +To provide a concrete example, a blob with the following specification: +.Bl -dash +.It +Two VOIs; ID 0 and 2; added to the template in that order +.It +VOI 0 is of data type +.Vt int64_t , +is configured with +.Dv SB_VOI_RELUPDATE +to enable support for relative updates using +.Fn stats_voi_update_rel_ , +and has a +.Dv VS_STYPE_MIN +statistic associated with it. +.It +VOI 2 is of data type +.Vt uint32_t +with +.Dv VS_STYPE_SUM +and +.Dv VS_STYPE_MAX +statistics associated with it. +.El +.Pp +would have the following memory layout: +.Bd -literal +-------------------------------------- +| header | struct statsblobv1 @ 32b +|------------------------------------| +| voi[0] | struct voi @ 8b +| voi[1] (vacant) | struct voi @ 8b +| voi[2] | struct voi @ 8b +|------------------------------------| +| voi[2]voistat[VOISTATE] (vacant) | struct voistat @ 8b +| voi[2]voistat[SUM] | struct voistat @ 8b +| voi[2]voistat[MAX] | struct voistat @ 8b +| voi[0]voistat[VOISTATE] | struct voistat @ 8b +| voi[0]voistat[SUM] (vacant) | struct voistat @ 8b +| voi[0]voistat[MAX] (vacant) | struct voistat @ 8b +| voi[0]voistat[MIN] | struct voistat @ 8b +|------------------------------------| +| voi[2]voistat[SUM]voistatdata | struct voistatdata_int32 @ 4b +| voi[2]voistat[MAX]voistatdata | struct voistatdata_int32 @ 4b +| voi[0]voistat[VOISTATE]voistatdata | struct voistatdata_numeric @ 8b +| voi[0]voistat[MIN]voistatdata | struct voistatdata_int64 @ 8b +-------------------------------------- + TOTAL @ 136b +.Ed +.Pp +When rendered to string format using +.Fn stats_blob_tostr , +the +.Dv SB_STRFMT_FREEFORM +.Fa fmt +and the +.Dv SB_TOSTR_OBJDUMP +flag, the rendered output is: +.Bd -literal +struct statsblobv1@0x8016250a0, abi=1, endian=1, maxsz=136, cursz=136, \\ + created=6294158585626144, lastrst=6294158585626144, flags=0x0000, \\ + stats_off=56, statsdata_off=112, tplhash=2994056564 + vois[0]: id=0, name="", flags=0x0001, dtype=INT_S64, voistatmaxid=3, \\ + stats_off=80 + vois[0]stat[0]: stype=VOISTATE, flags=0x0000, dtype=VOISTATE, \\ + dsz=8, data_off=120 + voistatdata: prev=0 + vois[0]stat[1]: stype=-1 + vois[0]stat[2]: stype=-1 + vois[0]stat[3]: stype=MIN, flags=0x0000, dtype=INT_S64, \\ + dsz=8, data_off=128 + voistatdata: 9223372036854775807 + vois[1]: id=-1 + vois[2]: id=2, name="", flags=0x0000, dtype=INT_U32, voistatmaxid=2, \\ + stats_off=56 + vois[2]stat[0]: stype=-1 + vois[2]stat[1]: stype=SUM, flags=0x0000, dtype=INT_U32, dsz=4, \\ + data_off=112 + voistatdata: 0 + vois[2]stat[2]: stype=MAX, flags=0x0000, dtype=INT_U32, dsz=4, \\ + data_off=116 + voistatdata: 0 +.Ed +.Pp +Note: The +.Qq \e +present in the rendered output above indicates a manual line break inserted to +keep the man page within 80 columns and is not part of the actual output. +.Ss Locking +The +.Nm +framework does not provide any concurrency protection at the individual blob +level, instead requiring that consumers guarantee mutual exclusion when calling +API functions that reference a non-template blob. +.Pp +The list of templates is protected with a +.Xr rwlock 9 +in-kernel, and +.Xr pthread 3 +rw lock in user space to support concurrency between template management and +blob initialisation operations. +.Sh RETURN VALUES +.Fn stats_tpl_alloc +returns a runtime-stable template slot ID on success, or a negative errno on +failure. +-EINVAL is returned if any problems are detected with the arguments. +-EEXIST is returned if an existing template is registered with the same name. +-ENOMEM is returned if a required memory allocation fails. +.Pp +.Fn stats_tpl_fetch_allocid +returns a runtime-stable template slot ID, or negative errno on failure. +-ESRCH is returned if no registered template matches the specified name and/or +hash. +.Pp +.Fn stats_tpl_fetch +returns 0 on success, or ENOENT if an invalid +.Fa tpl_id +is specified. +.Pp +.Fn stats_tpl_id2name +returns 0 on success, or an errno on failure. +EOVERFLOW is returned if the length of +.Fa buf +specified by +.Fa len +is too short to hold the template's name. +ENOENT is returned if an invalid +.Fa tpl_id +is specified. +.Pp +.Fn stats_tpl_sample_rollthedice +returns a valid template slot id selected from +.Fa rates +or -1 if a NULL selection was made, that is no stats collection this roll. +.Pp +.Fn stats_tpl_add_voistats +return 0 on success, or an errno on failure. +EINVAL is returned if any problems are detected with the arguments. +EFBIG is returned if the resulting blob would have exceeded the maximum size. +EOPNOTSUPP is returned if an attempt is made to add more VOI stats to a +previously configured VOI. +ENOMEM is returned if a required memory allocation fails. +.Pp +.Fn stats_voi_update_abs_ +and +.Fn stats_voi_update_rel_ +return 0 on success, or EINVAL if any problems are detected with the arguments. +.Pp +.Fn stats_blob_init +returns 0 on success, or an errno on failure. +EINVAL is returned if any problems are detected with the arguments. +EOVERFLOW is returned if the template blob's +.Fa cursz +is larger than the +.Fa maxsz +of the blob being intialised. +.Pp +.Fn stats_blob_alloc +returns a pointer to a newly allocated and initialised blob based on the +specified template with slot ID +.Fa tpl_id , +or NULL if the memory allocation failed. +.Pp +.Fn stats_blob_clone +and +.Fn stats_blob_snapshot +return 0 on success, or an errno on failure. +EINVAL is returned if any problems are detected with the arguments. +ENOMEM is returned if the SB_CLONE_ALLOCDST flag was specified and the memory +allocation for +.Fa dst +fails. +EOVERFLOW is returned if the src blob's +.Fa cursz +is larger than the +.Fa maxsz +of the +.Fa dst +blob. +.Pp +.Fn stats_blob_visit +returns 0 on success, or EINVAL if any problems are detected with the arguments. +.Pp +.Fn stats_blob_tostr +and +.Fn stats_voistatdata_tostr +return 0 on success, or an errno on failure. +EINVAL is returned if any problems are detected with the arguments, otherwise +any error returned by +.Fn sbuf_error +for +.Fa buf +is returned. +.Pp +.Fn stats_voistat_fetch_dptr +returns 0 on success, or EINVAL if any problems are detected with the arguments. +.Pp +.Fn stats_voistat_fetch_ +returns 0 on success, or an errno on failure. +EINVAL is returned if any problems are detected with the arguments. +EFTYPE is returned if the requested data type does not match the blob's data +type for the specified +.Fa voi_id +and +.Fa stype . +.Sh SEE ALSO +.Xr errno 2 , +.Xr tcp 4 , +.Xr sbuf 9 +.Rs +.%A "Ted Dunning" +.%A "Otmar Ertl" +.%T "Computing Extremely Accurate Quantiles Using t-digests" +.%U "https://github.com/tdunning/t-digest/raw/master/docs/t-digest-paper/histo.pdf" +.Re +.Sh HISTORY +The +.Nm +framework first appeared in +.Fx 13.0 . +.Sh AUTHORS +.An -nosplit +The +.Nm +framework and this manual page were written by +.An Lawrence Stewart Aq lstewart@FreeBSD.org +and sponsored by Netflix, Inc. Index: share/mk/bsd.libnames.mk =================================================================== --- share/mk/bsd.libnames.mk +++ share/mk/bsd.libnames.mk @@ -138,6 +138,7 @@ LIBSMB?= ${LIBDESTDIR}${LIBDIR_BASE}/libsmb.a LIBSSL?= ${LIBDESTDIR}${LIBDIR_BASE}/libssl.a LIBSSP_NONSHARED?= ${LIBDESTDIR}${LIBDIR_BASE}/libssp_nonshared.a +LIBSTATS?= ${LIBDESTDIR}${LIBDIR_BASE}/libstats.a LIBSTDCPLUSPLUS?= ${LIBDESTDIR}${LIBDIR_BASE}/libstdc++.a LIBSTDTHREADS?= ${LIBDESTDIR}${LIBDIR_BASE}/libstdthreads.a LIBSYSDECODE?= ${LIBDESTDIR}${LIBDIR_BASE}/libsysdecode.a Index: share/mk/src.libnames.mk =================================================================== --- share/mk/src.libnames.mk +++ share/mk/src.libnames.mk @@ -167,6 +167,7 @@ smb \ ssl \ ssp_nonshared \ + stats \ stdthreads \ supcplusplus \ sysdecode \ @@ -349,6 +350,7 @@ .if ${MK_SSP} != "no" _DP_c+= ssp_nonshared .endif +_DP_stats= sbuf pthread _DP_stdthreads= pthread _DP_tacplus= md _DP_panel= ncurses Index: sys/amd64/conf/NOTES =================================================================== --- sys/amd64/conf/NOTES +++ sys/amd64/conf/NOTES @@ -666,3 +666,5 @@ options LINDEBUGFS options GCOV + +options STATS Index: sys/conf/NOTES =================================================================== --- sys/conf/NOTES +++ sys/conf/NOTES @@ -2964,3 +2964,5 @@ options SPIGEN_LEGACY_CDEVNAME # legacy device names for spigen device xz # xz_embedded LZMA de-compression library + +options STATS Index: sys/conf/files =================================================================== --- sys/conf/files +++ sys/conf/files @@ -3850,6 +3850,7 @@ kern/subr_sleepqueue.c standard kern/subr_smp.c standard kern/subr_stack.c optional ddb | stack | ktr +kern/subr_stats.c optional stats kern/subr_taskqueue.c standard kern/subr_terminal.c optional vt kern/subr_trap.c standard Index: sys/conf/options =================================================================== --- sys/conf/options +++ sys/conf/options @@ -424,6 +424,7 @@ RATELIMIT_DEBUG opt_ratelimit.h INET opt_inet.h INET6 opt_inet6.h +STATS opt_global.h IPDIVERT IPFILTER opt_ipfilter.h IPFILTER_DEFAULT_BLOCK opt_ipfilter.h Index: sys/kern/subr_stats.c =================================================================== --- /dev/null +++ sys/kern/subr_stats.c @@ -0,0 +1,3916 @@ +/*- + * Copyright (c) 2014-2018 Netflix, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* + * Author: Lawrence Stewart + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* Must come after qmath.h and tree.h */ + +#ifdef _KERNEL +#include +#include +#include +#include +#include +#else /* ! _KERNEL */ +#include +#include +#include +#include +#include +#include +#endif /* _KERNEL */ + +struct voistatdata_voistate { + /* Previous VOI value for diff calculation. */ + struct voistatdata_numeric prev; +}; + +#define VS_VSDVALID 0x0001 /* Stat's voistatdata updated at least once. */ +struct voistat { + int8_t stype; /* Type of stat e.g. VS_STYPE_SUM. */ + enum vsd_dtype dtype : 8; /* Data type of this stat's data. */ + uint16_t data_off; /* Blob offset for this stat's data. */ + uint16_t dsz; /* Size of stat's data. */ +#define VS_EBITS 8 + uint16_t errs : VS_EBITS;/* Non-wrapping error count. */ + uint16_t flags : 16 - VS_EBITS; +}; +/* The voistat error count is capped to avoid wrapping. */ +#define VS_INCERRS(vs) do { \ + if ((vs)->errs < (1U << VS_EBITS) - 1) \ + (vs)->errs++; \ +} while (0) + +/* + * Ideas for flags: + * - Global or entity specific (global would imply use of counter(9)?) + * - Whether to reset stats on read or not + * - Signal an overflow? + * - Compressed voistat array + */ +#define VOI_REQSTATE 0x0001 /* VOI requires VS_STYPE_VOISTATE. */ +struct voi { + int16_t id; /* VOI id. */ + enum vsd_dtype dtype : 8; /* Data type of the VOI itself. */ + int8_t voistatmaxid; /* Largest allocated voistat index. */ + uint16_t stats_off; /* Blob offset for this VOIs stats. */ + uint16_t flags; +}; + +/* + * Memory for the entire blob is allocated as a slab and then offsets are + * maintained to carve up the slab into sections holding different data types. + * + * Ideas for flags: + * - Compressed voi array (trade off memory usage vs search time) + * - Units of offsets (default bytes, flag for e.g. vm_page/KiB/Mib) + */ +struct statsblobv1 { + uint8_t abi; + uint8_t endian; + uint16_t flags; + uint16_t maxsz; + uint16_t cursz; + /* Fields from here down are opaque to consumers. */ + uint32_t tplhash; /* Base template hash ID. */ + uint16_t stats_off; /* voistat array blob offset. */ + uint16_t statsdata_off; /* voistatdata array blob offset. */ + sbintime_t created; /* Blob creation time. */ + sbintime_t lastrst; /* Time of last reset. */ + struct voi vois[]; /* Array indexed by [voi_id]. */ +} __aligned(sizeof(void *)); +_Static_assert(offsetof(struct statsblobv1, cursz) + + SIZEOF_MEMBER(struct statsblobv1, cursz) == sizeof(struct statsblob), + "statsblobv1 ABI mismatch"); + +struct statsblobv1_tpl { + struct metablob *mb; + struct statsblobv1 *sb; +}; + +/* Context passed to iterator callbacks. */ +struct sb_iter_ctx { + void *usrctx; /* Caller supplied context. */ + uint32_t flags; /* Flags for current iteration. */ + int16_t vslot; /* struct voi slot index. */ + int8_t vsslot; /* struct voistat slot index. */ +}; + +struct sb_tostrcb_ctx { + struct sbuf *buf; + struct statsblob_tpl *tpl; + enum sb_str_fmt fmt; + uint32_t flags; +}; + +struct sb_visitcb_ctx { + stats_blob_visitcb_t cb; + void *usrctx; +}; + +/* Stats blob iterator callback. */ +typedef int (*stats_v1_blob_itercb_t)(struct statsblobv1 *sb, struct voi *v, + struct voistat *vs, struct sb_iter_ctx *ctx); + +#ifdef _KERNEL +static struct rwlock tpllistlock; +RW_SYSINIT(stats_tpl_list, &tpllistlock, "Stat template list lock"); +#define TPL_LIST_RLOCK() rw_rlock(&tpllistlock) +#define TPL_LIST_RUNLOCK() rw_runlock(&tpllistlock) +#define TPL_LIST_WLOCK() rw_wlock(&tpllistlock) +#define TPL_LIST_WUNLOCK() rw_wunlock(&tpllistlock) +#define TPL_LIST_LOCK_ASSERT() rw_assert(&tpllistlock, RA_LOCKED) +#define TPL_LIST_RLOCK_ASSERT() rw_assert(&tpllistlock, RA_RLOCKED) +#define TPL_LIST_WLOCK_ASSERT() rw_assert(&tpllistlock, RA_WLOCKED) +MALLOC_DEFINE(M_STATS, "stats(9) related memory", "stats(9) related memory"); +#define stats_free(ptr) free((ptr), M_STATS) +#else /* ! _KERNEL */ +static void stats_constructor(void); +static void stats_destructor(void); +static pthread_rwlock_t tpllistlock; +#define TPL_LIST_UNLOCK() pthread_rwlock_unlock(&tpllistlock) +#define TPL_LIST_RLOCK() pthread_rwlock_rdlock(&tpllistlock) +#define TPL_LIST_RUNLOCK() TPL_LIST_UNLOCK() +#define TPL_LIST_WLOCK() pthread_rwlock_wrlock(&tpllistlock) +#define TPL_LIST_WUNLOCK() TPL_LIST_UNLOCK() +#define TPL_LIST_LOCK_ASSERT() do { } while (0) +#define TPL_LIST_RLOCK_ASSERT() do { } while (0) +#define TPL_LIST_WLOCK_ASSERT() do { } while (0) +#ifdef NDEBUG +#define KASSERT(cond, msg) do {} while (0) +#define stats_abort() do {} while (0) +#else /* ! NDEBUG */ +#define KASSERT(cond, msg) do { \ + if (!(cond)) { \ + panic msg; \ + } \ +} while (0) +#define stats_abort() abort() +#endif /* NDEBUG */ +#define stats_free(ptr) free(ptr) +#define panic(fmt, ...) do { \ + fprintf(stderr, (fmt), ##__VA_ARGS__); \ + stats_abort(); \ +} while (0) +#endif /* _KERNEL */ + +#define SB_V1_MAXSZ 65535 + +/* Obtain a blob offset pointer. */ +#define BLOB_OFFSET(sb, off) ((void *)(((uint8_t *)(sb)) + (off))) + +/* + * Number of VOIs in the blob's vois[] array. By virtue of struct voi being a + * power of 2 size, we can shift instead of divide. The shift amount must be + * updated if sizeof(struct voi) ever changes, which the assert should catch. + */ +#define NVOIS(sb) ((int32_t)((((struct statsblobv1 *)(sb))->stats_off - \ + sizeof(struct statsblobv1)) >> 3)) +_Static_assert(sizeof(struct voi) == 8, "statsblobv1 voi ABI mismatch"); + +/* Try restrict names to alphanumeric and underscore to simplify JSON compat. */ +const char *vs_stype2name[VS_NUM_STYPES] = { + "VOISTATE", /* VS_STYPE_VOISTATE */ + "SUM", /* VS_STYPE_SUM */ + "MAX", /* VS_STYPE_MAX */ + "MIN", /* VS_STYPE_MIN */ + "HIST", /* VS_STYPE_HIST */ + "TDGST", /* VS_STYPE_TDGST */ +}; + +const char *vs_stype2desc[VS_NUM_STYPES] = { + "VOI related state data (not a real stat)", /* VS_STYPE_VOISTATE */ + "Simple arithmetic accumulator", /* VS_STYPE_SUM */ + "Maximum observed VOI value", /* VS_STYPE_MAX */ + "Minimum observed VOI value", /* VS_STYPE_MIN */ + "Histogram of observed VOI values", /* VS_STYPE_HIST */ + "t-digest of observed VOI values", /* VS_STYPE_TDGST */ +}; + +const char *vsd_dtype2name[VSD_NUM_DTYPES] = { + "VOISTATE", /* VSD_DTYPE_VOISTATE */ + "INT_S32", /* VSD_DTYPE_INT_S32 */ + "INT_U32", /* VSD_DTYPE_INT_U32 */ + "INT_S64", /* VSD_DTYPE_INT_S64 */ + "INT_U64", /* VSD_DTYPE_INT_U64 */ + "INT_SLONG", /* VSD_DTYPE_INT_SLONG */ + "INT_ULONG", /* VSD_DTYPE_INT_ULONG */ + "Q_S32", /* VSD_DTYPE_Q_S32 */ + "Q_U32", /* VSD_DTYPE_Q_U32 */ + "Q_S64", /* VSD_DTYPE_Q_S64 */ + "Q_U64", /* VSD_DTYPE_Q_U64 */ + "CRHIST32", /* VSD_DTYPE_CRHIST32 */ + "DRHIST32", /* VSD_DTYPE_DRHIST32 */ + "DVHIST32", /* VSD_DTYPE_DVHIST32 */ + "CRHIST64", /* VSD_DTYPE_CRHIST64 */ + "DRHIST64", /* VSD_DTYPE_DRHIST64 */ + "DVHIST64", /* VSD_DTYPE_DVHIST64 */ + "TDGSTCLUST32", /* VSD_DTYPE_TDGSTCLUST32 */ + "TDGSTCLUST64", /* VSD_DTYPE_TDGSTCLUST64 */ +}; + +const size_t vsd_dtype2size[VSD_NUM_DTYPES] = { + sizeof(struct voistatdata_voistate), /* VSD_DTYPE_VOISTATE */ + sizeof(struct voistatdata_int32), /* VSD_DTYPE_INT_S32 */ + sizeof(struct voistatdata_int32), /* VSD_DTYPE_INT_U32 */ + sizeof(struct voistatdata_int64), /* VSD_DTYPE_INT_S64 */ + sizeof(struct voistatdata_int64), /* VSD_DTYPE_INT_U64 */ + sizeof(struct voistatdata_intlong), /* VSD_DTYPE_INT_SLONG */ + sizeof(struct voistatdata_intlong), /* VSD_DTYPE_INT_ULONG */ + sizeof(struct voistatdata_q32), /* VSD_DTYPE_Q_S32 */ + sizeof(struct voistatdata_q32), /* VSD_DTYPE_Q_U32 */ + sizeof(struct voistatdata_q64), /* VSD_DTYPE_Q_S64 */ + sizeof(struct voistatdata_q64), /* VSD_DTYPE_Q_U64 */ + sizeof(struct voistatdata_crhist32), /* VSD_DTYPE_CRHIST32 */ + sizeof(struct voistatdata_drhist32), /* VSD_DTYPE_DRHIST32 */ + sizeof(struct voistatdata_dvhist32), /* VSD_DTYPE_DVHIST32 */ + sizeof(struct voistatdata_crhist64), /* VSD_DTYPE_CRHIST64 */ + sizeof(struct voistatdata_drhist64), /* VSD_DTYPE_DRHIST64 */ + sizeof(struct voistatdata_dvhist64), /* VSD_DTYPE_DVHIST64 */ + sizeof(struct voistatdata_tdgstclust32),/* VSD_DTYPE_TDGSTCLUST32 */ + sizeof(struct voistatdata_tdgstclust64),/* VSD_DTYPE_TDGSTCLUST64 */ +}; + +static const bool vsd_compoundtype[VSD_NUM_DTYPES] = { + true, /* VSD_DTYPE_VOISTATE */ + false, /* VSD_DTYPE_INT_S32 */ + false, /* VSD_DTYPE_INT_U32 */ + false, /* VSD_DTYPE_INT_S64 */ + false, /* VSD_DTYPE_INT_U64 */ + false, /* VSD_DTYPE_INT_SLONG */ + false, /* VSD_DTYPE_INT_ULONG */ + false, /* VSD_DTYPE_Q_S32 */ + false, /* VSD_DTYPE_Q_U32 */ + false, /* VSD_DTYPE_Q_S64 */ + false, /* VSD_DTYPE_Q_U64 */ + true, /* VSD_DTYPE_CRHIST32 */ + true, /* VSD_DTYPE_DRHIST32 */ + true, /* VSD_DTYPE_DVHIST32 */ + true, /* VSD_DTYPE_CRHIST64 */ + true, /* VSD_DTYPE_DRHIST64 */ + true, /* VSD_DTYPE_DVHIST64 */ + true, /* VSD_DTYPE_TDGSTCLUST32 */ + true, /* VSD_DTYPE_TDGSTCLUST64 */ +}; + +const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1] = { + { /* LIM_MIN */ + {}, /* VSD_DTYPE_VOISTATE dummy */ + {.int32={.s32=INT32_MIN}}, /* VSD_DTYPE_INT_S32 */ + {.int32={.u32=0}}, /* VSD_DTYPE_INT_U32 */ + {.int64={.s64=INT64_MIN}}, /* VSD_DTYPE_INT_S64 */ + {.int64={.u64=0}}, /* VSD_DTYPE_INT_U64 */ + {.intlong={.slong=LONG_MIN}}, /* VSD_DTYPE_INT_SLONG */ + {.intlong={.ulong=0}}, /* VSD_DTYPE_INT_ULONG */ + {.q32={.sq32=Q_IFMINVAL(INT32_MIN)}}, /* VSD_DTYPE_Q_S32 */ + {.q32={.uq32=0}}, /* VSD_DTYPE_Q_U32 */ + {.q64={.sq64=Q_IFMINVAL(INT64_MIN)}}, /* VSD_DTYPE_Q_S64 */ + {.q64={.uq64=0}}, /* VSD_DTYPE_Q_U64 */ + }, + { /* LIM_MAX */ + {}, /* VSD_DTYPE_VOISTATE dummy */ + {.int32={.s32=INT32_MAX}}, /* VSD_DTYPE_INT_S32 */ + {.int32={.u32=UINT32_MAX}}, /* VSD_DTYPE_INT_U32 */ + {.int64={.s64=INT64_MAX}}, /* VSD_DTYPE_INT_S64 */ + {.int64={.u64=UINT64_MAX}}, /* VSD_DTYPE_INT_U64 */ + {.intlong={.slong=LONG_MAX}}, /* VSD_DTYPE_INT_SLONG */ + {.intlong={.ulong=ULONG_MAX}}, /* VSD_DTYPE_INT_ULONG */ + {.q32={.sq32=Q_IFMAXVAL(INT32_MAX)}}, /* VSD_DTYPE_Q_S32 */ + {.q32={.uq32=Q_IFMAXVAL(UINT32_MAX)}},/* VSD_DTYPE_Q_U32 */ + {.q64={.sq64=Q_IFMAXVAL(INT64_MAX)}}, /* VSD_DTYPE_Q_S64 */ + {.q64={.uq64=Q_IFMAXVAL(UINT64_MAX)}},/* VSD_DTYPE_Q_U64 */ + } +}; + +/* tpllistlock protects tpllist and ntpl */ +static uint32_t ntpl; +static struct statsblob_tpl **tpllist; + +static inline void * stats_realloc(void *ptr, size_t oldsz, size_t newsz, + int flags); +//static void stats_v1_blob_finalise(struct statsblobv1 *sb); +static int stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, + uint32_t flags); +static int stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, + int newvoistatbytes, int newvoistatdatabytes); +static void stats_v1_blob_iter(struct statsblobv1 *sb, + stats_v1_blob_itercb_t icb, void *usrctx, uint32_t flags); +static inline int stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, + struct voistatdata_tdgst *tdgst, s64q_t x, uint64_t weight, int attempt); + +static inline int +ctd32cmp(const struct voistatdata_tdgstctd32 *c1, const struct voistatdata_tdgstctd32 *c2) +{ + + KASSERT(Q_PRECEQ(c1->mu, c2->mu), + ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, + Q_RELPREC(c1->mu, c2->mu))); + + return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); +} +ARB_GENERATE_STATIC(ctdth32, voistatdata_tdgstctd32, ctdlnk, ctd32cmp); + +static inline int +ctd64cmp(const struct voistatdata_tdgstctd64 *c1, const struct voistatdata_tdgstctd64 *c2) +{ + + KASSERT(Q_PRECEQ(c1->mu, c2->mu), + ("%s: Q_RELPREC(c1->mu,c2->mu)=%d", __func__, + Q_RELPREC(c1->mu, c2->mu))); + + return (Q_QLTQ(c1->mu, c2->mu) ? -1 : 1); +} +ARB_GENERATE_STATIC(ctdth64, voistatdata_tdgstctd64, ctdlnk, ctd64cmp); + +#ifdef DIAGNOSTIC +RB_GENERATE_STATIC(rbctdth32, voistatdata_tdgstctd32, rblnk, ctd32cmp); +RB_GENERATE_STATIC(rbctdth64, voistatdata_tdgstctd64, rblnk, ctd64cmp); +#endif + +static inline sbintime_t +stats_sbinuptime() +{ + sbintime_t sbt; +#ifdef _KERNEL + + sbt = sbinuptime(); +#else /* ! _KERNEL */ + struct timespec tp; + + clock_gettime(CLOCK_MONOTONIC_FAST, &tp); + sbt = tstosbt(tp); +#endif /* _KERNEL */ + + return (sbt); +} + +static inline void * +stats_realloc(void *ptr, size_t oldsz, size_t newsz, int flags) +{ + +#ifdef _KERNEL + /* Default to M_NOWAIT if neither M_NOWAIT or M_WAITOK are set. */ + if (!(flags & (M_WAITOK | M_NOWAIT))) + flags |= M_NOWAIT; + ptr = realloc(ptr, newsz, M_STATS, flags); +#else /* ! _KERNEL */ + ptr = realloc(ptr, newsz); + if ((flags & M_ZERO) && ptr != NULL) { + if (oldsz == 0) + memset(ptr, '\0', newsz); + else if (newsz > oldsz) + memset(BLOB_OFFSET(ptr, oldsz), '\0', newsz - oldsz); + } +#endif /* _KERNEL */ + + return (ptr); +} + +static inline char * +stats_strdup(const char *s, +#ifdef _KERNEL + int flags) +{ + char *copy; + size_t len; + + if (!(flags & (M_WAITOK | M_NOWAIT))) + flags |= M_NOWAIT; + + len = strlen(s) + 1; + if ((copy = malloc(len, M_STATS, flags)) != NULL) + bcopy(s, copy, len); + + return (copy); +#else + int flags __unused) +{ + return (strdup(s)); +#endif +} + +static inline void +stats_tpl_update_hash(struct statsblob_tpl *tpl) +{ + + TPL_LIST_WLOCK_ASSERT(); + tpl->mb->tplhash = hash32_str(tpl->mb->tplname, 0); + for (int voi_id = 0; voi_id < NVOIS(tpl->sb); voi_id++) { + if (tpl->mb->voi_meta[voi_id].name != NULL) + tpl->mb->tplhash = hash32_str( + tpl->mb->voi_meta[voi_id].name, tpl->mb->tplhash); + } + tpl->mb->tplhash = hash32_buf(tpl->sb, tpl->sb->cursz, + tpl->mb->tplhash); +} + +static inline uint64_t +stats_pow_u64(uint64_t base, uint64_t exp) +{ + uint64_t result = 1; + + while (exp) { + if (exp & 1) + result *= base; + exp >>= 1; + base *= base; + } + + return (result); +} + +static inline int +stats_vss_hist_bkt_hlpr(struct vss_hist_hlpr_info *info, uint32_t curbkt, + struct voistatdata_numeric *bkt_lb, struct voistatdata_numeric *bkt_ub) +{ + uint64_t step = 0; + int error = 0; + + switch (info->scheme) { + case BKT_LIN: + step = info->lin.stepinc; + break; + case BKT_EXP: + step = stats_pow_u64(info->exp.stepbase, + info->exp.stepexp + curbkt); + break; + case BKT_LINEXP: + { + uint64_t curstepexp = 1; + + switch (info->voi_dtype) { + case VSD_DTYPE_INT_S32: + while ((int32_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= bkt_lb->int32.s32) + curstepexp++; + break; + case VSD_DTYPE_INT_U32: + while ((uint32_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= bkt_lb->int32.u32) + curstepexp++; + break; + case VSD_DTYPE_INT_S64: + while ((int64_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= bkt_lb->int64.s64) + curstepexp++; + break; + case VSD_DTYPE_INT_U64: + while ((uint64_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= bkt_lb->int64.u64) + curstepexp++; + break; + case VSD_DTYPE_INT_SLONG: + while ((long)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= bkt_lb->intlong.slong) + curstepexp++; + break; + case VSD_DTYPE_INT_ULONG: + while ((unsigned long)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= bkt_lb->intlong.ulong) + curstepexp++; + break; + case VSD_DTYPE_Q_S32: + while ((s32q_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= Q_GIVAL(bkt_lb->q32.sq32)) + break; + case VSD_DTYPE_Q_U32: + while ((u32q_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= Q_GIVAL(bkt_lb->q32.uq32)) + break; + case VSD_DTYPE_Q_S64: + while ((s64q_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= Q_GIVAL(bkt_lb->q64.sq64)) + curstepexp++; + break; + case VSD_DTYPE_Q_U64: + while ((u64q_t)stats_pow_u64(info->linexp.stepbase, + curstepexp) <= Q_GIVAL(bkt_lb->q64.uq64)) + curstepexp++; + break; + default: + break; + } + + step = stats_pow_u64(info->linexp.stepbase, curstepexp) / + info->linexp.linstepdiv; + if (step == 0) + step = 1; + break; + } + default: + break; + } + + if (info->scheme == BKT_USR) { + *bkt_lb = info->usr.bkts[curbkt].lb; + *bkt_ub = info->usr.bkts[curbkt].ub; + } else if (step != 0) { + switch (info->voi_dtype) { + case VSD_DTYPE_INT_S32: + bkt_ub->int32.s32 += (int32_t)step; + break; + case VSD_DTYPE_INT_U32: + bkt_ub->int32.u32 += (uint32_t)step; + break; + case VSD_DTYPE_INT_S64: + bkt_ub->int64.s64 += (int64_t)step; + break; + case VSD_DTYPE_INT_U64: + bkt_ub->int64.u64 += (uint64_t)step; + break; + case VSD_DTYPE_INT_SLONG: + bkt_ub->intlong.slong += (long)step; + break; + case VSD_DTYPE_INT_ULONG: + bkt_ub->intlong.ulong += (unsigned long)step; + break; + case VSD_DTYPE_Q_S32: + error = Q_QADDI(&bkt_ub->q32.sq32, step); + break; + case VSD_DTYPE_Q_U32: + error = Q_QADDI(&bkt_ub->q32.uq32, step); + break; + case VSD_DTYPE_Q_S64: + error = Q_QADDI(&bkt_ub->q64.sq64, step); + break; + case VSD_DTYPE_Q_U64: + error = Q_QADDI(&bkt_ub->q64.uq64, step); + break; + default: + break; + } + } else { /* info->scheme != BKT_USR && step == 0 */ + return (EINVAL); + } + + return (error); +} + +static uint32_t +stats_vss_hist_nbkts_hlpr(struct vss_hist_hlpr_info *info) +{ + struct voistatdata_numeric bkt_lb, bkt_ub; + uint32_t nbkts; + int done; + + if (info->scheme == BKT_USR) { + /* XXXLAS: Setting info->{lb,ub} from macro is tricky. */ + info->lb = info->usr.bkts[0].lb; + info->ub = info->usr.bkts[info->usr.nbkts - 1].lb; + } + + nbkts = 0; + done = 0; + bkt_ub = info->lb; + + do { + bkt_lb = bkt_ub; + if (stats_vss_hist_bkt_hlpr(info, nbkts++, &bkt_lb, &bkt_ub)) + return (0); + + if (info->scheme == BKT_USR) + done = (nbkts == info->usr.nbkts); + else { + switch (info->voi_dtype) { + case VSD_DTYPE_INT_S32: + done = (bkt_ub.int32.s32 > info->ub.int32.s32); + break; + case VSD_DTYPE_INT_U32: + done = (bkt_ub.int32.u32 > info->ub.int32.u32); + break; + case VSD_DTYPE_INT_S64: + done = (bkt_ub.int64.s64 > info->ub.int64.s64); + break; + case VSD_DTYPE_INT_U64: + done = (bkt_ub.int64.u64 > info->ub.int64.u64); + break; + case VSD_DTYPE_INT_SLONG: + done = (bkt_ub.intlong.slong > + info->ub.intlong.slong); + break; + case VSD_DTYPE_INT_ULONG: + done = (bkt_ub.intlong.ulong > + info->ub.intlong.ulong); + break; + case VSD_DTYPE_Q_S32: + done = Q_QGTQ(bkt_ub.q32.sq32, + info->ub.q32.sq32); + break; + case VSD_DTYPE_Q_U32: + done = Q_QGTQ(bkt_ub.q32.uq32, + info->ub.q32.uq32); + break; + case VSD_DTYPE_Q_S64: + done = Q_QGTQ(bkt_ub.q64.sq64, + info->ub.q64.sq64); + break; + case VSD_DTYPE_Q_U64: + done = Q_QGTQ(bkt_ub.q64.uq64, + info->ub.q64.uq64); + break; + default: + return (0); + } + } + } while (!done); + + if (info->flags & VSD_HIST_LBOUND_INF) + nbkts++; + if (info->flags & VSD_HIST_UBOUND_INF) + nbkts++; + + return (nbkts); +} + +int +stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, + struct vss_hist_hlpr_info *info) +{ + struct voistatdata_hist *hist; + struct voistatdata_numeric bkt_lb, bkt_ub, *lbinfbktlb, *lbinfbktub, + *ubinfbktlb, *ubinfbktub; + uint32_t bkt, nbkts, nloop; + + if (vss == NULL || info == NULL || (info->flags & + (VSD_HIST_LBOUND_INF|VSD_HIST_UBOUND_INF) && (info->hist_dtype == + VSD_DTYPE_DVHIST32 || info->hist_dtype == VSD_DTYPE_DVHIST64))) + return (EINVAL); + + info->voi_dtype = voi_dtype; + + if ((nbkts = stats_vss_hist_nbkts_hlpr(info)) == 0) + return (EINVAL); + + switch (info->hist_dtype) { + case VSD_DTYPE_CRHIST32: + vss->vsdsz = HIST_NBKTS2VSDSZ(crhist32, nbkts); + break; + case VSD_DTYPE_DRHIST32: + vss->vsdsz = HIST_NBKTS2VSDSZ(drhist32, nbkts); + break; + case VSD_DTYPE_DVHIST32: + vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist32, nbkts); + break; + case VSD_DTYPE_CRHIST64: + vss->vsdsz = HIST_NBKTS2VSDSZ(crhist64, nbkts); + break; + case VSD_DTYPE_DRHIST64: + vss->vsdsz = HIST_NBKTS2VSDSZ(drhist64, nbkts); + break; + case VSD_DTYPE_DVHIST64: + vss->vsdsz = HIST_NBKTS2VSDSZ(dvhist64, nbkts); + break; + default: + return (EINVAL); + } + + vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); + if (vss->iv == NULL) + return (ENOMEM); + + hist = (struct voistatdata_hist *)vss->iv; + bkt_ub = info->lb; + + for (bkt = (info->flags & VSD_HIST_LBOUND_INF), nloop = 0; + bkt < nbkts; + bkt++, nloop++) { + bkt_lb = bkt_ub; + if (stats_vss_hist_bkt_hlpr(info, nloop, &bkt_lb, &bkt_ub)) + return (EINVAL); + + switch (info->hist_dtype) { + case VSD_DTYPE_CRHIST32: + VSD(crhist32, hist)->bkts[bkt].lb = bkt_lb; + break; + case VSD_DTYPE_DRHIST32: + VSD(drhist32, hist)->bkts[bkt].lb = bkt_lb; + VSD(drhist32, hist)->bkts[bkt].ub = bkt_ub; + break; + case VSD_DTYPE_DVHIST32: + VSD(dvhist32, hist)->bkts[bkt].val = bkt_lb; + break; + case VSD_DTYPE_CRHIST64: + VSD(crhist64, hist)->bkts[bkt].lb = bkt_lb; + break; + case VSD_DTYPE_DRHIST64: + VSD(drhist64, hist)->bkts[bkt].lb = bkt_lb; + VSD(drhist64, hist)->bkts[bkt].ub = bkt_ub; + break; + case VSD_DTYPE_DVHIST64: + VSD(dvhist64, hist)->bkts[bkt].val = bkt_lb; + break; + default: + return (EINVAL); + } + } + + lbinfbktlb = lbinfbktub = ubinfbktlb = ubinfbktub = NULL; + + switch (info->hist_dtype) { + case VSD_DTYPE_CRHIST32: + lbinfbktlb = &VSD(crhist32, hist)->bkts[0].lb; + ubinfbktlb = &VSD(crhist32, hist)->bkts[nbkts - 1].lb; + break; + case VSD_DTYPE_DRHIST32: + lbinfbktlb = &VSD(drhist32, hist)->bkts[0].lb; + lbinfbktub = &VSD(drhist32, hist)->bkts[0].ub; + ubinfbktlb = &VSD(drhist32, hist)->bkts[nbkts - 1].lb; + ubinfbktub = &VSD(drhist32, hist)->bkts[nbkts - 1].ub; + break; + case VSD_DTYPE_CRHIST64: + lbinfbktlb = &VSD(crhist64, hist)->bkts[0].lb; + ubinfbktlb = &VSD(crhist64, hist)->bkts[nbkts - 1].lb; + break; + case VSD_DTYPE_DRHIST64: + lbinfbktlb = &VSD(drhist64, hist)->bkts[0].lb; + lbinfbktub = &VSD(drhist64, hist)->bkts[0].ub; + ubinfbktlb = &VSD(drhist64, hist)->bkts[nbkts - 1].lb; + ubinfbktub = &VSD(drhist64, hist)->bkts[nbkts - 1].ub; + break; + case VSD_DTYPE_DVHIST32: + case VSD_DTYPE_DVHIST64: + break; + default: + return (EINVAL); + } + + if ((info->flags & VSD_HIST_LBOUND_INF) && lbinfbktlb) { + *lbinfbktlb = numeric_limits[LIM_MIN][info->voi_dtype]; + /* + * Assignment from numeric_limit array for Q types assigns max + * possible integral/fractional value for underlying data type, + * but we must set control bits for this specific histogram per + * the user's choice of fractional bits, which we extract from + * info->lb. + */ + if (info->voi_dtype == VSD_DTYPE_Q_S32 || + info->voi_dtype == VSD_DTYPE_Q_U32) { + /* Signedness doesn't matter for setting control bits. */ + Q_SCVAL(lbinfbktlb->q32.sq32, + Q_GCVAL(info->lb.q32.sq32)); + } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || + info->voi_dtype == VSD_DTYPE_Q_U64) { + /* Signedness doesn't matter for setting control bits. */ + Q_SCVAL(lbinfbktlb->q64.sq64, + Q_GCVAL(info->lb.q64.sq64)); + } + if (lbinfbktub) + *lbinfbktub = info->lb; + } + if ((info->flags & VSD_HIST_UBOUND_INF) && ubinfbktlb) { + *ubinfbktlb = bkt_lb; + if (ubinfbktub) { + *ubinfbktub = numeric_limits[LIM_MAX][info->voi_dtype]; + if (info->voi_dtype == VSD_DTYPE_Q_S32 || + info->voi_dtype == VSD_DTYPE_Q_U32) { + Q_SCVAL(ubinfbktub->q32.sq32, + Q_GCVAL(info->lb.q32.sq32)); + } else if (info->voi_dtype == VSD_DTYPE_Q_S64 || + info->voi_dtype == VSD_DTYPE_Q_U64) { + Q_SCVAL(ubinfbktub->q64.sq64, + Q_GCVAL(info->lb.q64.sq64)); + } + } + } + + return (0); +} + +int +stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, + struct vss_tdgst_hlpr_info *info) +{ + struct voistatdata_tdgst *tdgst; + struct ctdth32 *ctd32tree; + struct ctdth64 *ctd64tree; + struct voistatdata_tdgstctd32 *ctd32; + struct voistatdata_tdgstctd64 *ctd64; + + info->voi_dtype = voi_dtype; + + switch (info->tdgst_dtype) { + case VSD_DTYPE_TDGSTCLUST32: + vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust32, info->nctds); + break; + case VSD_DTYPE_TDGSTCLUST64: + vss->vsdsz = TDGST_NCTRS2VSDSZ(tdgstclust64, info->nctds); + break; + default: + return (EINVAL); + } + + vss->iv = stats_realloc(NULL, 0, vss->vsdsz, M_ZERO); + if (vss->iv == NULL) + return (ENOMEM); + + tdgst = (struct voistatdata_tdgst *)vss->iv; + + switch (info->tdgst_dtype) { + case VSD_DTYPE_TDGSTCLUST32: + ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; + ARB_INIT(ctd32, ctdlnk, ctd32tree, info->nctds) { + Q_INI(&ctd32->mu, 0, info->prec); + } + break; + case VSD_DTYPE_TDGSTCLUST64: + ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; + ARB_INIT(ctd64, ctdlnk, ctd64tree, info->nctds) { + Q_INI(&ctd64->mu, 0, info->prec); + } + break; + default: + return (EINVAL); + } + + return (0); +} + +int +stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, + struct vss_numeric_hlpr_info *info) +{ + struct voistatdata_numeric iv; + + switch (vss->stype) { + case VS_STYPE_SUM: + iv = stats_ctor_vsd_numeric(0); + break; + case VS_STYPE_MIN: + iv = numeric_limits[LIM_MAX][voi_dtype]; + break; + case VS_STYPE_MAX: + iv = numeric_limits[LIM_MIN][voi_dtype]; + break; + default: + return (EINVAL); + } + + vss->iv = stats_realloc(NULL, 0, vsd_dtype2size[voi_dtype], 0); + if (vss->iv == NULL) + return (ENOMEM); + + vss->vs_dtype = voi_dtype; + vss->vsdsz = vsd_dtype2size[voi_dtype]; + switch (voi_dtype) { + case VSD_DTYPE_INT_S32: + *((int32_t *)vss->iv) = iv.int32.s32; + break; + case VSD_DTYPE_INT_U32: + *((uint32_t *)vss->iv) = iv.int32.u32; + break; + case VSD_DTYPE_INT_S64: + *((int64_t *)vss->iv) = iv.int64.s64; + break; + case VSD_DTYPE_INT_U64: + *((uint64_t *)vss->iv) = iv.int64.u64; + break; + case VSD_DTYPE_INT_SLONG: + *((long *)vss->iv) = iv.intlong.slong; + break; + case VSD_DTYPE_INT_ULONG: + *((unsigned long *)vss->iv) = iv.intlong.ulong; + break; + case VSD_DTYPE_Q_S32: + *((s32q_t *)vss->iv) = Q_SCVAL(iv.q32.sq32, + Q_CTRLINI(info->prec)); + break; + case VSD_DTYPE_Q_U32: + *((u32q_t *)vss->iv) = Q_SCVAL(iv.q32.uq32, + Q_CTRLINI(info->prec)); + break; + case VSD_DTYPE_Q_S64: + *((s64q_t *)vss->iv) = Q_SCVAL(iv.q64.sq64, + Q_CTRLINI(info->prec)); + break; + case VSD_DTYPE_Q_U64: + *((u64q_t *)vss->iv) = Q_SCVAL(iv.q64.uq64, + Q_CTRLINI(info->prec)); + break; + default: + break; + } + + return (0); +} + +int +stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss, + struct voistatspec *vss) +{ + int i, ret; + + for (i = nvss - 1; i >= 0; i--) { + if (vss[i].hlpr && (ret = vss[i].hlpr(voi_dtype, &vss[i], + vss[i].hlprinfo)) != 0) + return (ret); + } + + return (0); +} + +void +stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss) +{ + int i; + + for (i = nvss - 1; i >= 0; i--) { + if (vss[i].hlpr) { + stats_free((void *)vss[i].iv); + vss[i].iv = NULL; + } + } +} + +int +stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl) +{ + int error; + + error = 0; + + TPL_LIST_WLOCK(); + if (tpl_id < 0 || tpl_id >= (int)ntpl) { + error = ENOENT; + } else { + *tpl = tpllist[tpl_id]; + /* XXXLAS: Acquire refcount on tpl. */ + } + TPL_LIST_WUNLOCK(); + + return (error); +} + +int +stats_tpl_fetch_allocid(const char *name, uint32_t hash) +{ + int i, tpl_id; + + tpl_id = -ESRCH; + + TPL_LIST_RLOCK(); + for (i = ntpl - 1; i >= 0; i--) { + if (name != NULL) { + if (strlen(name) == strlen(tpllist[i]->mb->tplname) && + strncmp(name, tpllist[i]->mb->tplname, + TPL_MAX_NAME_LEN) == 0 && (!hash || hash == + tpllist[i]->mb->tplhash)) { + tpl_id = i; + break; + } + } else if (hash == tpllist[i]->mb->tplhash) { + tpl_id = i; + break; + } + } + TPL_LIST_RUNLOCK(); + + return (tpl_id); +} + +int +stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len) +{ + int error; + + error = 0; + + TPL_LIST_RLOCK(); + if (tpl_id < ntpl) { + if (buf != NULL && len > strlen(tpllist[tpl_id]->mb->tplname)) + strlcpy(buf, tpllist[tpl_id]->mb->tplname, len); + else + error = EOVERFLOW; + } else + error = ENOENT; + TPL_LIST_RUNLOCK(); + + return (error); +} + +int +stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, int nrates, + void *seed_bytes, size_t seed_len) +{ + uint32_t cum_pct, rnd_pct; + int i; + + cum_pct = 0; + + /* + * Choose a true random or seeded random number in range [0,100] and use + * it to make a sampling decision and template selection where required. + * If no seed is supplied, a PRNG is used to generate a true random + * number so that every selection is independent. If a seed is supplied, + * the caller desires random selection across different seeds, but + * deterministic selection given the same seed. This is achieved by + * hashing the seed and using the hash as the random number source. + * + * XXXLAS: Characterise hash function output distribution. + */ + if (seed_bytes == NULL) + rnd_pct = random() / (INT32_MAX / 100); + else + rnd_pct = hash32_buf(seed_bytes, seed_len, 0) / + (UINT32_MAX / 100U); + + /* + * We map the randomly selected percentage on to the interval [0,100] + * consisting of the cumulatively summed template sampling percentages. + * The difference between the cumulative sum of all template sampling + * percentages and 100 is treated as a NULL assignment i.e. no stats + * template will be assigned, and -1 returned instead. + */ + for (i = 0; i < nrates; i++) { + cum_pct += rates[i].tpl_sample_pct; + + KASSERT(cum_pct <= 100, ("%s cum_pct %u > 100", __func__, + cum_pct)); + if (rnd_pct > cum_pct || rates[i].tpl_sample_pct == 0) + continue; + + return (rates[i].tpl_slot_id); + } + + return (-1); +} + +int +stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz, + struct statsblobv1 *src, uint32_t flags) +{ + int error; + + error = 0; + + if (src == NULL || dst == NULL || + src->cursz < sizeof(struct statsblob) || + ((flags & SB_CLONE_ALLOCDST) && + (flags & (SB_CLONE_USRDSTNOFAULT | SB_CLONE_USRDST)))) { + error = EINVAL; + } else if (flags & SB_CLONE_ALLOCDST) { + *dst = stats_realloc(NULL, 0, src->cursz, 0); + if (*dst) + (*dst)->maxsz = dstmaxsz = src->cursz; + else + error = ENOMEM; + } else if (*dst == NULL || dstmaxsz < sizeof(struct statsblob)) { + error = EINVAL; + } + + if (!error) { + size_t postcurszlen; + + /* + * Clone src into dst except for the maxsz field. If dst is too + * small to hold all of src, only copy src's header and return + * EOVERFLOW. + */ +#ifdef _KERNEL + if (flags & SB_CLONE_USRDSTNOFAULT) + copyout_nofault(src, *dst, + offsetof(struct statsblob, maxsz)); + else if (flags & SB_CLONE_USRDST) + copyout(src, *dst, offsetof(struct statsblob, maxsz)); + else +#endif + memcpy(*dst, src, offsetof(struct statsblob, maxsz)); + + if (dstmaxsz >= src->cursz) { + postcurszlen = src->cursz - + offsetof(struct statsblob, cursz); + } else { + error = EOVERFLOW; + postcurszlen = sizeof(struct statsblob) - + offsetof(struct statsblob, cursz); + } +#ifdef _KERNEL + if (flags & SB_CLONE_USRDSTNOFAULT) + copyout_nofault(&(src->cursz), &((*dst)->cursz), + postcurszlen); + else if (flags & SB_CLONE_USRDST) + copyout(&(src->cursz), &((*dst)->cursz), postcurszlen); + else +#endif + memcpy(&((*dst)->cursz), &(src->cursz), postcurszlen); + } + + return (error); +} + +int +stats_v1_tpl_alloc(const char *name, uint32_t flags __unused) +{ + struct statsblobv1_tpl *tpl, **newtpllist; + struct statsblobv1 *tpl_sb; + struct metablob *tpl_mb; + int tpl_id; + + if (name != NULL && strlen(name) > TPL_MAX_NAME_LEN) + return (-EINVAL); + + if (name != NULL && stats_tpl_fetch_allocid(name, 0) >= 0) + return (-EEXIST); + + tpl = stats_realloc(NULL, 0, sizeof(struct statsblobv1_tpl), M_ZERO); + tpl_mb = stats_realloc(NULL, 0, sizeof(struct metablob), M_ZERO); + tpl_sb = stats_realloc(NULL, 0, sizeof(struct statsblobv1), M_ZERO); + + if (tpl_mb != NULL && name != NULL) + tpl_mb->tplname = stats_strdup(name, 0); + + if (tpl == NULL || tpl_sb == NULL || tpl_mb == NULL || + tpl_mb->tplname == NULL) { + stats_free(tpl); + stats_free(tpl_sb); + if (tpl_mb != NULL) { + stats_free(tpl_mb->tplname); + stats_free(tpl_mb); + } + return (-ENOMEM); + } + + tpl->mb = tpl_mb; + tpl->sb = tpl_sb; + + tpl_sb->abi = STATS_ABI_V1; + tpl_sb->endian = +#if BYTE_ORDER == LITTLE_ENDIAN + SB_LE; +#elif BYTE_ORDER == BIG_ENDIAN + SB_BE; +#else + SB_UE; +#endif + tpl_sb->cursz = tpl_sb->maxsz = sizeof(struct statsblobv1); + tpl_sb->stats_off = tpl_sb->statsdata_off = sizeof(struct statsblobv1); + + TPL_LIST_WLOCK(); + newtpllist = stats_realloc(tpllist, ntpl * sizeof(void *), + (ntpl + 1) * sizeof(void *), 0); + if (newtpllist != NULL) { + tpl_id = ntpl++; + tpllist = (struct statsblob_tpl **)newtpllist; + tpllist[tpl_id] = (struct statsblob_tpl *)tpl; + stats_tpl_update_hash(tpllist[tpl_id]); + } else { + stats_free(tpl); + stats_free(tpl_sb); + if (tpl_mb != NULL) { + stats_free(tpl_mb->tplname); + stats_free(tpl_mb); + } + tpl_id = -ENOMEM; + } + TPL_LIST_WUNLOCK(); + + return (tpl_id); +} + +int +stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name, + enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss, + uint32_t flags) +{ + struct voi *voi; + struct voistat *tmpstat; + struct statsblobv1 *tpl_sb; + struct metablob *tpl_mb; + int error, i, newstatdataidx, newvoibytes, newvoistatbytes, + newvoistatdatabytes, newvoistatmaxid; + uint32_t nbytes; + + if (voi_id < 0 || voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || + nvss == 0 || vss == NULL) + return (EINVAL); + + error = nbytes = newvoibytes = newvoistatbytes = + newvoistatdatabytes = 0; + newvoistatmaxid = -1; + + /* Calculate the number of bytes required for the new voistats. */ + for (i = nvss - 1; i >= 0; i--) { + if (vss[i].stype == 0 || vss[i].stype >= VS_NUM_STYPES || + vss[i].vs_dtype == 0 || vss[i].vs_dtype >= VSD_NUM_DTYPES || + vss[i].iv == NULL || vss[i].vsdsz == 0) + return (EINVAL); + if ((int)vss[i].stype > newvoistatmaxid) + newvoistatmaxid = vss[i].stype; + newvoistatdatabytes += vss[i].vsdsz; + } + + if (flags & SB_VOI_RELUPDATE) { + /* XXXLAS: VOI state bytes may need to vary based on stat types. */ + newvoistatdatabytes += sizeof(struct voistatdata_voistate); + } + nbytes += newvoistatdatabytes; + + TPL_LIST_WLOCK(); + if (tpl_id < ntpl) { + tpl_sb = (struct statsblobv1 *)tpllist[tpl_id]->sb; + tpl_mb = tpllist[tpl_id]->mb; + + if (voi_id >= NVOIS(tpl_sb) || tpl_sb->vois[voi_id].id == -1) { + /* Adding a new VOI and associated stats. */ + if (voi_id >= NVOIS(tpl_sb)) { + /* We need to grow the tpl_sb->vois array. */ + newvoibytes = (voi_id - (NVOIS(tpl_sb) - 1)) * + sizeof(struct voi); + nbytes += newvoibytes; + } + newvoistatbytes = + (newvoistatmaxid + 1) * sizeof(struct voistat); + } else { + /* Adding stats to an existing VOI. */ + if (newvoistatmaxid > + tpl_sb->vois[voi_id].voistatmaxid) { + newvoistatbytes = (newvoistatmaxid - + tpl_sb->vois[voi_id].voistatmaxid) * + sizeof(struct voistat); + } + /* XXXLAS: KPI does not yet support expanding VOIs. */ + error = EOPNOTSUPP; + } + nbytes += newvoistatbytes; + + if (!error && newvoibytes > 0) { + struct voi_meta *voi_meta = tpl_mb->voi_meta; + + voi_meta = stats_realloc(voi_meta, voi_meta == NULL ? + 0 : NVOIS(tpl_sb) * sizeof(struct voi_meta), + (1 + voi_id) * sizeof(struct voi_meta), + M_ZERO); + + if (voi_meta == NULL) + error = ENOMEM; + else + tpl_mb->voi_meta = voi_meta; + } + + if (!error) { + /* NB: Resizing can change where tpl_sb points. */ + error = stats_v1_blob_expand(&tpl_sb, newvoibytes, + newvoistatbytes, newvoistatdatabytes); + } + + if (!error) { + tpl_mb->voi_meta[voi_id].name = stats_strdup(voi_name, + 0); + if (tpl_mb->voi_meta[voi_id].name == NULL) + error = ENOMEM; + } + + if (!error) { + /* Update the template list with the resized pointer. */ + tpllist[tpl_id]->sb = (struct statsblob *)tpl_sb; + + /* Update the template. */ + voi = &tpl_sb->vois[voi_id]; + + if (voi->id < 0) { + /* VOI is new and needs to be initialised. */ + voi->id = voi_id; + voi->dtype = voi_dtype; + voi->stats_off = tpl_sb->stats_off; + if (flags & SB_VOI_RELUPDATE) + voi->flags |= VOI_REQSTATE; + } else { + /* + * XXXLAS: When this else block is written, the + * "KPI does not yet support expanding VOIs" + * error earlier in this function can be + * removed. What is required here is to shuffle + * the voistat array such that the new stats for + * the voi are contiguous, which will displace + * stats for other vois that reside after the + * voi being updated. The other vois then need + * to have their stats_off adjusted post + * shuffle. + */ + } + + voi->voistatmaxid = newvoistatmaxid; + newstatdataidx = 0; + + if (voi->flags & VOI_REQSTATE) { + /* Initialise the voistate stat in slot 0. */ + tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off); + tmpstat->stype = VS_STYPE_VOISTATE; + tmpstat->flags = 0; + tmpstat->dtype = VSD_DTYPE_VOISTATE; + newstatdataidx = tmpstat->dsz = + sizeof(struct voistatdata_numeric); + tmpstat->data_off = tpl_sb->statsdata_off; + } + + for (i = 0; (uint32_t)i < nvss; i++) { + tmpstat = BLOB_OFFSET(tpl_sb, voi->stats_off + + (vss[i].stype * sizeof(struct voistat))); + KASSERT(tmpstat->stype < 0, ("voistat %p " + "already initialised", tmpstat)); + tmpstat->stype = vss[i].stype; + tmpstat->flags = vss[i].flags; + tmpstat->dtype = vss[i].vs_dtype; + tmpstat->dsz = vss[i].vsdsz; + tmpstat->data_off = tpl_sb->statsdata_off + + newstatdataidx; + memcpy(BLOB_OFFSET(tpl_sb, tmpstat->data_off), + vss[i].iv, vss[i].vsdsz); + newstatdataidx += vss[i].vsdsz; + } + + /* Update the template version hash. */ + stats_tpl_update_hash(tpllist[tpl_id]); + /* XXXLAS: Confirm tpl name/hash pair remains unique. */ + } + } else + error = EINVAL; + TPL_LIST_WUNLOCK(); + + return (error); +} + +struct statsblobv1 * +stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags __unused) +{ + struct statsblobv1 *sb; + int error; + + sb = NULL; + + TPL_LIST_RLOCK(); + if (tpl_id < ntpl) { + sb = stats_realloc(NULL, 0, tpllist[tpl_id]->sb->maxsz, 0); + if (sb != NULL) { + sb->maxsz = tpllist[tpl_id]->sb->maxsz; + error = stats_v1_blob_init_locked(sb, tpl_id, 0); + } else + error = ENOMEM; + + if (error) { + stats_free(sb); + sb = NULL; + } + } + TPL_LIST_RUNLOCK(); + + return (sb); +} + +void +stats_v1_blob_destroy(struct statsblobv1 *sb) +{ + + stats_free(sb); +} + +int +stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id, + enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd, + size_t *retvsdsz) +{ + struct voi *v; + struct voistat *vs; + + if (retvsd == NULL || sb == NULL || sb->abi != STATS_ABI_V1 || + voi_id >= NVOIS(sb)) + return (EINVAL); + + v = &sb->vois[voi_id]; + if ((__typeof(v->voistatmaxid))stype > v->voistatmaxid) + return (EINVAL); + + vs = BLOB_OFFSET(sb, v->stats_off + (stype * sizeof(struct voistat))); + *retvsd = BLOB_OFFSET(sb, vs->data_off); + if (retdtype != NULL) + *retdtype = vs->dtype; + if (retvsdsz != NULL) + *retvsdsz = vs->dsz; + + return (0); +} + +int +stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags) +{ + int error; + + error = 0; + + TPL_LIST_RLOCK(); + if (sb == NULL || tpl_id >= ntpl) { + error = EINVAL; + } else { + error = stats_v1_blob_init_locked(sb, tpl_id, flags); + } + TPL_LIST_RUNLOCK(); + + return (error); +} + +static inline int +stats_v1_blob_init_locked(struct statsblobv1 *sb, uint32_t tpl_id, + uint32_t flags __unused) +{ + int error; + + TPL_LIST_RLOCK_ASSERT(); + error = (sb->maxsz >= tpllist[tpl_id]->sb->cursz) ? 0 : EOVERFLOW; + KASSERT(!error, + ("sb %d instead of %d bytes", sb->maxsz, tpllist[tpl_id]->sb->cursz)); + + if (!error) { + memcpy(sb, tpllist[tpl_id]->sb, tpllist[tpl_id]->sb->cursz); + sb->created = sb->lastrst = stats_sbinuptime(); + sb->tplhash = tpllist[tpl_id]->mb->tplhash; + } + + return (error); +} + +static int +stats_v1_blob_expand(struct statsblobv1 **sbpp, int newvoibytes, + int newvoistatbytes, int newvoistatdatabytes) +{ + struct statsblobv1 *sb; + struct voi *tmpvoi; + struct voistat *tmpvoistat, *voistat_array; + int error, i, idxnewvois, idxnewvoistats, nbytes, nvoistats; + + KASSERT(newvoibytes % sizeof(struct voi) == 0, + ("Bad newvoibytes %d", newvoibytes)); + KASSERT(newvoistatbytes % sizeof(struct voistat) == 0, + ("Bad newvoistatbytes %d", newvoistatbytes)); + + error = ((newvoibytes % sizeof(struct voi) == 0) && + (newvoistatbytes % sizeof(struct voistat) == 0)) ? 0 : EINVAL; + sb = *sbpp; + nbytes = newvoibytes + newvoistatbytes + newvoistatdatabytes; + + /* + * XXXLAS: Required until we gain support for flags which alter the + * units of size/offset fields in key structs. + */ + if (!error && ((((int)sb->cursz) + nbytes) > SB_V1_MAXSZ)) + error = EFBIG; + + if (!error && (sb->cursz + nbytes > sb->maxsz)) { + /* Need to expand our blob. */ + sb = stats_realloc(sb, sb->maxsz, sb->cursz + nbytes, M_ZERO); + if (sb != NULL) { + sb->maxsz = sb->cursz + nbytes; + *sbpp = sb; + } else + error = ENOMEM; + } + + if (!error) { + /* + * Shuffle memory within the expanded blob working from the end + * backwards, leaving gaps for the new voistat and voistatdata + * structs at the beginning of their respective blob regions, + * and for the new voi structs at the end of their blob region. + */ + memmove(BLOB_OFFSET(sb, sb->statsdata_off + nbytes), + BLOB_OFFSET(sb, sb->statsdata_off), + sb->cursz - sb->statsdata_off); + memmove(BLOB_OFFSET(sb, sb->stats_off + newvoibytes + + newvoistatbytes), BLOB_OFFSET(sb, sb->stats_off), + sb->statsdata_off - sb->stats_off); + + /* First index of new voi/voistat structs to be initialised. */ + idxnewvois = NVOIS(sb); + idxnewvoistats = (newvoistatbytes / sizeof(struct voistat)) - 1; + + /* Update housekeeping variables and offsets. */ + sb->cursz += nbytes; + sb->stats_off += newvoibytes; + sb->statsdata_off += newvoibytes + newvoistatbytes; + + /* XXXLAS: Zeroing not strictly needed but aids debugging. */ + memset(&sb->vois[idxnewvois], '\0', newvoibytes); + memset(BLOB_OFFSET(sb, sb->stats_off), '\0', + newvoistatbytes); + memset(BLOB_OFFSET(sb, sb->statsdata_off), '\0', + newvoistatdatabytes); + + /* Initialise new voi array members and update offsets. */ + for (i = 0; i < NVOIS(sb); i++) { + tmpvoi = &sb->vois[i]; + if (i >= idxnewvois) { + tmpvoi->id = tmpvoi->voistatmaxid = -1; + } else if (tmpvoi->id > -1) { + tmpvoi->stats_off += newvoibytes + + newvoistatbytes; + } + } + + /* Initialise new voistat array members and update offsets. */ + nvoistats = (sb->statsdata_off - sb->stats_off) / + sizeof(struct voistat); + voistat_array = BLOB_OFFSET(sb, sb->stats_off); + for (i = 0; i < nvoistats; i++) { + tmpvoistat = &voistat_array[i]; + if (i <= idxnewvoistats) { + tmpvoistat->stype = -1; + } else if (tmpvoistat->stype > -1) { + tmpvoistat->data_off += nbytes; + } + } + } + + return (error); +} + +static void +stats_v1_blob_finalise(struct statsblobv1 *sb __unused) +{ + + /* XXXLAS: Fill this in. */ +} + +static void +stats_v1_blob_iter(struct statsblobv1 *sb, stats_v1_blob_itercb_t icb, + void *usrctx, uint32_t flags) +{ + struct voi *v; + struct voistat *vs; + struct sb_iter_ctx ctx; + int i, j, firstvoi; + + ctx.usrctx = usrctx; + ctx.flags |= SB_IT_FIRST_CB; + ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT | + SB_IT_LAST_VOISTAT); + firstvoi = 1; + + for (i = 0; i < NVOIS(sb); i++) { + v = &sb->vois[i]; + ctx.vslot = i; + ctx.vsslot = -1; + ctx.flags |= SB_IT_FIRST_VOISTAT; + + if (firstvoi) + ctx.flags |= SB_IT_FIRST_VOI; + else if (i == (NVOIS(sb) - 1)) + ctx.flags |= SB_IT_LAST_VOI | SB_IT_LAST_CB; + + if (v->id < 0 && (flags & SB_IT_NULLVOI)) { + if (icb(sb, v, NULL, &ctx)) + return; + firstvoi = 0; + ctx.flags &= ~SB_IT_FIRST_CB; + } + + /* If NULL voi, v->voistatmaxid == -1 */ + for (j = 0; j <= v->voistatmaxid; j++) { + vs = &((struct voistat *)BLOB_OFFSET(sb, + v->stats_off))[j]; + if (vs->stype < 0 && + !(flags & SB_IT_NULLVOISTAT)) + continue; + + if (j == v->voistatmaxid) { + ctx.flags |= SB_IT_LAST_VOISTAT; + if (i == (NVOIS(sb) - 1)) + ctx.flags |= + SB_IT_LAST_CB; + } else + ctx.flags &= ~SB_IT_LAST_CB; + + ctx.vsslot = j; + if (icb(sb, v, vs, &ctx)) + return; + + ctx.flags &= ~(SB_IT_FIRST_CB | SB_IT_FIRST_VOISTAT | + SB_IT_LAST_VOISTAT); + } + ctx.flags &= ~(SB_IT_FIRST_VOI | SB_IT_LAST_VOI); + } +} + +static inline void +stats_voistatdata_tdgst_tostr(enum vsd_dtype voi_dtype __unused, + const struct voistatdata_tdgst *tdgst, enum vsd_dtype tdgst_dtype, + size_t tdgst_dsz __unused, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) +{ + const struct ctdth32 *ctd32tree; + const struct ctdth64 *ctd64tree; + const struct voistatdata_tdgstctd32 *ctd32; + const struct voistatdata_tdgstctd64 *ctd64; + const char *fmtstr; + uint64_t smplcnt, compcnt; + int is32bit, qmaxstrlen; + uint16_t maxctds, curctds; + + switch (tdgst_dtype) { + case VSD_DTYPE_TDGSTCLUST32: + smplcnt = CONSTVSD(tdgstclust32, tdgst)->smplcnt; + compcnt = CONSTVSD(tdgstclust32, tdgst)->compcnt; + maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); + curctds = ARB_CURNODES(&CONSTVSD(tdgstclust32, tdgst)->ctdtree); + ctd32tree = &CONSTVSD(tdgstclust32, tdgst)->ctdtree; + ctd32 = (objdump ? ARB_CNODE(ctd32tree, 0) : + ARB_CMIN(ctdth32, ctd32tree)); + qmaxstrlen = (ctd32 == NULL) ? 1 : Q_MAXSTRLEN(ctd32->mu, 10); + is32bit = 1; + ctd64tree = NULL; + ctd64 = NULL; + break; + case VSD_DTYPE_TDGSTCLUST64: + smplcnt = CONSTVSD(tdgstclust64, tdgst)->smplcnt; + compcnt = CONSTVSD(tdgstclust64, tdgst)->compcnt; + maxctds = ARB_MAXNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); + curctds = ARB_CURNODES(&CONSTVSD(tdgstclust64, tdgst)->ctdtree); + ctd64tree = &CONSTVSD(tdgstclust64, tdgst)->ctdtree; + ctd64 = (objdump ? ARB_CNODE(ctd64tree, 0) : + ARB_CMIN(ctdth64, ctd64tree)); + qmaxstrlen = (ctd64 == NULL) ? 1 : Q_MAXSTRLEN(ctd64->mu, 10); + is32bit = 0; + ctd32tree = NULL; + ctd32 = NULL; + break; + default: + return; + } + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "smplcnt=%ju, compcnt=%ju, maxctds=%hu, nctds=%hu"; + break; + case SB_STRFMT_JSON: + default: + fmtstr = + "\"smplcnt\":%ju,\"compcnt\":%ju,\"maxctds\":%hu," + "\"nctds\":%hu,\"ctds\":["; + break; + } + sbuf_printf(buf, fmtstr, (uintmax_t)smplcnt, (uintmax_t)compcnt, + maxctds, curctds); + + while ((is32bit ? NULL != ctd32 : NULL != ctd64)) { + char qstr[qmaxstrlen]; + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "\n\t\t\t\t"; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "{"; + break; + } + sbuf_cat(buf, fmtstr); + + if (objdump) { + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "ctd[%hu]."; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "\"ctd\":%hu,"; + break; + } + sbuf_printf(buf, fmtstr, is32bit ? + ARB_SELFIDX(ctd32tree, ctd32) : + ARB_SELFIDX(ctd64tree, ctd64)); + } + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "{mu="; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "\"mu\":"; + break; + } + sbuf_cat(buf, fmtstr); + Q_TOSTR((is32bit ? ctd32->mu : ctd64->mu), -1, 10, qstr, + sizeof(qstr)); + sbuf_cat(buf, qstr); + + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; + break; + case SB_STRFMT_JSON: + default: + fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; + break; + } + sbuf_printf(buf, fmtstr, + is32bit ? ctd32->cnt : (uintmax_t)ctd64->cnt); + + if (is32bit) + ctd32 = (objdump ? ARB_CNODE(ctd32tree, + ARB_SELFIDX(ctd32tree, ctd32) + 1) : + ARB_CNEXT(ctdth32, ctd32tree, ctd32)); + else + ctd64 = (objdump ? ARB_CNODE(ctd64tree, + ARB_SELFIDX(ctd64tree, ctd64) + 1) : + ARB_CNEXT(ctdth64, ctd64tree, ctd64)); + + if (fmt == SB_STRFMT_JSON && + (is32bit ? NULL != ctd32 : NULL != ctd64)) + sbuf_putc(buf, ','); + } + if (fmt == SB_STRFMT_JSON) + sbuf_cat(buf, "]"); +} + +static inline void +stats_voistatdata_hist_tostr(enum vsd_dtype voi_dtype, + const struct voistatdata_hist *hist, enum vsd_dtype hist_dtype, + size_t hist_dsz, enum sb_str_fmt fmt, struct sbuf *buf, int objdump) +{ + const struct voistatdata_numeric *bkt_lb, *bkt_ub; + const char *fmtstr; + int is32bit; + uint16_t i, nbkts; + + switch (hist_dtype) { + case VSD_DTYPE_CRHIST32: + nbkts = HIST_VSDSZ2NBKTS(crhist32, hist_dsz); + is32bit = 1; + break; + case VSD_DTYPE_DRHIST32: + nbkts = HIST_VSDSZ2NBKTS(drhist32, hist_dsz); + is32bit = 1; + break; + case VSD_DTYPE_DVHIST32: + nbkts = HIST_VSDSZ2NBKTS(dvhist32, hist_dsz); + is32bit = 1; + break; + case VSD_DTYPE_CRHIST64: + nbkts = HIST_VSDSZ2NBKTS(crhist64, hist_dsz); + is32bit = 0; + break; + case VSD_DTYPE_DRHIST64: + nbkts = HIST_VSDSZ2NBKTS(drhist64, hist_dsz); + is32bit = 0; + break; + case VSD_DTYPE_DVHIST64: + nbkts = HIST_VSDSZ2NBKTS(dvhist64, hist_dsz); + is32bit = 0; + break; + default: + return; + } + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "nbkts=%hu, "; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "\"nbkts\":%hu,"; + break; + } + sbuf_printf(buf, fmtstr, nbkts); + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = (is32bit ? "oob=%u" : "oob=%ju"); + break; + case SB_STRFMT_JSON: + default: + fmtstr = (is32bit ? "\"oob\":%u,\"bkts\":[" : + "\"oob\":%ju,\"bkts\":["); + break; + } + sbuf_printf(buf, fmtstr, is32bit ? VSD_CONSTHIST_FIELDVAL(hist, + hist_dtype, oob) : (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, + hist_dtype, oob)); + + for (i = 0; i < nbkts; i++) { + switch (hist_dtype) { + case VSD_DTYPE_CRHIST32: + case VSD_DTYPE_CRHIST64: + bkt_lb = VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype, + bkts[i].lb); + if (i < nbkts - 1) + bkt_ub = VSD_CONSTCRHIST_FIELDPTR(hist, + hist_dtype, bkts[i + 1].lb); + else + bkt_ub = &numeric_limits[LIM_MAX][voi_dtype]; + break; + case VSD_DTYPE_DRHIST32: + case VSD_DTYPE_DRHIST64: + bkt_lb = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, + bkts[i].lb); + bkt_ub = VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, + bkts[i].ub); + break; + case VSD_DTYPE_DVHIST32: + case VSD_DTYPE_DVHIST64: + bkt_lb = bkt_ub = VSD_CONSTDVHIST_FIELDPTR(hist, + hist_dtype, bkts[i].val); + break; + default: + break; + } + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "\n\t\t\t\t"; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "{"; + break; + } + sbuf_cat(buf, fmtstr); + + if (objdump) { + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "bkt[%hu]."; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "\"bkt\":%hu,"; + break; + } + sbuf_printf(buf, fmtstr, i); + } + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "{lb="; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "\"lb\":"; + break; + } + sbuf_cat(buf, fmtstr); + stats_voistatdata_tostr((const struct voistatdata *)bkt_lb, + voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), + fmt, buf, objdump); + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = ",ub="; + break; + case SB_STRFMT_JSON: + default: + fmtstr = ",\"ub\":"; + break; + } + sbuf_cat(buf, fmtstr); + stats_voistatdata_tostr((const struct voistatdata *)bkt_ub, + voi_dtype, voi_dtype, sizeof(struct voistatdata_numeric), + fmt, buf, objdump); + + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = is32bit ? ",cnt=%u}" : ",cnt=%ju}"; + break; + case SB_STRFMT_JSON: + default: + fmtstr = is32bit ? ",\"cnt\":%u}" : ",\"cnt\":%ju}"; + break; + } + sbuf_printf(buf, fmtstr, is32bit ? + VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, bkts[i].cnt) : + (uintmax_t)VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, + bkts[i].cnt)); + + if (fmt == SB_STRFMT_JSON && i < nbkts - 1) + sbuf_putc(buf, ','); + } + if (fmt == SB_STRFMT_JSON) + sbuf_cat(buf, "]"); +} + +int +stats_voistatdata_tostr(const struct voistatdata *vsd, enum vsd_dtype voi_dtype, + enum vsd_dtype vsd_dtype, size_t vsd_sz, enum sb_str_fmt fmt, + struct sbuf *buf, int objdump) +{ + const char *fmtstr; + + if (vsd == NULL || buf == NULL || voi_dtype >= VSD_NUM_DTYPES || + vsd_dtype >= VSD_NUM_DTYPES || fmt >= SB_STRFMT_NUM_FMTS) + return (EINVAL); + + switch (vsd_dtype) { + case VSD_DTYPE_VOISTATE: + switch (fmt) { + case SB_STRFMT_FREEFORM: + fmtstr = "prev="; + break; + case SB_STRFMT_JSON: + default: + fmtstr = "\"prev\":"; + break; + } + sbuf_cat(buf, fmtstr); + /* + * Render prev by passing it as *vsd and voi_dtype as vsd_dtype. + */ + stats_voistatdata_tostr( + (const struct voistatdata *)&CONSTVSD(voistate, vsd)->prev, + voi_dtype, voi_dtype, vsd_sz, fmt, buf, objdump); + break; + case VSD_DTYPE_INT_S32: + sbuf_printf(buf, "%d", vsd->int32.s32); + break; + case VSD_DTYPE_INT_U32: + sbuf_printf(buf, "%u", vsd->int32.u32); + break; + case VSD_DTYPE_INT_S64: + sbuf_printf(buf, "%jd", (intmax_t)vsd->int64.s64); + break; + case VSD_DTYPE_INT_U64: + sbuf_printf(buf, "%ju", (uintmax_t)vsd->int64.u64); + break; + case VSD_DTYPE_INT_SLONG: + sbuf_printf(buf, "%ld", vsd->intlong.slong); + break; + case VSD_DTYPE_INT_ULONG: + sbuf_printf(buf, "%lu", vsd->intlong.ulong); + break; + case VSD_DTYPE_Q_S32: + { + char qstr[Q_MAXSTRLEN(vsd->q32.sq32, 10)]; + Q_TOSTR((s32q_t)vsd->q32.sq32, -1, 10, qstr, sizeof(qstr)); + sbuf_cat(buf, qstr); + } + break; + case VSD_DTYPE_Q_U32: + { + char qstr[Q_MAXSTRLEN(vsd->q32.uq32, 10)]; + Q_TOSTR((u32q_t)vsd->q32.uq32, -1, 10, qstr, sizeof(qstr)); + sbuf_cat(buf, qstr); + } + break; + case VSD_DTYPE_Q_S64: + { + char qstr[Q_MAXSTRLEN(vsd->q64.sq64, 10)]; + Q_TOSTR((s64q_t)vsd->q64.sq64, -1, 10, qstr, sizeof(qstr)); + sbuf_cat(buf, qstr); + } + break; + case VSD_DTYPE_Q_U64: + { + char qstr[Q_MAXSTRLEN(vsd->q64.uq64, 10)]; + Q_TOSTR((u64q_t)vsd->q64.uq64, -1, 10, qstr, sizeof(qstr)); + sbuf_cat(buf, qstr); + } + break; + case VSD_DTYPE_CRHIST32: + case VSD_DTYPE_DRHIST32: + case VSD_DTYPE_DVHIST32: + case VSD_DTYPE_CRHIST64: + case VSD_DTYPE_DRHIST64: + case VSD_DTYPE_DVHIST64: + stats_voistatdata_hist_tostr(voi_dtype, CONSTVSD(hist, vsd), + vsd_dtype, vsd_sz, fmt, buf, objdump); + break; + case VSD_DTYPE_TDGSTCLUST32: + case VSD_DTYPE_TDGSTCLUST64: + stats_voistatdata_tdgst_tostr(voi_dtype, + CONSTVSD(tdgst, vsd), vsd_dtype, vsd_sz, fmt, buf, + objdump); + break; + default: + break; + } + + return (sbuf_error(buf)); +} + +static void +stats_v1_itercb_tostr_freeform(struct statsblobv1 *sb, struct voi *v, + struct voistat *vs, struct sb_iter_ctx *ctx) +{ + struct sb_tostrcb_ctx *sctx; + struct metablob *tpl_mb; + struct sbuf *buf; + void *vsd; + uint8_t dump; + + sctx = ctx->usrctx; + buf = sctx->buf; + tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; + dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); + + if (ctx->flags & SB_IT_FIRST_CB) { + sbuf_printf(buf, "struct statsblobv1@%p", sb); + if (dump) { + sbuf_printf(buf, ", abi=%hhu, endian=%hhu, maxsz=%hu, " + "cursz=%hu, created=%jd, lastrst=%jd, flags=0x%04hx, " + "stats_off=%hu, statsdata_off=%hu", + sb->abi, sb->endian, sb->maxsz, sb->cursz, + sb->created, sb->lastrst, sb->flags, sb->stats_off, + sb->statsdata_off); + } + sbuf_printf(buf, ", tplhash=%u", sb->tplhash); + } + + if (ctx->flags & SB_IT_FIRST_VOISTAT) { + sbuf_printf(buf, "\n\tvois[%hd]: id=%hd", ctx->vslot, v->id); + if (v->id < 0) + return; + sbuf_printf(buf, ", name=\"%s\"", (tpl_mb == NULL) ? "" : + tpl_mb->voi_meta[v->id].name); + if (dump) + sbuf_printf(buf, ", flags=0x%04hx, dtype=%s, " + "voistatmaxid=%hhd, stats_off=%hu", v->flags, + vsd_dtype2name[v->dtype], v->voistatmaxid, v->stats_off); + } + + if (!dump && vs->stype <= 0) + return; + + sbuf_printf(buf, "\n\t\tvois[%hd]stat[%hhd]: stype=", v->id, ctx->vsslot); + if (vs->stype < 0) { + sbuf_printf(buf, "%hhd", vs->stype); + return; + } else + sbuf_printf(buf, "%s, errs=%hu", vs_stype2name[vs->stype], + vs->errs); + vsd = BLOB_OFFSET(sb, vs->data_off); + if (dump) + sbuf_printf(buf, ", flags=0x%04x, dtype=%s, dsz=%hu, " + "data_off=%hu", vs->flags, vsd_dtype2name[vs->dtype], + vs->dsz, vs->data_off); + + sbuf_printf(buf, "\n\t\t\tvoistatdata: "); + stats_voistatdata_tostr(BLOB_OFFSET(sb, vs->data_off), v->dtype, + vs->dtype, vs->dsz, sctx->fmt, buf, dump); +} + +static void +stats_v1_itercb_tostr_json(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, + struct sb_iter_ctx *ctx) +{ + struct sb_tostrcb_ctx *sctx; + struct metablob *tpl_mb; + struct sbuf *buf; + const char *fmtstr; + void *vsd; + uint8_t dump; + + sctx = ctx->usrctx; + buf = sctx->buf; + tpl_mb = sctx->tpl ? sctx->tpl->mb : NULL; + dump = ((sctx->flags & SB_TOSTR_OBJDUMP) != 0); + + if (ctx->flags & SB_IT_FIRST_CB) { + sbuf_putc(buf, '{'); + if (dump) { + sbuf_printf(buf, "\"abi\":%hhu,\"endian\":%hhu," + "\"maxsz\":%hu,\"cursz\":%hu,\"created\":%jd," + "\"lastrst\":%jd,\"flags\":%hu,\"stats_off\":%hu," + "\"statsdata_off\":%hu,", sb->abi, + sb->endian, sb->maxsz, sb->cursz, sb->created, + sb->lastrst, sb->flags, sb->stats_off, + sb->statsdata_off); + } + + if (tpl_mb == NULL) + fmtstr = "\"tplname\":%s,\"tplhash\":%u,\"vois\":{"; + else + fmtstr = "\"tplname\":\"%s\",\"tplhash\":%u,\"vois\":{"; + + sbuf_printf(buf, fmtstr, tpl_mb ? tpl_mb->tplname : "null", + sb->tplhash); + } + + if (ctx->flags & SB_IT_FIRST_VOISTAT) { + if (dump) { + sbuf_printf(buf, "\"[%d]\":{\"id\":%d", ctx->vslot, + v->id); + if (v->id < 0) { + sbuf_printf(buf, "},"); + return; + } + + if (tpl_mb == NULL) + fmtstr = ",\"name\":%s,\"flags\":%hu," + "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," + "\"stats_off\":%hu,"; + else + fmtstr = ",\"name\":\"%s\",\"flags\":%hu," + "\"dtype\":\"%s\",\"voistatmaxid\":%hhd," + "\"stats_off\":%hu,"; + + sbuf_printf(buf, fmtstr, tpl_mb ? + tpl_mb->voi_meta[v->id].name : "null", v->flags, + vsd_dtype2name[v->dtype], v->voistatmaxid, + v->stats_off); + } else { + if (tpl_mb == NULL) { + sbuf_printf(buf, "\"[%hd]\":{", v->id); + } else { + sbuf_printf(buf, "\"%s\":{", + tpl_mb->voi_meta[v->id].name); + } + } + sbuf_cat(buf, "\"stats\":{"); + } + + vsd = BLOB_OFFSET(sb, vs->data_off); + if (dump) { + sbuf_printf(buf, "\"[%hhd]\":", ctx->vsslot); + if (vs->stype < 0) { + sbuf_printf(buf, "{\"stype\":-1},"); + return; + } + sbuf_printf(buf, "{\"stype\":\"%s\",\"errs\":%hu,\"flags\":%hu," + "\"dtype\":\"%s\",\"data_off\":%hu,\"voistatdata\":{", + vs_stype2name[vs->stype], vs->errs, vs->flags, + vsd_dtype2name[vs->dtype], vs->data_off); + } else if (vs->stype > 0) { + if (tpl_mb == NULL) + sbuf_printf(buf, "\"[%hhd]\":", vs->stype); + else + sbuf_printf(buf, "\"%s\":", vs_stype2name[vs->stype]); + } else + return; + + if ((vs->flags & VS_VSDVALID) || dump) { + if (!dump) + sbuf_printf(buf, "{\"errs\":%hu,", vs->errs); + /* Simple non-compound VSD types need a key. */ + if (!vsd_compoundtype[vs->dtype]) + sbuf_cat(buf, "\"val\":"); + stats_voistatdata_tostr(vsd, v->dtype, vs->dtype, vs->dsz, + sctx->fmt, buf, dump); + sbuf_cat(buf, dump ? "}}" : "}"); + } else + sbuf_cat(buf, dump ? "null}" : "null"); + + if (ctx->flags & SB_IT_LAST_VOISTAT) + sbuf_cat(buf, "}}"); + + if (ctx->flags & SB_IT_LAST_CB) + sbuf_cat(buf, "}}"); + else + sbuf_putc(buf, ','); +} + +static int +stats_v1_itercb_tostr(struct statsblobv1 *sb, struct voi *v, struct voistat *vs, + struct sb_iter_ctx *ctx) +{ + struct sb_tostrcb_ctx *sctx; + + sctx = ctx->usrctx; + + switch (sctx->fmt) { + case SB_STRFMT_FREEFORM: + stats_v1_itercb_tostr_freeform(sb, v, vs, ctx); + break; + case SB_STRFMT_JSON: + stats_v1_itercb_tostr_json(sb, v, vs, ctx); + break; + default: + break; + } + + return (sbuf_error(sctx->buf)); +} + +int +stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf, + enum sb_str_fmt fmt, uint32_t flags) +{ + struct sb_tostrcb_ctx sctx; + uint32_t iflags; + + if (sb == NULL || sb->abi != STATS_ABI_V1 || buf == NULL || + fmt >= SB_STRFMT_NUM_FMTS) + return (EINVAL); + + sctx.buf = buf; + sctx.fmt = fmt; + sctx.flags = flags; + + if (flags & SB_TOSTR_META) { + if (stats_tpl_fetch(stats_tpl_fetch_allocid(NULL, sb->tplhash), + &sctx.tpl)) + return (EINVAL); + } else + sctx.tpl = NULL; + + iflags = 0; + if (flags & SB_TOSTR_OBJDUMP) + iflags |= (SB_IT_NULLVOI | SB_IT_NULLVOISTAT); + stats_v1_blob_iter(sb, stats_v1_itercb_tostr, &sctx, iflags); + + return (sbuf_error(buf)); +} + +static int +stats_v1_itercb_visit(struct statsblobv1 *sb, struct voi *v, + struct voistat *vs, struct sb_iter_ctx *ctx) +{ + struct sb_visitcb_ctx *vctx; + struct sb_visit sbv; + + vctx = ctx->usrctx; + + sbv.tplhash = sb->tplhash; + sbv.voi_id = v->id; + sbv.voi_dtype = v->dtype; + sbv.vs_stype = vs->stype; + sbv.vs_dtype = vs->dtype; + sbv.vs_dsz = vs->dsz; + sbv.vs_data = BLOB_OFFSET(sb, vs->data_off); + sbv.vs_errs = vs->errs; + sbv.flags = ctx->flags & (SB_IT_FIRST_CB | SB_IT_LAST_CB | + SB_IT_FIRST_VOI | SB_IT_LAST_VOI | SB_IT_FIRST_VOISTAT | + SB_IT_LAST_VOISTAT); + + return (vctx->cb(&sbv, vctx->usrctx)); +} + +int +stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func, + void *usrctx) +{ + struct sb_visitcb_ctx vctx; + + if (sb == NULL || sb->abi != STATS_ABI_V1 || func == NULL) + return (EINVAL); + + vctx.cb = func; + vctx.usrctx = usrctx; + + stats_v1_blob_iter(sb, stats_v1_itercb_visit, &vctx, 0); + + return (0); +} + +static int +stats_v1_icb_reset_voistat(struct statsblobv1 *sb, struct voi *v __unused, + struct voistat *vs, struct sb_iter_ctx *ctx __unused) +{ + void *vsd; + + if (vs->stype == VS_STYPE_VOISTATE) + return (0); + + vsd = BLOB_OFFSET(sb, vs->data_off); + + /* Perform the stat type's default reset action. */ + switch (vs->stype) { + case VS_STYPE_SUM: + switch (vs->dtype) { + case VSD_DTYPE_Q_S32: + Q_SIFVAL(VSD(q32, vsd)->sq32, 0); + break; + case VSD_DTYPE_Q_U32: + Q_SIFVAL(VSD(q32, vsd)->uq32, 0); + break; + case VSD_DTYPE_Q_S64: + Q_SIFVAL(VSD(q64, vsd)->sq64, 0); + break; + case VSD_DTYPE_Q_U64: + Q_SIFVAL(VSD(q64, vsd)->uq64, 0); + break; + default: + bzero(vsd, vs->dsz); + break; + } + break; + case VS_STYPE_MAX: + switch (vs->dtype) { + case VSD_DTYPE_Q_S32: + Q_SIFVAL(VSD(q32, vsd)->sq32, + Q_IFMINVAL(VSD(q32, vsd)->sq32)); + break; + case VSD_DTYPE_Q_U32: + Q_SIFVAL(VSD(q32, vsd)->uq32, + Q_IFMINVAL(VSD(q32, vsd)->uq32)); + break; + case VSD_DTYPE_Q_S64: + Q_SIFVAL(VSD(q64, vsd)->sq64, + Q_IFMINVAL(VSD(q64, vsd)->sq64)); + break; + case VSD_DTYPE_Q_U64: + Q_SIFVAL(VSD(q64, vsd)->uq64, + Q_IFMINVAL(VSD(q64, vsd)->uq64)); + break; + default: + memcpy(vsd, &numeric_limits[LIM_MIN][vs->dtype], + vs->dsz); + break; + } + break; + case VS_STYPE_MIN: + switch (vs->dtype) { + case VSD_DTYPE_Q_S32: + Q_SIFVAL(VSD(q32, vsd)->sq32, + Q_IFMAXVAL(VSD(q32, vsd)->sq32)); + break; + case VSD_DTYPE_Q_U32: + Q_SIFVAL(VSD(q32, vsd)->uq32, + Q_IFMAXVAL(VSD(q32, vsd)->uq32)); + break; + case VSD_DTYPE_Q_S64: + Q_SIFVAL(VSD(q64, vsd)->sq64, + Q_IFMAXVAL(VSD(q64, vsd)->sq64)); + break; + case VSD_DTYPE_Q_U64: + Q_SIFVAL(VSD(q64, vsd)->uq64, + Q_IFMAXVAL(VSD(q64, vsd)->uq64)); + break; + default: + memcpy(vsd, &numeric_limits[LIM_MAX][vs->dtype], + vs->dsz); + break; + } + break; + case VS_STYPE_HIST: + { + /* Reset bucket counts. */ + struct voistatdata_hist *hist; + int i, is32bit; + uint16_t nbkts; + + hist = VSD(hist, vsd); + switch (vs->dtype) { + case VSD_DTYPE_CRHIST32: + nbkts = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); + is32bit = 1; + break; + case VSD_DTYPE_DRHIST32: + nbkts = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); + is32bit = 1; + break; + case VSD_DTYPE_DVHIST32: + nbkts = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); + is32bit = 1; + break; + case VSD_DTYPE_CRHIST64: + nbkts = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); + is32bit = 0; + break; + case VSD_DTYPE_DRHIST64: + nbkts = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); + is32bit = 0; + break; + case VSD_DTYPE_DVHIST64: + nbkts = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); + is32bit = 0; + break; + default: + return (0); + } + + bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, oob), + is32bit ? sizeof(uint32_t) : sizeof(uint64_t)); + for (i = nbkts - 1; i >= 0; i--) { + bzero(VSD_HIST_FIELDPTR(hist, vs->dtype, + bkts[i].cnt), is32bit ? sizeof(uint32_t) : + sizeof(uint64_t)); + } + break; + } + case VS_STYPE_TDGST: + { + /* Reset sample count centroids array/tree. */ + struct voistatdata_tdgst *tdgst; + struct ctdth32 *ctd32tree; + struct ctdth64 *ctd64tree; + struct voistatdata_tdgstctd32 *ctd32; + struct voistatdata_tdgstctd64 *ctd64; + + tdgst = VSD(tdgst, vsd); + switch (vs->dtype) { + case VSD_DTYPE_TDGSTCLUST32: + VSD(tdgstclust32, tdgst)->smplcnt = 0; + VSD(tdgstclust32, tdgst)->compcnt = 0; + ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; + ARB_INIT(ctd32, ctdlnk, ctd32tree, + ARB_MAXNODES(ctd32tree)) { + ctd32->cnt = 0; + Q_SIFVAL(ctd32->mu, 0); + } +#ifdef DIAGNOSTIC + RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); +#endif + break; + case VSD_DTYPE_TDGSTCLUST64: + VSD(tdgstclust64, tdgst)->smplcnt = 0; + VSD(tdgstclust64, tdgst)->compcnt = 0; + ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; + ARB_INIT(ctd64, ctdlnk, ctd64tree, + ARB_MAXNODES(ctd64tree)) { + ctd64->cnt = 0; + Q_SIFVAL(ctd64->mu, 0); + } +#ifdef DIAGNOSTIC + RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); +#endif + break; + default: + return (0); + } + break; + } + default: + KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); + break; + } + + vs->errs = 0; + vs->flags &= ~VS_VSDVALID; + + return (0); +} + +int +stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz, + struct statsblobv1 *src, uint32_t flags) +{ + int error; + + if (src != NULL && src->abi == STATS_ABI_V1) { + error = stats_v1_blob_clone(dst, dstmaxsz, src, flags); + if (!error) { + if (flags & SB_CLONE_RSTSRC) { + stats_v1_blob_iter(src, + stats_v1_icb_reset_voistat, NULL, 0); + src->lastrst = stats_sbinuptime(); + } + stats_v1_blob_finalise(*dst); + } + } else + error = EINVAL; + + return (error); +} + +static inline int +stats_v1_voi_update_max(enum vsd_dtype voi_dtype __unused, + struct voistatdata *voival, struct voistat *vs, void *vsd) +{ + int error; + + KASSERT(vs->dtype < VSD_NUM_DTYPES, + ("Unknown VSD dtype %d", vs->dtype)); + + error = 0; + + switch (vs->dtype) { + case VSD_DTYPE_INT_S32: + if (VSD(int32, vsd)->s32 < voival->int32.s32) { + VSD(int32, vsd)->s32 = voival->int32.s32; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_U32: + if (VSD(int32, vsd)->u32 < voival->int32.u32) { + VSD(int32, vsd)->u32 = voival->int32.u32; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_S64: + if (VSD(int64, vsd)->s64 < voival->int64.s64) { + VSD(int64, vsd)->s64 = voival->int64.s64; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_U64: + if (VSD(int64, vsd)->u64 < voival->int64.u64) { + VSD(int64, vsd)->u64 = voival->int64.u64; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_SLONG: + if (VSD(intlong, vsd)->slong < voival->intlong.slong) { + VSD(intlong, vsd)->slong = voival->intlong.slong; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_ULONG: + if (VSD(intlong, vsd)->ulong < voival->intlong.ulong) { + VSD(intlong, vsd)->ulong = voival->intlong.ulong; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_S32: + if (Q_QLTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && + (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, + voival->q32.sq32)))) { + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_U32: + if (Q_QLTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && + (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, + voival->q32.uq32)))) { + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_S64: + if (Q_QLTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && + (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, + voival->q64.sq64)))) { + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_U64: + if (Q_QLTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && + (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, + voival->q64.uq64)))) { + vs->flags |= VS_VSDVALID; + } + break; + default: + error = EINVAL; + break; + } + + return (error); +} + +static inline int +stats_v1_voi_update_min(enum vsd_dtype voi_dtype __unused, + struct voistatdata *voival, struct voistat *vs, void *vsd) +{ + int error; + + KASSERT(vs->dtype < VSD_NUM_DTYPES, + ("Unknown VSD dtype %d", vs->dtype)); + + error = 0; + + switch (vs->dtype) { + case VSD_DTYPE_INT_S32: + if (VSD(int32, vsd)->s32 > voival->int32.s32) { + VSD(int32, vsd)->s32 = voival->int32.s32; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_U32: + if (VSD(int32, vsd)->u32 > voival->int32.u32) { + VSD(int32, vsd)->u32 = voival->int32.u32; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_S64: + if (VSD(int64, vsd)->s64 > voival->int64.s64) { + VSD(int64, vsd)->s64 = voival->int64.s64; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_U64: + if (VSD(int64, vsd)->u64 > voival->int64.u64) { + VSD(int64, vsd)->u64 = voival->int64.u64; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_SLONG: + if (VSD(intlong, vsd)->slong > voival->intlong.slong) { + VSD(intlong, vsd)->slong = voival->intlong.slong; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_INT_ULONG: + if (VSD(intlong, vsd)->ulong > voival->intlong.ulong) { + VSD(intlong, vsd)->ulong = voival->intlong.ulong; + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_S32: + if (Q_QGTQ(VSD(q32, vsd)->sq32, voival->q32.sq32) && + (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->sq32, + voival->q32.sq32)))) { + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_U32: + if (Q_QGTQ(VSD(q32, vsd)->uq32, voival->q32.uq32) && + (0 == (error = Q_QCPYVALQ(&VSD(q32, vsd)->uq32, + voival->q32.uq32)))) { + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_S64: + if (Q_QGTQ(VSD(q64, vsd)->sq64, voival->q64.sq64) && + (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->sq64, + voival->q64.sq64)))) { + vs->flags |= VS_VSDVALID; + } + break; + case VSD_DTYPE_Q_U64: + if (Q_QGTQ(VSD(q64, vsd)->uq64, voival->q64.uq64) && + (0 == (error = Q_QCPYVALQ(&VSD(q64, vsd)->uq64, + voival->q64.uq64)))) { + vs->flags |= VS_VSDVALID; + } + break; + default: + error = EINVAL; + break; + } + + return (error); +} + +static inline int +stats_v1_voi_update_sum(enum vsd_dtype voi_dtype __unused, + struct voistatdata *voival, struct voistat *vs, void *vsd) +{ + int error; + + KASSERT(vs->dtype < VSD_NUM_DTYPES, + ("Unknown VSD dtype %d", vs->dtype)); + + error = 0; + + switch (vs->dtype) { + case VSD_DTYPE_INT_S32: + VSD(int32, vsd)->s32 += voival->int32.s32; + break; + case VSD_DTYPE_INT_U32: + VSD(int32, vsd)->u32 += voival->int32.u32; + break; + case VSD_DTYPE_INT_S64: + VSD(int64, vsd)->s64 += voival->int64.s64; + break; + case VSD_DTYPE_INT_U64: + VSD(int64, vsd)->u64 += voival->int64.u64; + break; + case VSD_DTYPE_INT_SLONG: + VSD(intlong, vsd)->slong += voival->intlong.slong; + break; + case VSD_DTYPE_INT_ULONG: + VSD(intlong, vsd)->ulong += voival->intlong.ulong; + break; + case VSD_DTYPE_Q_S32: + error = Q_QADDQ(&VSD(q32, vsd)->sq32, voival->q32.sq32); + break; + case VSD_DTYPE_Q_U32: + error = Q_QADDQ(&VSD(q32, vsd)->uq32, voival->q32.uq32); + break; + case VSD_DTYPE_Q_S64: + error = Q_QADDQ(&VSD(q64, vsd)->sq64, voival->q64.sq64); + break; + case VSD_DTYPE_Q_U64: + error = Q_QADDQ(&VSD(q64, vsd)->uq64, voival->q64.uq64); + break; + default: + error = EINVAL; + break; + } + + if (!error) + vs->flags |= VS_VSDVALID; + + return (error); +} + +static inline int +stats_v1_voi_update_hist(enum vsd_dtype voi_dtype, struct voistatdata *voival, + struct voistat *vs, struct voistatdata_hist *hist) +{ + struct voistatdata_numeric *bkt_lb, *bkt_ub; + uint64_t *oob64, *cnt64; + uint32_t *oob32, *cnt32; + int error, i, found, is32bit, has_ub, eq_only; + + error = 0; + + switch (vs->dtype) { + case VSD_DTYPE_CRHIST32: + i = HIST_VSDSZ2NBKTS(crhist32, vs->dsz); + is32bit = 1; + has_ub = eq_only = 0; + oob32 = &VSD(crhist32, hist)->oob; + break; + case VSD_DTYPE_DRHIST32: + i = HIST_VSDSZ2NBKTS(drhist32, vs->dsz); + is32bit = has_ub = 1; + eq_only = 0; + oob32 = &VSD(drhist32, hist)->oob; + break; + case VSD_DTYPE_DVHIST32: + i = HIST_VSDSZ2NBKTS(dvhist32, vs->dsz); + is32bit = eq_only = 1; + has_ub = 0; + oob32 = &VSD(dvhist32, hist)->oob; + break; + case VSD_DTYPE_CRHIST64: + i = HIST_VSDSZ2NBKTS(crhist64, vs->dsz); + is32bit = has_ub = eq_only = 0; + oob64 = &VSD(crhist64, hist)->oob; + break; + case VSD_DTYPE_DRHIST64: + i = HIST_VSDSZ2NBKTS(drhist64, vs->dsz); + is32bit = eq_only = 0; + has_ub = 1; + oob64 = &VSD(drhist64, hist)->oob; + break; + case VSD_DTYPE_DVHIST64: + i = HIST_VSDSZ2NBKTS(dvhist64, vs->dsz); + is32bit = has_ub = 0; + eq_only = 1; + oob64 = &VSD(dvhist64, hist)->oob; + break; + default: + return (EINVAL); + } + i--; /* Adjust for 0-based array index. */ + + /* XXXLAS: Should probably use a better bucket search algorithm. ARB? */ + for (found = 0; i >= 0 && !found; i--) { + switch (vs->dtype) { + case VSD_DTYPE_CRHIST32: + bkt_lb = &VSD(crhist32, hist)->bkts[i].lb; + cnt32 = &VSD(crhist32, hist)->bkts[i].cnt; + break; + case VSD_DTYPE_DRHIST32: + bkt_lb = &VSD(drhist32, hist)->bkts[i].lb; + bkt_ub = &VSD(drhist32, hist)->bkts[i].ub; + cnt32 = &VSD(drhist32, hist)->bkts[i].cnt; + break; + case VSD_DTYPE_DVHIST32: + bkt_lb = &VSD(dvhist32, hist)->bkts[i].val; + cnt32 = &VSD(dvhist32, hist)->bkts[i].cnt; + break; + case VSD_DTYPE_CRHIST64: + bkt_lb = &VSD(crhist64, hist)->bkts[i].lb; + cnt64 = &VSD(crhist64, hist)->bkts[i].cnt; + break; + case VSD_DTYPE_DRHIST64: + bkt_lb = &VSD(drhist64, hist)->bkts[i].lb; + bkt_ub = &VSD(drhist64, hist)->bkts[i].ub; + cnt64 = &VSD(drhist64, hist)->bkts[i].cnt; + break; + case VSD_DTYPE_DVHIST64: + bkt_lb = &VSD(dvhist64, hist)->bkts[i].val; + cnt64 = &VSD(dvhist64, hist)->bkts[i].cnt; + break; + default: + return (EINVAL); + } + + switch (voi_dtype) { + case VSD_DTYPE_INT_S32: + if (voival->int32.s32 >= bkt_lb->int32.s32) { + if ((eq_only && voival->int32.s32 == + bkt_lb->int32.s32) || + (!eq_only && (!has_ub || + voival->int32.s32 < bkt_ub->int32.s32))) + found = 1; + } + break; + case VSD_DTYPE_INT_U32: + if (voival->int32.u32 >= bkt_lb->int32.u32) { + if ((eq_only && voival->int32.u32 == + bkt_lb->int32.u32) || + (!eq_only && (!has_ub || + voival->int32.u32 < bkt_ub->int32.u32))) + found = 1; + } + break; + case VSD_DTYPE_INT_S64: + if (voival->int64.s64 >= bkt_lb->int64.s64) + if ((eq_only && voival->int64.s64 == + bkt_lb->int64.s64) || + (!eq_only && (!has_ub || + voival->int64.s64 < bkt_ub->int64.s64))) + found = 1; + break; + case VSD_DTYPE_INT_U64: + if (voival->int64.u64 >= bkt_lb->int64.u64) + if ((eq_only && voival->int64.u64 == + bkt_lb->int64.u64) || + (!eq_only && (!has_ub || + voival->int64.u64 < bkt_ub->int64.u64))) + found = 1; + break; + case VSD_DTYPE_INT_SLONG: + if (voival->intlong.slong >= bkt_lb->intlong.slong) + if ((eq_only && voival->intlong.slong == + bkt_lb->intlong.slong) || + (!eq_only && (!has_ub || + voival->intlong.slong < + bkt_ub->intlong.slong))) + found = 1; + break; + case VSD_DTYPE_INT_ULONG: + if (voival->intlong.ulong >= bkt_lb->intlong.ulong) + if ((eq_only && voival->intlong.ulong == + bkt_lb->intlong.ulong) || + (!eq_only && (!has_ub || + voival->intlong.ulong < + bkt_ub->intlong.ulong))) + found = 1; + break; + case VSD_DTYPE_Q_S32: + if (Q_QGEQ(voival->q32.sq32, bkt_lb->q32.sq32)) + if ((eq_only && Q_QEQ(voival->q32.sq32, + bkt_lb->q32.sq32)) || + (!eq_only && (!has_ub || + Q_QLTQ(voival->q32.sq32, + bkt_ub->q32.sq32)))) + found = 1; + break; + case VSD_DTYPE_Q_U32: + if (Q_QGEQ(voival->q32.uq32, bkt_lb->q32.uq32)) + if ((eq_only && Q_QEQ(voival->q32.uq32, + bkt_lb->q32.uq32)) || + (!eq_only && (!has_ub || + Q_QLTQ(voival->q32.uq32, + bkt_ub->q32.uq32)))) + found = 1; + break; + case VSD_DTYPE_Q_S64: + if (Q_QGEQ(voival->q64.sq64, bkt_lb->q64.sq64)) + if ((eq_only && Q_QEQ(voival->q64.sq64, + bkt_lb->q64.sq64)) || + (!eq_only && (!has_ub || + Q_QLTQ(voival->q64.sq64, + bkt_ub->q64.sq64)))) + found = 1; + break; + case VSD_DTYPE_Q_U64: + if (Q_QGEQ(voival->q64.uq64, bkt_lb->q64.uq64)) + if ((eq_only && Q_QEQ(voival->q64.uq64, + bkt_lb->q64.uq64)) || + (!eq_only && (!has_ub || + Q_QLTQ(voival->q64.uq64, + bkt_ub->q64.uq64)))) + found = 1; + break; + default: + break; + } + } + + if (found) { + if (is32bit) + *cnt32 += 1; + else + *cnt64 += 1; + } else { + if (is32bit) + *oob32 += 1; + else + *oob64 += 1; + } + + vs->flags |= VS_VSDVALID; + return (error); +} + +static inline int +stats_v1_vsd_tdgst_compress(enum vsd_dtype vs_dtype, + struct voistatdata_tdgst *tdgst, int attempt) +{ + struct ctdth32 *ctd32tree; + struct ctdth64 *ctd64tree; + struct voistatdata_tdgstctd32 *ctd32; + struct voistatdata_tdgstctd64 *ctd64; + uint64_t ebits, idxmask; + uint32_t bitsperidx, nebits; + int error, idx, is32bit, maxctds, remctds, tmperr; + + error = 0; + + switch (vs_dtype) { + case VSD_DTYPE_TDGSTCLUST32: + ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; + if (!ARB_FULL(ctd32tree)) + return (0); + VSD(tdgstclust32, tdgst)->compcnt++; + maxctds = remctds = ARB_MAXNODES(ctd32tree); + /* + * Reset the ARB tree. Centroid values remain intact, and unlike + * ARB_INIT(), this does not initialise the free list. + */ + *ctd32tree = ARB_INITIALIZER(ctdth32, maxctds); + VSD(tdgstclust32, tdgst)->smplcnt = 0; + is32bit = 1; + ctd64tree = NULL; + ctd64 = NULL; +#ifdef DIAGNOSTIC + RB_INIT(&VSD(tdgstclust32, tdgst)->rbctdtree); +#endif + break; + case VSD_DTYPE_TDGSTCLUST64: + ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; + if (!ARB_FULL(ctd64tree)) + return (0); + VSD(tdgstclust64, tdgst)->compcnt++; + maxctds = remctds = ARB_MAXNODES(ctd64tree); + /* + * Reset the ARB tree. Centroid mu/cnt values remain intact, and + * unlike ARB_INIT(), this does not initialise the free list. + */ + *ctd64tree = ARB_INITIALIZER(ctdth64, maxctds); + VSD(tdgstclust64, tdgst)->smplcnt = 0; + is32bit = 0; + ctd32tree = NULL; + ctd32 = NULL; +#ifdef DIAGNOSTIC + RB_INIT(&VSD(tdgstclust64, tdgst)->rbctdtree); +#endif + break; + default: + return (EINVAL); + } + + /* + * Rebuild the t-digest ARB by randomly selecting centroids and + * re-inserting the mu/cnt of each as a value and corresponding weight. + */ + +#define bitsperrand 31 /* Per random(3). */ + ebits = 0; + nebits = 0; + bitsperidx = fls(maxctds); + KASSERT(bitsperidx <= sizeof(ebits) << 3, + ("%s: bitsperidx=%d, ebits=%d", + __func__, bitsperidx, (int)(sizeof(ebits) << 3))); + idxmask = (UINT64_C(1) << bitsperidx) - 1; + srandom(stats_sbinuptime()); + + /* Initialise the free list with randomised centroid indices. */ + for (; remctds > 0; remctds--) { + while (nebits < bitsperidx) { + ebits |= ((uint64_t)random()) << nebits; + nebits += bitsperrand; + if (nebits > (sizeof(ebits) << 3)) + nebits = sizeof(ebits) << 3; + } + idx = ebits & idxmask; + nebits -= bitsperidx; + ebits >>= bitsperidx; + + /* + * Select the next centroid to put on the ARB free list. We + * start with the centroid at our randomly selected array index, + * and work our way forwards until finding one (the latter + * aspect reduces re-insertion randomness, but is good enough). + */ + do { + if (idx >= maxctds) + idx %= maxctds; + + if (is32bit) + ctd32 = ARB_NODE(ctd32tree, idx); + else + ctd64 = ARB_NODE(ctd64tree, idx); + } while ((is32bit ? ARB_ISFREE(ctd32, ctdlnk) : + ARB_ISFREE(ctd64, ctdlnk)) && ++idx); + + /* Put the centroid on the ARB free list. */ + if (is32bit) + ARB_RETURNFREE(ctd32tree, ctd32, ctdlnk); + else + ARB_RETURNFREE(ctd64tree, ctd64, ctdlnk); + } + + /* + * The free list now contains the randomised indices of every centroid. + * Walk the free list from start to end, re-inserting each centroid's + * mu/cnt. The tdgst_add() call may or may not consume the free centroid + * we re-insert values from during each loop iteration, so we must latch + * the index of the next free list centroid before the re-insertion + * call. The previous loop above should have left the centroid pointer + * pointing to the element at the head of the free list. + */ + KASSERT((is32bit ? + ARB_FREEIDX(ctd32tree) == ARB_SELFIDX(ctd32tree, ctd32) : + ARB_FREEIDX(ctd64tree) == ARB_SELFIDX(ctd64tree, ctd64)), + ("%s: t-digest ARB@%p free list bug", __func__, + (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); + remctds = maxctds; + while ((is32bit ? ctd32 != NULL : ctd64 != NULL)) { + tmperr = 0; + if (is32bit) { + s64q_t x; + + idx = ARB_NEXTFREEIDX(ctd32, ctdlnk); + /* Cloning a s32q_t into a s64q_t should never fail. */ + tmperr = Q_QCLONEQ(&x, ctd32->mu); + tmperr = tmperr ? tmperr : stats_v1_vsd_tdgst_add( + vs_dtype, tdgst, x, ctd32->cnt, attempt); + ctd32 = ARB_NODE(ctd32tree, idx); + KASSERT(ctd32 == NULL || ARB_ISFREE(ctd32, ctdlnk), + ("%s: t-digest ARB@%p free list bug", __func__, + ctd32tree)); + } else { + idx = ARB_NEXTFREEIDX(ctd64, ctdlnk); + tmperr = stats_v1_vsd_tdgst_add(vs_dtype, tdgst, + ctd64->mu, ctd64->cnt, attempt); + ctd64 = ARB_NODE(ctd64tree, idx); + KASSERT(ctd64 == NULL || ARB_ISFREE(ctd64, ctdlnk), + ("%s: t-digest ARB@%p free list bug", __func__, + ctd64tree)); + } + /* + * This process should not produce errors, bugs notwithstanding. + * Just in case, latch any errors and attempt all re-insertions. + */ + error = tmperr ? tmperr : error; + remctds--; + } + + KASSERT(remctds == 0, ("%s: t-digest ARB@%p free list bug", __func__, + (is32bit ? (void *)ctd32tree : (void *)ctd64tree))); + + return (error); +} + +static inline int +stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst, + s64q_t x, uint64_t weight, int attempt) +{ +#ifdef DIAGNOSTIC + char qstr[Q_MAXSTRLEN(x, 10)]; +#endif + struct ctdth32 *ctd32tree; + struct ctdth64 *ctd64tree; + void *closest, *cur, *lb, *ub; + struct voistatdata_tdgstctd32 *ctd32; + struct voistatdata_tdgstctd64 *ctd64; + uint64_t cnt, smplcnt, sum, tmpsum; + s64q_t k, minz, q, z; + int error, is32bit, n; + + error = 0; + minz = Q_INI(&z, 0, Q_NFBITS(x)); + + switch (vs_dtype) { + case VSD_DTYPE_TDGSTCLUST32: + if ((UINT32_MAX - weight) < VSD(tdgstclust32, tdgst)->smplcnt) + error = EOVERFLOW; + smplcnt = VSD(tdgstclust32, tdgst)->smplcnt; + ctd32tree = &VSD(tdgstclust32, tdgst)->ctdtree; + is32bit = 1; + ctd64tree = NULL; + ctd64 = NULL; + break; + case VSD_DTYPE_TDGSTCLUST64: + if ((UINT64_MAX - weight) < VSD(tdgstclust64, tdgst)->smplcnt) + error = EOVERFLOW; + smplcnt = VSD(tdgstclust64, tdgst)->smplcnt; + ctd64tree = &VSD(tdgstclust64, tdgst)->ctdtree; + is32bit = 0; + ctd32tree = NULL; + ctd32 = NULL; + break; + default: + error = EINVAL; + break; + } + + if (error) + return (error); + + /* + * Inspired by Ted Dunning's AVLTreeDigest.java + */ + do { +#if defined(DIAGNOSTIC) + KASSERT(attempt < 5, + ("%s: Too many attempts", __func__)); +#endif + if (attempt >= 5) + return (EAGAIN); + + Q_SIFVAL(minz, Q_IFMAXVAL(minz)); + closest = ub = NULL; + sum = tmpsum = 0; + + if (is32bit) + lb = cur = (void *)(ctd32 = ARB_MIN(ctdth32, ctd32tree)); + else + lb = cur = (void *)(ctd64 = ARB_MIN(ctdth64, ctd64tree)); + + if (lb == NULL) /* Empty tree. */ + lb = (is32bit ? (void *)ARB_ROOT(ctd32tree) : + (void *)ARB_ROOT(ctd64tree)); + + /* + * Find the set of centroids with minimum distance to x and + * compute the sum of counts for all centroids with mean less + * than the first centroid in the set. + */ + for (; cur != NULL; + cur = (is32bit ? + (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : + (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { + if (is32bit) { + cnt = ctd32->cnt; + KASSERT(Q_PRECEQ(ctd32->mu, x), + ("%s: Q_RELPREC(mu,x)=%d", __func__, + Q_RELPREC(ctd32->mu, x))); + /* Ok to assign as both have same precision. */ + z = ctd32->mu; + } else { + cnt = ctd64->cnt; + KASSERT(Q_PRECEQ(ctd64->mu, x), + ("%s: Q_RELPREC(mu,x)=%d", __func__, + Q_RELPREC(ctd64->mu, x))); + /* Ok to assign as both have same precision. */ + z = ctd64->mu; + } + + error = Q_QSUBQ(&z, x); +#if defined(DIAGNOSTIC) + KASSERT(!error, ("%s: unexpected error %d", __func__, + error)); +#endif + if (error) + return (error); + + z = Q_QABS(z); + if (Q_QLTQ(z, minz)) { + minz = z; + lb = cur; + sum = tmpsum; + tmpsum += cnt; + } else if (Q_QGTQ(z, minz)) { + ub = cur; + break; + } + } + + cur = (is32bit ? + (void *)(ctd32 = (struct voistatdata_tdgstctd32 *)lb) : + (void *)(ctd64 = (struct voistatdata_tdgstctd64 *)lb)); + + for (n = 0; cur != ub; cur = (is32bit ? + (void *)(ctd32 = ARB_NEXT(ctdth32, ctd32tree, ctd32)) : + (void *)(ctd64 = ARB_NEXT(ctdth64, ctd64tree, ctd64)))) { + if (is32bit) + cnt = ctd32->cnt; + else + cnt = ctd64->cnt; + + q = Q_CTRLINI(16); + if (smplcnt == 1) + error = Q_QFRACI(&q, 1, 2); + else + /* [ sum + ((cnt - 1) / 2) ] / (smplcnt - 1) */ + error = Q_QFRACI(&q, (sum << 1) + cnt - 1, + (smplcnt - 1) << 1); + k = q; + /* k = q x 4 x samplcnt x attempt */ + error |= Q_QMULI(&k, 4 * smplcnt * attempt); + /* k = k x (1 - q) */ + error |= Q_QSUBI(&q, 1); + q = Q_QABS(q); + error |= Q_QMULQ(&k, q); +#if defined(DIAGNOSTIC) +#if !defined(_KERNEL) + double q_dbl, k_dbl, q2d, k2d; + q2d = Q_Q2D(q); + k2d = Q_Q2D(k); + q_dbl = smplcnt == 1 ? 0.5 : + (sum + ((cnt - 1) / 2.0)) / (double)(smplcnt - 1); + k_dbl = 4 * smplcnt * q_dbl * (1.0 - q_dbl) * attempt; + /* + * If the difference between q and q_dbl is greater than + * the fractional precision of q, something is off. + * NB: q is holding the value of 1 - q + */ + q_dbl = 1.0 - q_dbl; + KASSERT((q_dbl > q2d ? q_dbl - q2d : q2d - q_dbl) < + (1.05 * ((double)1 / (double)(1ULL << Q_NFBITS(q)))), + ("Q-type q bad precision")); + KASSERT((k_dbl > k2d ? k_dbl - k2d : k2d - k_dbl) < + 1.0 + (0.01 * smplcnt), + ("Q-type k bad precision")); +#endif /* !_KERNEL */ + KASSERT(!error, ("%s: unexpected error %d", __func__, + error)); +#endif /* DIAGNOSTIC */ + if (error) + return (error); + if ((is32bit && ((ctd32->cnt + weight) <= + (uint64_t)Q_GIVAL(k))) || + (!is32bit && ((ctd64->cnt + weight) <= + (uint64_t)Q_GIVAL(k)))) { + n++; + /* random() produces 31 bits. */ + if (random() < (INT32_MAX / n)) + closest = cur; + } + sum += cnt; + } + } while (closest == NULL && + (is32bit ? ARB_FULL(ctd32tree) : ARB_FULL(ctd64tree)) && + (error = stats_v1_vsd_tdgst_compress(vs_dtype, tdgst, + attempt++)) == 0); + + if (error) + return (error); + + if (closest != NULL) { + /* Merge with an existing centroid. */ + if (is32bit) { + ctd32 = (struct voistatdata_tdgstctd32 *)closest; + error = Q_QSUBQ(&x, ctd32->mu); + error = error ? error : + Q_QDIVI(&x, ctd32->cnt + weight); + if (error || (error = Q_QADDQ(&ctd32->mu, x))) { +#ifdef DIAGNOSTIC + KASSERT(!error, ("%s: unexpected error %d", + __func__, error)); +#endif + return (error); + } + ctd32->cnt += weight; + error = ARB_REBALANCE(ctdth32, ctd32tree, ctd32) == + NULL ? 0 : EALREADY; +#ifdef DIAGNOSTIC + RB_REBALANCE(rbctdth32, + &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); +#endif + } else { + ctd64 = (struct voistatdata_tdgstctd64 *)closest; + error = Q_QSUBQ(&x, ctd64->mu); + error = error ? error : + Q_QDIVI(&x, ctd64->cnt + weight); + if (error || (error = Q_QADDQ(&ctd64->mu, x))) { + KASSERT(!error, ("%s: unexpected error %d", + __func__, error)); + return (error); + } + ctd64->cnt += weight; + error = ARB_REBALANCE(ctdth64, ctd64tree, ctd64) == + NULL ? 0 : EALREADY; +#ifdef DIAGNOSTIC + RB_REBALANCE(rbctdth64, + &VSD(tdgstclust64, tdgst)->rbctdtree, ctd64); +#endif + } + } else { + /* + * Add a new centroid. If digest compression is working + * correctly, there should always be at least one free. + */ + if (is32bit) { + ctd32 = ARB_GETFREE(ctd32tree, ctdlnk); +#ifdef DIAGNOSTIC + KASSERT(ctd32 != NULL, + ("%s: t-digest@%p has no free centroids", + __func__, tdgst)); +#endif + if (ctd32 == NULL) + return (EAGAIN); + if ((error = Q_QCPYVALQ(&ctd32->mu, x))) + return (error); + ctd32->cnt = weight; + error = ARB_INSERT(ctdth32, ctd32tree, ctd32) == NULL ? + 0 : EALREADY; +#ifdef DIAGNOSTIC + RB_INSERT(rbctdth32, + &VSD(tdgstclust32, tdgst)->rbctdtree, ctd32); +#endif + } else { + ctd64 = ARB_GETFREE(ctd64tree, ctdlnk); +#ifdef DIAGNOSTIC + KASSERT(ctd64 != NULL, + ("%s: t-digest@%p has no free centroids", + __func__, tdgst)); +#endif + if (ctd64 == NULL) /* Should not happen. */ + return (EAGAIN); + /* Direct assignment ok as both have same type/prec. */ + ctd64->mu = x; + ctd64->cnt = weight; + error = ARB_INSERT(ctdth64, ctd64tree, ctd64) == NULL ? + 0 : EALREADY; +#ifdef DIAGNOSTIC + RB_INSERT(rbctdth64, &VSD(tdgstclust64, + tdgst)->rbctdtree, ctd64); +#endif + } + } + + if (is32bit) + VSD(tdgstclust32, tdgst)->smplcnt += weight; + else { + VSD(tdgstclust64, tdgst)->smplcnt += weight; + +#ifdef DIAGNOSTIC + struct rbctdth64 *rbctdtree = + &VSD(tdgstclust64, tdgst)->rbctdtree; + struct voistatdata_tdgstctd64 *rbctd64; + int i = 0; + ARB_FOREACH(ctd64, ctdth64, ctd64tree) { + rbctd64 = (i == 0 ? RB_MIN(rbctdth64, rbctdtree) : + RB_NEXT(rbctdth64, rbctdtree, rbctd64)); + + if (i >= ARB_CURNODES(ctd64tree) + || ctd64 != rbctd64 + || ARB_MIN(ctdth64, ctd64tree) != + RB_MIN(rbctdth64, rbctdtree) + || ARB_MAX(ctdth64, ctd64tree) != + RB_MAX(rbctdth64, rbctdtree) + || ARB_LEFTIDX(ctd64, ctdlnk) != + ARB_SELFIDX(ctd64tree, RB_LEFT(rbctd64, rblnk)) + || ARB_RIGHTIDX(ctd64, ctdlnk) != + ARB_SELFIDX(ctd64tree, RB_RIGHT(rbctd64, rblnk)) + || ARB_PARENTIDX(ctd64, ctdlnk) != + ARB_SELFIDX(ctd64tree, + RB_PARENT(rbctd64, rblnk))) { + Q_TOSTR(ctd64->mu, -1, 10, qstr, sizeof(qstr)); + printf("ARB ctd=%3d p=%3d l=%3d r=%3d c=%2d " + "mu=%s\n", + (int)ARB_SELFIDX(ctd64tree, ctd64), + ARB_PARENTIDX(ctd64, ctdlnk), + ARB_LEFTIDX(ctd64, ctdlnk), + ARB_RIGHTIDX(ctd64, ctdlnk), + ARB_COLOR(ctd64, ctdlnk), + qstr); + + Q_TOSTR(rbctd64->mu, -1, 10, qstr, + sizeof(qstr)); + printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d " + "mu=%s\n", + (int)ARB_SELFIDX(ctd64tree, rbctd64), + (int)ARB_SELFIDX(ctd64tree, + RB_PARENT(rbctd64, rblnk)), + (int)ARB_SELFIDX(ctd64tree, + RB_LEFT(rbctd64, rblnk)), + (int)ARB_SELFIDX(ctd64tree, + RB_RIGHT(rbctd64, rblnk)), + RB_COLOR(rbctd64, rblnk), + qstr); + + panic("RB@%p and ARB@%p trees differ\n", + rbctdtree, ctd64tree); + } + i++; + } +#endif /* DIAGNOSTIC */ + } + + return (error); +} + +static inline int +stats_v1_voi_update_tdgst(enum vsd_dtype voi_dtype, struct voistatdata *voival, + struct voistat *vs, struct voistatdata_tdgst *tdgst) +{ + s64q_t x; + int error; + + error = 0; + + switch (vs->dtype) { + case VSD_DTYPE_TDGSTCLUST32: + /* Use same precision as the user's centroids. */ + Q_INI(&x, 0, Q_NFBITS( + ARB_CNODE(&VSD(tdgstclust32, tdgst)->ctdtree, 0)->mu)); + break; + case VSD_DTYPE_TDGSTCLUST64: + /* Use same precision as the user's centroids. */ + Q_INI(&x, 0, Q_NFBITS( + ARB_CNODE(&VSD(tdgstclust64, tdgst)->ctdtree, 0)->mu)); + break; + default: + KASSERT(vs->dtype == VSD_DTYPE_TDGSTCLUST32 || + vs->dtype == VSD_DTYPE_TDGSTCLUST64, + ("%s: vs->dtype(%d) != VSD_DTYPE_TDGSTCLUST<32|64>", + __func__, vs->dtype)); + return (EINVAL); + } + + /* + * XXXLAS: Should have both a signed and unsigned 'x' variable to avoid + * returning EOVERFLOW if the voival would have fit in a u64q_t. + */ + switch (voi_dtype) { + case VSD_DTYPE_INT_S32: + error = Q_QCPYVALI(&x, voival->int32.s32); + break; + case VSD_DTYPE_INT_U32: + error = Q_QCPYVALI(&x, voival->int32.u32); + break; + case VSD_DTYPE_INT_S64: + error = Q_QCPYVALI(&x, voival->int64.s64); + break; + case VSD_DTYPE_INT_U64: + error = Q_QCPYVALI(&x, voival->int64.u64); + break; + case VSD_DTYPE_INT_SLONG: + error = Q_QCPYVALI(&x, voival->intlong.slong); + break; + case VSD_DTYPE_INT_ULONG: + error = Q_QCPYVALI(&x, voival->intlong.ulong); + break; + case VSD_DTYPE_Q_S32: + error = Q_QCPYVALQ(&x, voival->q32.sq32); + break; + case VSD_DTYPE_Q_U32: + error = Q_QCPYVALQ(&x, voival->q32.uq32); + break; + case VSD_DTYPE_Q_S64: + error = Q_QCPYVALQ(&x, voival->q64.sq64); + break; + case VSD_DTYPE_Q_U64: + error = Q_QCPYVALQ(&x, voival->q64.uq64); + break; + default: + error = EINVAL; + break; + } + + if (error || + (error = stats_v1_vsd_tdgst_add(vs->dtype, tdgst, x, 1, 1))) + return (error); + + vs->flags |= VS_VSDVALID; + return (0); +} + +int +stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id, + enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags) +{ + struct voi *v; + struct voistat *vs; + void *statevsd, *vsd; + int error, i, tmperr; + + error = 0; + + if (sb == NULL || sb->abi != STATS_ABI_V1 || voi_id >= NVOIS(sb) || + voi_dtype == 0 || voi_dtype >= VSD_NUM_DTYPES || voival == NULL) + return (EINVAL); + v = &sb->vois[voi_id]; + if (voi_dtype != v->dtype || v->id < 0 || + ((flags & SB_VOI_RELUPDATE) && !(v->flags & VOI_REQSTATE))) + return (EINVAL); + + vs = BLOB_OFFSET(sb, v->stats_off); + if (v->flags & VOI_REQSTATE) + statevsd = BLOB_OFFSET(sb, vs->data_off); + else + statevsd = NULL; + + if (flags & SB_VOI_RELUPDATE) { + switch (voi_dtype) { + case VSD_DTYPE_INT_S32: + voival->int32.s32 += + VSD(voistate, statevsd)->prev.int32.s32; + break; + case VSD_DTYPE_INT_U32: + voival->int32.u32 += + VSD(voistate, statevsd)->prev.int32.u32; + break; + case VSD_DTYPE_INT_S64: + voival->int64.s64 += + VSD(voistate, statevsd)->prev.int64.s64; + break; + case VSD_DTYPE_INT_U64: + voival->int64.u64 += + VSD(voistate, statevsd)->prev.int64.u64; + break; + case VSD_DTYPE_INT_SLONG: + voival->intlong.slong += + VSD(voistate, statevsd)->prev.intlong.slong; + break; + case VSD_DTYPE_INT_ULONG: + voival->intlong.ulong += + VSD(voistate, statevsd)->prev.intlong.ulong; + break; + case VSD_DTYPE_Q_S32: + error = Q_QADDQ(&voival->q32.sq32, + VSD(voistate, statevsd)->prev.q32.sq32); + break; + case VSD_DTYPE_Q_U32: + error = Q_QADDQ(&voival->q32.uq32, + VSD(voistate, statevsd)->prev.q32.uq32); + break; + case VSD_DTYPE_Q_S64: + error = Q_QADDQ(&voival->q64.sq64, + VSD(voistate, statevsd)->prev.q64.sq64); + break; + case VSD_DTYPE_Q_U64: + error = Q_QADDQ(&voival->q64.uq64, + VSD(voistate, statevsd)->prev.q64.uq64); + break; + default: + KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); + break; + } + } + + if (error) + return (error); + + for (i = v->voistatmaxid; i > 0; i--) { + vs = &((struct voistat *)BLOB_OFFSET(sb, v->stats_off))[i]; + if (vs->stype < 0) + continue; + + vsd = BLOB_OFFSET(sb, vs->data_off); + + switch (vs->stype) { + case VS_STYPE_MAX: + tmperr = stats_v1_voi_update_max(voi_dtype, voival, + vs, vsd); + break; + case VS_STYPE_MIN: + tmperr = stats_v1_voi_update_min(voi_dtype, voival, + vs, vsd); + break; + case VS_STYPE_SUM: + tmperr = stats_v1_voi_update_sum(voi_dtype, voival, + vs, vsd); + break; + case VS_STYPE_HIST: + tmperr = stats_v1_voi_update_hist(voi_dtype, voival, + vs, vsd); + break; + case VS_STYPE_TDGST: + tmperr = stats_v1_voi_update_tdgst(voi_dtype, voival, + vs, vsd); + break; + default: + KASSERT(0, ("Unknown VOI stat type %d", vs->stype)); + break; + } + + if (tmperr) { + error = tmperr; + VS_INCERRS(vs); + } + } + + if (statevsd) { + switch (voi_dtype) { + case VSD_DTYPE_INT_S32: + VSD(voistate, statevsd)->prev.int32.s32 = + voival->int32.s32; + break; + case VSD_DTYPE_INT_U32: + VSD(voistate, statevsd)->prev.int32.u32 = + voival->int32.u32; + break; + case VSD_DTYPE_INT_S64: + VSD(voistate, statevsd)->prev.int64.s64 = + voival->int64.s64; + case VSD_DTYPE_INT_U64: + VSD(voistate, statevsd)->prev.int64.u64 = + voival->int64.u64; + break; + case VSD_DTYPE_INT_SLONG: + VSD(voistate, statevsd)->prev.intlong.slong = + voival->intlong.slong; + break; + case VSD_DTYPE_INT_ULONG: + VSD(voistate, statevsd)->prev.intlong.ulong = + voival->intlong.ulong; + break; + case VSD_DTYPE_Q_S32: + error = Q_QCPYVALQ( + &VSD(voistate, statevsd)->prev.q32.sq32, + voival->q32.sq32); + break; + case VSD_DTYPE_Q_U32: + error = Q_QCPYVALQ( + &VSD(voistate, statevsd)->prev.q32.uq32, + voival->q32.uq32); + break; + case VSD_DTYPE_Q_S64: + error = Q_QCPYVALQ( + &VSD(voistate, statevsd)->prev.q64.sq64, + voival->q64.sq64); + break; + case VSD_DTYPE_Q_U64: + error = Q_QCPYVALQ( + &VSD(voistate, statevsd)->prev.q64.uq64, + voival->q64.uq64); + break; + default: + KASSERT(0, ("Unknown VOI data type %d", voi_dtype)); + break; + } + } + + return (error); +} + +#ifdef _KERNEL + +static void +stats_init(void *arg) +{ + +} +SYSINIT(stats, SI_SUB_KDTRACE, SI_ORDER_FIRST, stats_init, NULL); + +/* + * Sysctl handler to display the list of available stats templates. + */ +static int +stats_tpl_list_available(SYSCTL_HANDLER_ARGS) +{ + struct sbuf *s; + int err, i; + + err = 0; + + /* We can tolerate ntpl being stale, so do not take the lock. */ + s = sbuf_new(NULL, NULL, /* +1 per tpl for , */ + ntpl * (STATS_TPL_MAX_STR_SPEC_LEN + 1), SBUF_FIXEDLEN); + if (s == NULL) + return (ENOMEM); + + TPL_LIST_RLOCK(); + for (i = 0; i < ntpl; i++) { + err = sbuf_printf(s, "%s\"%s\":%u", i ? "," : "", + tpllist[i]->mb->tplname, tpllist[i]->mb->tplhash); + if (err) { + /* Sbuf overflow condition. */ + err = EOVERFLOW; + break; + } + } + TPL_LIST_RUNLOCK(); + + if (!err) { + sbuf_finish(s); + err = sysctl_handle_string(oidp, sbuf_data(s), 0, req); + } + + sbuf_delete(s); + return (err); +} + +/* + * Called by subsystem-specific sysctls to report and/or parse the list of + * templates being sampled and their sampling rates. A stats_tpl_sr_cb_t + * conformant function pointer must be passed in as arg1, which is used to + * interact with the subsystem's stats template sample rates list. If arg2 > 0, + * a zero-initialised allocation of arg2-sized contextual memory is + * heap-allocated and passed in to all subsystem callbacks made during the + * operation of stats_tpl_sample_rates(). + * + * XXXLAS: Assumes templates are never removed, which is currently true but may + * need to be reworked in future if dynamic template management becomes a + * requirement e.g. to support kernel module based templates. + */ +int +stats_tpl_sample_rates(SYSCTL_HANDLER_ARGS) +{ + char kvpair_fmt[16], tplspec_fmt[16]; + char tpl_spec[STATS_TPL_MAX_STR_SPEC_LEN]; + char tpl_name[TPL_MAX_NAME_LEN + 2]; /* +2 for "" */ + stats_tpl_sr_cb_t subsys_cb; + void *subsys_ctx; + char *buf, *new_rates_usr_str, *tpl_name_p; + struct stats_tpl_sample_rate *rates; + struct sbuf *s, _s; + uint32_t cum_pct, pct, tpl_hash; + int err, i, off, len, newlen, nrates; + + buf = NULL; + rates = NULL; + err = nrates = 0; + subsys_cb = (stats_tpl_sr_cb_t)arg1; + KASSERT(subsys_cb != NULL, ("%s: subsys_cb == arg1 == NULL", __func__)); + if (arg2 > 0) + subsys_ctx = malloc(arg2, M_TEMP, M_WAITOK | M_ZERO); + else + subsys_ctx = NULL; + + /* Grab current count of subsystem rates. */ + err = subsys_cb(TPL_SR_UNLOCKED_GET, NULL, &nrates, subsys_ctx); + if (err) + goto done; + + /* +1 to ensure we can append '\0' post copyin, +5 per rate for =nnn, */ + len = max(req->newlen + 1, nrates * (STATS_TPL_MAX_STR_SPEC_LEN + 5)); + + if (req->oldptr != NULL || req->newptr != NULL) + buf = malloc(len, M_TEMP, M_WAITOK); + + if (req->oldptr != NULL) { + if (nrates == 0) { + /* No rates, so return an empty string via oldptr. */ + err = SYSCTL_OUT(req, "", 1); + if (err) + goto done; + goto process_new; + } + + s = sbuf_new(&_s, buf, len, SBUF_FIXEDLEN | SBUF_INCLUDENUL); + + /* Grab locked count of, and ptr to, subsystem rates. */ + err = subsys_cb(TPL_SR_RLOCKED_GET, &rates, &nrates, + subsys_ctx); + if (err) + goto done; + TPL_LIST_RLOCK(); + for (i = 0; i < nrates && !err; i++) { + err = sbuf_printf(s, "%s\"%s\":%u=%u", i ? "," : "", + tpllist[rates[i].tpl_slot_id]->mb->tplname, + tpllist[rates[i].tpl_slot_id]->mb->tplhash, + rates[i].tpl_sample_pct); + } + TPL_LIST_RUNLOCK(); + /* Tell subsystem that we're done with its rates list. */ + err = subsys_cb(TPL_SR_RUNLOCK, &rates, &nrates, subsys_ctx); + if (err) + goto done; + + err = sbuf_finish(s); + if (err) + goto done; /* We lost a race for buf to be too small. */ + + /* Return the rendered string data via oldptr. */ + err = SYSCTL_OUT(req, sbuf_data(s), sbuf_len(s)); + } else { + /* Return the upper bound size for buffer sizing requests. */ + err = SYSCTL_OUT(req, NULL, len); + } + +process_new: + if (err || req->newptr == NULL) + goto done; + + newlen = req->newlen - req->newidx; + err = SYSCTL_IN(req, buf, newlen); + if (err) + goto done; + + /* + * Initialise format strings at run time. + * + * Write the max template spec string length into the + * template_spec=percent key-value pair parsing format string as: + * " %[^=]=%u %n" + * + * Write the max template name string length into the tplname:tplhash + * parsing format string as: + * "%[^:]:%u" + * + * Subtract 1 for \0 appended by sscanf(). + */ + sprintf(kvpair_fmt, " %%%zu[^=]=%%u %%n", sizeof(tpl_spec) - 1); + sprintf(tplspec_fmt, "%%%zu[^:]:%%u", sizeof(tpl_name) - 1); + + /* + * Parse each CSV key-value pair specifying a template and its sample + * percentage. Whitespace either side of a key-value pair is ignored. + * Templates can be specified by name, hash, or name and hash per the + * following formats (chars in [] are optional): + * ["]["]= + * :hash=pct + * ["]["]:hash= + */ + cum_pct = nrates = 0; + rates = NULL; + buf[newlen] = '\0'; /* buf is at least newlen+1 in size. */ + new_rates_usr_str = buf; + while (isspace(*new_rates_usr_str)) + new_rates_usr_str++; /* Skip leading whitespace. */ + while (*new_rates_usr_str != '\0') { + tpl_name_p = tpl_name; + tpl_name[0] = '\0'; + tpl_hash = 0; + off = 0; + + /* + * Parse key-value pair which must perform 2 conversions, then + * parse the template spec to extract either name, hash, or name + * and hash depending on the three possible spec formats. The + * tplspec_fmt format specifier parses name or name and hash + * template specs, while the ":%u" format specifier parses + * hash-only template specs. If parsing is successfull, ensure + * the cumulative sampling percentage does not exceed 100. + */ + err = EINVAL; + if (2 != sscanf(new_rates_usr_str, kvpair_fmt, tpl_spec, &pct, + &off)) + break; + if ((1 > sscanf(tpl_spec, tplspec_fmt, tpl_name, &tpl_hash)) && + (1 != sscanf(tpl_spec, ":%u", &tpl_hash))) + break; + if ((cum_pct += pct) > 100) + break; + err = 0; + + /* Strip surrounding "" from template name if present. */ + len = strlen(tpl_name); + if (len > 0) { + if (tpl_name[len - 1] == '"') + tpl_name[--len] = '\0'; + if (tpl_name[0] == '"') { + tpl_name_p++; + len--; + } + } + + rates = stats_realloc(rates, 0, /* oldsz is unused in kernel. */ + (nrates + 1) * sizeof(*rates), M_WAITOK); + rates[nrates].tpl_slot_id = + stats_tpl_fetch_allocid(len ? tpl_name_p : NULL, tpl_hash); + if (rates[nrates].tpl_slot_id < 0) { + err = -rates[nrates].tpl_slot_id; + break; + } + rates[nrates].tpl_sample_pct = pct; + nrates++; + new_rates_usr_str += off; + if (*new_rates_usr_str != ',') + break; /* End-of-input or malformed. */ + new_rates_usr_str++; /* Move past comma to next pair. */ + } + + if (!err) { + if ((new_rates_usr_str - buf) < newlen) { + /* Entire input has not been consumed. */ + err = EINVAL; + } else { + /* + * Give subsystem the new rates. They'll return the + * appropriate rates pointer for us to garbage collect. + */ + err = subsys_cb(TPL_SR_PUT, &rates, &nrates, + subsys_ctx); + } + } + stats_free(rates); + +done: + free(buf, M_TEMP); + free(subsys_ctx, M_TEMP); + return (err); +} + +SYSCTL_NODE(_kern, OID_AUTO, stats, CTLFLAG_RW, NULL, + "stats(9) MIB"); + +SYSCTL_PROC(_kern_stats, OID_AUTO, templates, CTLTYPE_STRING|CTLFLAG_RD, + NULL, 0, stats_tpl_list_available, "A", + "list the name/hash of all available stats(9) templates"); + +#else /* ! _KERNEL */ + +static void __attribute__ ((constructor (0))) +stats_constructor() +{ + + pthread_rwlock_init(&tpllistlock, NULL); +} + +static void __attribute__ ((destructor (0))) +stats_destructor() +{ + + pthread_rwlock_destroy(&tpllistlock); +} + +#endif /* _KERNEL */ Index: sys/sys/stats.h =================================================================== --- /dev/null +++ sys/sys/stats.h @@ -0,0 +1,1252 @@ +/*- + * Copyright (c) 2014-2018 Netflix, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* + * A kernel and user space statistics gathering API + infrastructure. + * + * Author: Lawrence Stewart + * + * Things to ponder: + * - Register callbacks for events e.g. counter stat passing a threshold + * + * - How could this become SIFTRv2? Perhaps publishing records to a ring + * mapped between userspace and kernel? + * + * - Potential stat types: + * RATE: events per unit time + * TIMESERIES: timestamped records. Stored in voistate? + * EWMA: Exponential weighted moving average. + * + * - How should second order stats work e.g. stat "A" depends on "B" + * + * - How do variable time windows work e.g. give me per-RTT stats + * + * - Should the API always require the caller to manage locking? Or should the + * API provide optional functionality to lock a blob during operations. + * + * - Should we continue to store unpacked naturally aligned structs in the + * blob or move to packed structs? Relates to inter-host + * serialisation/endian issues. + */ + +#ifndef _SYS_STATS_H_ +#define _SYS_STATS_H_ + +#include + +#ifndef _KERNEL +/* + * XXXLAS: Hacks to enable sharing template creation code between kernel and + * userland e.g. tcp_stats.c + */ +#define VNET(n) n +#define VNET_DEFINE(t, n) static t n __unused +#endif /* ! _KERNEL */ + +#define TPL_MAX_NAME_LEN 64 + +/* + * The longest template string spec format i.e. the normative spec format, is: + * + * "": + * + * Therefore, the max string length of a template string spec is: + * + * - TPL_MAX_NAME_LEN + * - 2 chars for "" + * - 1 char for : separating name and hash + * - 10 chars for 32bit hash + */ +#define STATS_TPL_MAX_STR_SPEC_LEN (TPL_MAX_NAME_LEN + 13) + +struct sbuf; +struct sysctl_oid; +struct sysctl_req; + +enum sb_str_fmt { + SB_STRFMT_FREEFORM = 0, + SB_STRFMT_JSON, + SB_STRFMT_NUM_FMTS /* +1 to highest numbered format type. */ +}; + +/* VOI stat types. */ +enum voi_stype { + VS_STYPE_VOISTATE = 0, /* Reserved for internal API use. */ + VS_STYPE_SUM, + VS_STYPE_MAX, + VS_STYPE_MIN, + VS_STYPE_HIST, + VS_STYPE_TDGST, + VS_NUM_STYPES /* +1 to highest numbered stat type. */ +}; + +/* + * VOI stat data types used as storage for certain stat types and to marshall + * data through various API calls. + */ +enum vsd_dtype { + VSD_DTYPE_VOISTATE = 0, /* Reserved for internal API use. */ + VSD_DTYPE_INT_S32, /* int32_t */ + VSD_DTYPE_INT_U32, /* uint32_t */ + VSD_DTYPE_INT_S64, /* int64_t */ + VSD_DTYPE_INT_U64, /* uint64_t */ + VSD_DTYPE_INT_SLONG, /* long */ + VSD_DTYPE_INT_ULONG, /* unsigned long */ + VSD_DTYPE_Q_S32, /* s32q_t */ + VSD_DTYPE_Q_U32, /* u32q_t */ + VSD_DTYPE_Q_S64, /* s64q_t */ + VSD_DTYPE_Q_U64, /* u64q_t */ + VSD_DTYPE_CRHIST32, /* continuous range histogram, 32bit buckets */ + VSD_DTYPE_DRHIST32, /* discrete range histogram, 32bit buckets */ + VSD_DTYPE_DVHIST32, /* discrete value histogram, 32bit buckets */ + VSD_DTYPE_CRHIST64, /* continuous range histogram, 64bit buckets */ + VSD_DTYPE_DRHIST64, /* discrete range histogram, 64bit buckets */ + VSD_DTYPE_DVHIST64, /* discrete value histogram, 64bit buckets */ + VSD_DTYPE_TDGSTCLUST32, /* clustering variant t-digest, 32bit buckets */ + VSD_DTYPE_TDGSTCLUST64, /* clustering variant t-digest, 64bit buckets */ + VSD_NUM_DTYPES /* +1 to highest numbered data type. */ +}; + +struct voistatdata_int32 { + union { + int32_t s32; + uint32_t u32; + }; +}; + +struct voistatdata_int64 { + union { + int64_t s64; + uint64_t u64; + //counter_u64_t u64pcpu; + }; +}; + +struct voistatdata_intlong { + union { + long slong; + unsigned long ulong; + }; +}; + +struct voistatdata_q32 { + union { + s32q_t sq32; + u32q_t uq32; + }; +}; + +struct voistatdata_q64 { + union { + s64q_t sq64; + u64q_t uq64; + }; +}; + +struct voistatdata_numeric { + union { + struct { +#if BYTE_ORDER == BIG_ENDIAN + uint32_t pad; +#endif + union { + int32_t s32; + uint32_t u32; + }; +#if BYTE_ORDER == LITTLE_ENDIAN + uint32_t pad; +#endif + } int32; + + struct { +#if BYTE_ORDER == BIG_ENDIAN + uint32_t pad; +#endif + union { + s32q_t sq32; + u32q_t uq32; + }; +#if BYTE_ORDER == LITTLE_ENDIAN + uint32_t pad; +#endif + } q32; + + struct { +#if BYTE_ORDER == BIG_ENDIAN && LONG_BIT == 32 + uint32_t pad; +#endif + union { + long slong; + unsigned long ulong; + }; +#if BYTE_ORDER == LITTLE_ENDIAN && LONG_BIT == 32 + uint32_t pad; +#endif + } intlong; + + struct voistatdata_int64 int64; + struct voistatdata_q64 q64; + }; +}; + +/* Continuous range histogram with 32bit buckets. */ +struct voistatdata_crhist32 { + uint32_t oob; + struct { + struct voistatdata_numeric lb; + uint32_t cnt; + } bkts[]; +}; + +/* Continuous range histogram with 64bit buckets. */ +struct voistatdata_crhist64 { + uint64_t oob; + struct { + struct voistatdata_numeric lb; + uint64_t cnt; + } bkts[]; +}; + +/* Discrete range histogram with 32bit buckets. */ +struct voistatdata_drhist32 { + uint32_t oob; + struct { + struct voistatdata_numeric lb, ub; + uint32_t cnt; + } bkts[]; +}; + +/* Discrete range histogram with 64bit buckets. */ +struct voistatdata_drhist64 { + uint64_t oob; + struct { + struct voistatdata_numeric lb, ub; + uint64_t cnt; + } bkts[]; +}; + +/* Discrete value histogram with 32bit buckets. */ +struct voistatdata_dvhist32 { + uint32_t oob; + struct { + struct voistatdata_numeric val; + uint32_t cnt; + } bkts[]; +}; + +/* Discrete value histogram with 64bit buckets. */ +struct voistatdata_dvhist64 { + uint64_t oob; + struct { + struct voistatdata_numeric val; + uint64_t cnt; + } bkts[]; +}; + +struct voistatdata_hist { + union { + struct voistatdata_crhist32 crhist32; + struct voistatdata_crhist64 crhist64; + struct voistatdata_dvhist32 dvhist32; + struct voistatdata_dvhist64 dvhist64; + struct voistatdata_drhist32 drhist32; + struct voistatdata_drhist64 drhist64; + }; +}; + +struct voistatdata_tdgstctd32 { + ARB16_ENTRY() ctdlnk; +#ifdef DIAGNOSTIC + RB_ENTRY(voistatdata_tdgstctd32) rblnk; +#endif + s32q_t mu; + int32_t cnt; +}; + +struct voistatdata_tdgstctd64 { + ARB16_ENTRY() ctdlnk; +#ifdef DIAGNOSTIC + RB_ENTRY(voistatdata_tdgstctd64) rblnk; +#endif + s64q_t mu; + int64_t cnt; +}; + +struct voistatdata_tdgstctd { + union { + struct voistatdata_tdgstctd32 tdgstctd32; + struct voistatdata_tdgstctd64 tdgstctd64; + }; +}; + +/* Clustering variant, fixed-point t-digest with 32bit mu/counts. */ +struct voistatdata_tdgstclust32 { + uint32_t smplcnt; /* Count of samples. */ + uint32_t compcnt; /* Count of digest compressions. */ +#ifdef DIAGNOSTIC + RB_HEAD(rbctdth32, voistatdata_tdgstctd32) rbctdtree; +#endif + /* Array-based red-black tree of centroids. */ + ARB16_HEAD(ctdth32, voistatdata_tdgstctd32) ctdtree; +}; + +/* Clustering variant, fixed-point t-digest with 64bit mu/counts. */ +struct voistatdata_tdgstclust64 { + uint64_t smplcnt; /* Count of samples. */ + uint32_t compcnt; /* Count of digest compressions. */ +#ifdef DIAGNOSTIC + RB_HEAD(rbctdth64, voistatdata_tdgstctd64) rbctdtree; +#endif + /* Array-based red-black tree of centroids. */ + ARB16_HEAD(ctdth64, voistatdata_tdgstctd64) ctdtree; +}; + +struct voistatdata_tdgst { + union { + struct voistatdata_tdgstclust32 tdgstclust32; + struct voistatdata_tdgstclust64 tdgstclust64; + }; +}; + +struct voistatdata { + union { + struct voistatdata_int32 int32; + struct voistatdata_int64 int64; + struct voistatdata_intlong intlong; + struct voistatdata_q32 q32; + struct voistatdata_q64 q64; + struct voistatdata_crhist32 crhist32; + struct voistatdata_crhist64 crhist64; + struct voistatdata_dvhist32 dvhist32; + struct voistatdata_dvhist64 dvhist64; + struct voistatdata_drhist32 drhist32; + struct voistatdata_drhist64 drhist64; + struct voistatdata_tdgstclust32 tdgstclust32; + struct voistatdata_tdgstclust64 tdgstclust64; + }; +}; + +#define VSD_HIST_LBOUND_INF 0x01 +#define VSD_HIST_UBOUND_INF 0x02 +struct vss_hist_hlpr_info { + enum hist_bkt_alloc { + BKT_LIN, /* Linear steps. */ + BKT_EXP, /* Exponential steps. */ + BKT_LINEXP, /* Exponential steps, linear sub-steps. */ + BKT_USR /* User specified buckets. */ + } scheme; + enum vsd_dtype voi_dtype; + enum vsd_dtype hist_dtype; + uint32_t flags; + struct voistatdata_numeric lb; + struct voistatdata_numeric ub; + union { + struct { + const uint64_t stepinc; + } lin; + struct { + const uint64_t stepbase; + const uint64_t stepexp; + } exp; + struct { + const uint64_t stepbase; + const uint64_t linstepdiv; + } linexp; + struct { + const uint16_t nbkts; + const struct { + struct voistatdata_numeric lb, ub; + } *bkts; + } usr; + }; +}; + +struct vss_tdgst_hlpr_info { + enum vsd_dtype voi_dtype; + enum vsd_dtype tdgst_dtype; + uint32_t nctds; + uint32_t prec; +} __aligned(sizeof(void *)); + +struct vss_numeric_hlpr_info { + uint32_t prec; +}; + +struct vss_hlpr_info { + union { + struct vss_tdgst_hlpr_info tdgst; + struct vss_hist_hlpr_info hist; + struct vss_numeric_hlpr_info numeric; + }; +}; + +struct voistatspec; +typedef int (*vss_hlpr_fn)(enum vsd_dtype, struct voistatspec *, + struct vss_hlpr_info *); + +struct voistatspec { + vss_hlpr_fn hlpr; /* iv helper function. */ + struct vss_hlpr_info *hlprinfo; /* Helper function context. */ + struct voistatdata *iv; /* Initialisation value. */ + size_t vsdsz; /* Size of iv. */ + uint32_t flags; /* Stat flags. */ + enum vsd_dtype vs_dtype : 8; /* Stat's dtype. */ + enum voi_stype stype : 8; /* Stat type. */ +}; + +extern const char *vs_stype2name[VS_NUM_STYPES]; +extern const char *vs_stype2desc[VS_NUM_STYPES]; +extern const char *vsd_dtype2name[VSD_NUM_DTYPES]; +extern const size_t vsd_dtype2size[VSD_NUM_DTYPES]; +#define LIM_MIN 0 +#define LIM_MAX 1 +extern const struct voistatdata_numeric numeric_limits[2][VSD_DTYPE_Q_U64 + 1]; + +#define TYPEOF_MEMBER(type, member) __typeof(((type *)0)->member) +#define TYPEOF_MEMBER_PTR(type, member) __typeof(*(((type *)0)->member)) +#define SIZEOF_MEMBER(type, member) sizeof(TYPEOF_MEMBER(type, member)) + +/* Cast a pointer to a voistatdata struct of requested type. */ +#define _VSD(cnst, type, ptr) ((cnst struct voistatdata_##type *)(ptr)) +#define VSD(type, ptr) _VSD(, type, ptr) +#define CONSTVSD(type, ptr) _VSD(const, type, ptr) + +#define NVSS(vss_slots) (sizeof((vss_slots)) / sizeof(struct voistatspec)) +#define STATS_VSS(st, vsf, dt, hlp, hlpi) \ +((struct voistatspec){ \ + .stype = (st), \ + .flags = (vsf), \ + .vs_dtype = (dt), \ + .hlpr = (hlp), \ + .hlprinfo = (hlpi), \ +}) + +#define STATS_VSS_SUM() STATS_VSS(VS_STYPE_SUM, 0, 0, \ + (vss_hlpr_fn)&stats_vss_numeric_hlpr, NULL) + +#define STATS_VSS_MAX() STATS_VSS(VS_STYPE_MAX, 0, 0, \ + (vss_hlpr_fn)&stats_vss_numeric_hlpr, NULL) + +#define STATS_VSS_MIN() STATS_VSS(VS_STYPE_MIN, 0, 0, \ + (vss_hlpr_fn)&stats_vss_numeric_hlpr, NULL) + +#define STATS_VSS_HIST(htype, hist_hlpr_info) STATS_VSS(VS_STYPE_HIST, 0, \ + htype, (vss_hlpr_fn)&stats_vss_hist_hlpr, \ + (struct vss_hlpr_info *)(hist_hlpr_info)) + +#define STATS_VSS_TDIGEST(tdtype, tdgst_hlpr_info) STATS_VSS(VS_STYPE_TDGST, \ + 0, tdtype, (vss_hlpr_fn)&stats_vss_tdgst_hlpr, \ + (struct vss_hlpr_info *)(tdgst_hlpr_info)) + +#define TDGST_NCTRS2VSDSZ(tdtype, nctds) (sizeof(struct voistatdata_##tdtype) + \ + ((nctds) * sizeof(TYPEOF_MEMBER_PTR(struct voistatdata_##tdtype, \ + ctdtree.arb_nodes)))) + +#define TDGST_HLPR_INFO(dt, nc, nf) \ +(&(struct vss_tdgst_hlpr_info){ \ + .tdgst_dtype = (dt), \ + .nctds = (nc), \ + .prec = (nf) \ +}) + +#define STATS_VSS_TDGSTCLUST32(nctds, prec) \ + STATS_VSS_TDIGEST(VSD_DTYPE_TDGSTCLUST32, \ + TDGST_HLPR_INFO(VSD_DTYPE_TDGSTCLUST32, nctds, prec)) + +#define STATS_VSS_TDGSTCLUST64(nctds, prec) \ + STATS_VSS_TDIGEST(VSD_DTYPE_TDGSTCLUST64, \ + TDGST_HLPR_INFO(VSD_DTYPE_TDGSTCLUST64, nctds, prec)) + +#define HIST_VSDSZ2NBKTS(htype, dsz) \ + ((dsz - sizeof(struct voistatdata_##htype)) / \ + sizeof(TYPEOF_MEMBER(struct voistatdata_##htype, bkts[0]))) + +#define HIST_NBKTS2VSDSZ(htype, nbkts) (sizeof(struct voistatdata_##htype) + \ + ((nbkts) * sizeof(TYPEOF_MEMBER_PTR(struct voistatdata_##htype, bkts)))) + +#define HIST_HLPR_INFO_LIN_FIELDS(si) .lin.stepinc = (si) + +#define HIST_HLPR_INFO_EXP_FIELDS(sb, se) \ + .exp.stepbase = (sb), .exp.stepexp = (se) + +#define HIST_HLPR_INFO_LINEXP_FIELDS(nss, sb) \ + .linexp.linstepdiv = (nss), .linexp.stepbase = (sb) + +#define HIST_HLPR_INFO_USR_FIELDS(bbs) \ + .usr.bkts = (TYPEOF_MEMBER(struct vss_hist_hlpr_info, usr.bkts))(bbs), \ + .usr.nbkts = (sizeof(bbs) / sizeof(struct voistatdata_numeric[2])) + +#define HIST_HLPR_INFO(dt, sch, f, lbd, ubd, bkthlpr_fields) \ +(&(struct vss_hist_hlpr_info){ \ + .scheme = (sch), \ + .hist_dtype = (dt), \ + .flags = (f), \ + .lb = stats_ctor_vsd_numeric(lbd), \ + .ub = stats_ctor_vsd_numeric(ubd), \ + bkthlpr_fields \ +}) + +#define STATS_VSS_CRHIST32_LIN(lb, ub, stepinc, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST32, HIST_HLPR_INFO(VSD_DTYPE_CRHIST32, \ + BKT_LIN, vsdflags, lb, ub, HIST_HLPR_INFO_LIN_FIELDS(stepinc))) +#define STATS_VSS_CRHIST64_LIN(lb, ub, stepinc, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST64, HIST_HLPR_INFO(VSD_DTYPE_CRHIST64, \ + BKT_LIN, vsdflags, lb, ub, HIST_HLPR_INFO_LIN_FIELDS(stepinc))) + +#define STATS_VSS_CRHIST32_EXP(lb, ub, stepbase, stepexp, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST32, HIST_HLPR_INFO(VSD_DTYPE_CRHIST32, \ + BKT_EXP, vsdflags, lb, ub, HIST_HLPR_INFO_EXP_FIELDS(stepbase, stepexp))) +#define STATS_VSS_CRHIST64_EXP(lb, ub, stepbase, stepexp, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST64, HIST_HLPR_INFO(VSD_DTYPE_CRHIST64, \ + BKT_EXP, vsdflags, lb, ub, HIST_HLPR_INFO_EXP_FIELDS(stepbase, stepexp))) + +#define STATS_VSS_CRHIST32_LINEXP(lb, ub, nlinsteps, stepbase, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST32, HIST_HLPR_INFO(VSD_DTYPE_CRHIST32, \ + BKT_LINEXP, vsdflags, lb, ub, HIST_HLPR_INFO_LINEXP_FIELDS(nlinsteps, \ + stepbase))) +#define STATS_VSS_CRHIST64_LINEXP(lb, ub, nlinsteps, stepbase, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST64, HIST_HLPR_INFO(VSD_DTYPE_CRHIST64, \ + BKT_LINEXP, vsdflags, lb, ub, HIST_HLPR_INFO_LINEXP_FIELDS(nlinsteps, \ + stepbase))) + +#define STATS_VSS_CRHIST32_USR(bkts, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST32, HIST_HLPR_INFO(VSD_DTYPE_CRHIST32, \ + BKT_USR, vsdflags, 0, 0, HIST_HLPR_INFO_USR_FIELDS(bkts))) +#define STATS_VSS_CRHIST64_USR(bkts, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_CRHIST64, HIST_HLPR_INFO(VSD_DTYPE_CRHIST64, \ + BKT_USR, vsdflags, 0, 0, HIST_HLPR_INFO_USR_FIELDS(bkts))) + +#define STATS_VSS_DRHIST32_USR(bkts, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_DRHIST32, HIST_HLPR_INFO(VSD_DTYPE_DRHIST32, \ + BKT_USR, vsdflags, 0, 0, HIST_HLPR_INFO_USR_FIELDS(bkts))) +#define STATS_VSS_DRHIST64_USR(bkts, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_DRHIST64, HIST_HLPR_INFO(VSD_DTYPE_DRHIST64, \ + BKT_USR, vsdflags, 0, 0, HIST_HLPR_INFO_USR_FIELDS(bkts))) + +#define STATS_VSS_DVHIST32_USR(vals, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_DVHIST32, HIST_HLPR_INFO(VSD_DTYPE_DVHIST32, \ + BKT_USR, vsdflags, 0, 0, HIST_HLPR_INFO_USR_FIELDS(vals))) +#define STATS_VSS_DVHIST64_USR(vals, vsdflags) \ + STATS_VSS_HIST(VSD_DTYPE_DVHIST64, HIST_HLPR_INFO(VSD_DTYPE_DVHIST64, \ + BKT_USR, vsdflags, 0, 0, HIST_HLPR_INFO_USR_FIELDS(vals))) + +#define DRBKT(lb, ub) { stats_ctor_vsd_numeric(lb), stats_ctor_vsd_numeric(ub) } +#define DVBKT(val) DRBKT(val, val) +#define CRBKT(lb) DRBKT(lb, lb) +#define HBKTS(...) ((struct voistatdata_numeric [][2]){__VA_ARGS__}) + +#define VSD_HIST_FIELD(hist, cnst, hist_dtype, op, field) \ + (VSD_DTYPE_CRHIST32 == (hist_dtype) ? \ + op(_VSD(cnst, crhist32, hist)->field) : \ + (VSD_DTYPE_DRHIST32 == (hist_dtype) ? \ + op(_VSD(cnst, drhist32, hist)->field) : \ + (VSD_DTYPE_DVHIST32 == (hist_dtype) ? \ + op(_VSD(cnst, dvhist32, hist)->field) : \ + (VSD_DTYPE_CRHIST64 == (hist_dtype) ? \ + op(_VSD(cnst, crhist64, hist)->field) : \ + (VSD_DTYPE_DRHIST64 == (hist_dtype) ? \ + op(_VSD(cnst, drhist64, hist)->field) : \ + (op(_VSD(cnst, dvhist64, hist)->field))))))) +#define VSD_HIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_HIST_FIELD(hist, , hist_dtype, ,field) +#define VSD_CONSTHIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_HIST_FIELD(hist, const, hist_dtype, ,field) +#define VSD_HIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_HIST_FIELD(hist, , hist_dtype, (void *)&,field) +#define VSD_CONSTHIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_HIST_FIELD(hist, const, hist_dtype, (void *)&,field) + +#define VSD_CRHIST_FIELD(hist, cnst, hist_dtype, op, field) \ + (VSD_DTYPE_CRHIST32 == (hist_dtype) ? \ + op(_VSD(cnst, crhist32, hist)->field) : \ + op(_VSD(cnst, crhist64, hist)->field)) +#define VSD_CRHIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_CRHIST_FIELD(hist, , hist_dtype, , field) +#define VSD_CONSTCRHIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_CRHIST_FIELD(hist, const, hist_dtype, , field) +#define VSD_CRHIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_CRHIST_FIELD(hist, , hist_dtype, &, field) +#define VSD_CONSTCRHIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_CRHIST_FIELD(hist, const, hist_dtype, &, field) + +#define VSD_DRHIST_FIELD(hist, cnst, hist_dtype, op, field) \ + (VSD_DTYPE_DRHIST32 == (hist_dtype) ? \ + op(_VSD(cnst, drhist32, hist)->field) : \ + op(_VSD(cnst, drhist64, hist)->field)) +#define VSD_DRHIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_DRHIST_FIELD(hist, , hist_dtype, , field) +#define VSD_CONSTDRHIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_DRHIST_FIELD(hist, const, hist_dtype, , field) +#define VSD_DRHIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_DRHIST_FIELD(hist, , hist_dtype, &, field) +#define VSD_CONSTDRHIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_DRHIST_FIELD(hist, const, hist_dtype, &, field) + +#define VSD_DVHIST_FIELD(hist, cnst, hist_dtype, op, field) \ + (VSD_DTYPE_DVHIST32 == (hist_dtype) ? \ + op(_VSD(cnst, dvhist32, hist)->field) : \ + op(_VSD(cnst, dvhist64, hist)->field)) +#define VSD_DVHIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_DVHIST_FIELD(hist, , hist_dtype, , field) +#define VSD_CONSTDVHIST_FIELDVAL(hist, hist_dtype, field) \ + VSD_DVHIST_FIELD(hist, const, hist_dtype, , field) +#define VSD_DVHIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_DVHIST_FIELD(hist, , hist_dtype, &, field) +#define VSD_CONSTDVHIST_FIELDPTR(hist, hist_dtype, field) \ + VSD_DVHIST_FIELD(hist, const, hist_dtype, &, field) + +#define STATS_ABI_V1 1 +struct statsblobv1; + +enum sb_endianness { + SB_UE = 0, /* Unknown endian. */ + SB_LE, /* Little endian. */ + SB_BE /* Big endian. */ +}; + +struct statsblob { + uint8_t abi; + uint8_t endian; + uint16_t flags; + uint16_t maxsz; + uint16_t cursz; + uint8_t opaque[]; +} __aligned(sizeof(void *)); + +struct metablob { + char *tplname; + uint32_t tplhash; + struct voi_meta { + char *name; + char *desc; + } *voi_meta; +}; + +struct statsblob_tpl { + struct metablob *mb; /* Template metadata */ + struct statsblob *sb; /* Template schema */ +}; + +struct stats_tpl_sample_rate { + /* XXXLAS: Storing slot_id assumes templates are never removed. */ + int32_t tpl_slot_id; + uint32_t tpl_sample_pct; +}; + +/* Template sample rates list management callback actions. */ +enum stats_tpl_sr_cb_action { + TPL_SR_UNLOCKED_GET, + TPL_SR_RLOCKED_GET, + TPL_SR_RUNLOCK, + TPL_SR_PUT +}; + +/* + * Callback function pointer passed as arg1 to stats_tpl_sample_rates(). ctx is + * a heap-allocated, zero-initialised blob of contextual memory valid during a + * single stats_tpl_sample_rates() call and sized per the value passed as arg2. + * Returns 0 on success, an errno on error. + * - When called with "action == TPL_SR_*_GET", return the subsystem's rates + * list ptr and count, locked or unlocked as requested. + * - When called with "action == TPL_SR_RUNLOCK", unlock the subsystem's rates + * list ptr and count. Pair with a prior "action == TPL_SR_RLOCKED_GET" call. + * - When called with "action == TPL_SR_PUT, update the subsystem's rates list + * ptr and count to the sysctl processed values and return the inactive list + * details in rates/nrates for garbage collection by stats_tpl_sample_rates(). + */ +typedef int (*stats_tpl_sr_cb_t)(enum stats_tpl_sr_cb_action action, + struct stats_tpl_sample_rate **rates, int *nrates, void *ctx); + +/* Flags related to iterating over a stats blob. */ +#define SB_IT_FIRST_CB 0x0001 +#define SB_IT_LAST_CB 0x0002 +#define SB_IT_FIRST_VOI 0x0004 +#define SB_IT_LAST_VOI 0x0008 +#define SB_IT_FIRST_VOISTAT 0x0010 +#define SB_IT_LAST_VOISTAT 0x0020 +#define SB_IT_NULLVOI 0x0040 +#define SB_IT_NULLVOISTAT 0x0080 + +struct sb_visit { + struct voistatdata *vs_data; + uint32_t tplhash; + uint32_t flags; + int16_t voi_id; + int16_t vs_dsz; + uint16_t vs_errs; + enum vsd_dtype voi_dtype : 8; + enum vsd_dtype vs_dtype : 8; + int8_t vs_stype; +}; + +/* Stats blob iterator callback called for each struct voi. */ +typedef int (*stats_blob_visitcb_t)(struct sb_visit *sbv, void *usrctx); + +/* ABI specific functions. */ +int stats_v1_tpl_alloc(const char *name, uint32_t flags); +int stats_v1_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, + const char *voi_name, enum vsd_dtype voi_dtype, uint32_t nvss, + struct voistatspec *vss, uint32_t flags); +int stats_v1_blob_init(struct statsblobv1 *sb, uint32_t tpl_id, uint32_t flags); +struct statsblobv1 * stats_v1_blob_alloc(uint32_t tpl_id, uint32_t flags); +int stats_v1_blob_clone(struct statsblobv1 **dst, size_t dstmaxsz, + struct statsblobv1 *src, uint32_t flags); +void stats_v1_blob_destroy(struct statsblobv1 *sb); +#define SB_CLONE_RSTSRC 0x0001 /* Reset src blob if clone successful. */ +#define SB_CLONE_ALLOCDST 0x0002 /* Allocate src->cursz memory for dst. */ +#define SB_CLONE_USRDSTNOFAULT 0x0004 /* Clone to wired userspace dst. */ +#define SB_CLONE_USRDST 0x0008 /* Clone to unwired userspace dst. */ +int stats_v1_blob_snapshot(struct statsblobv1 **dst, size_t dstmaxsz, + struct statsblobv1 *src, uint32_t flags); +#define SB_TOSTR_OBJDUMP 0x00000001 +#define SB_TOSTR_META 0x00000002 /* Lookup metablob and render metadata */ +int stats_v1_blob_tostr(struct statsblobv1 *sb, struct sbuf *buf, + enum sb_str_fmt fmt, uint32_t flags); +int stats_v1_blob_visit(struct statsblobv1 *sb, stats_blob_visitcb_t func, + void *usrctx); +/* VOI related function flags. */ +#define SB_VOI_RELUPDATE 0x00000001 /* voival is relative to previous value. */ +int stats_v1_voi_update(struct statsblobv1 *sb, int32_t voi_id, + enum vsd_dtype voi_dtype, struct voistatdata *voival, uint32_t flags); +int stats_v1_voistat_fetch_dptr(struct statsblobv1 *sb, int32_t voi_id, + enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd, + size_t *retvsdsz); + +/* End ABI specific functions. */ + +/* ABI agnostic functions. */ +int stats_vss_hlpr_init(enum vsd_dtype voi_dtype, uint32_t nvss, + struct voistatspec *vss); +void stats_vss_hlpr_cleanup(uint32_t nvss, struct voistatspec *vss); +int stats_vss_hist_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, + struct vss_hist_hlpr_info *info); +int stats_vss_numeric_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, + struct vss_numeric_hlpr_info *info); +int stats_vss_tdgst_hlpr(enum vsd_dtype voi_dtype, struct voistatspec *vss, + struct vss_tdgst_hlpr_info *info); +int stats_tpl_fetch(int tpl_id, struct statsblob_tpl **tpl); +int stats_tpl_fetch_allocid(const char *name, uint32_t hash); +int stats_tpl_id2name(uint32_t tpl_id, char *buf, size_t len); +int stats_tpl_sample_rates(struct sysctl_oid *oidp, void *arg1, intmax_t arg2, + struct sysctl_req *req); +int stats_tpl_sample_rollthedice(struct stats_tpl_sample_rate *rates, + int nrates, void *seed_bytes, size_t seed_len); +int stats_voistatdata_tostr(const struct voistatdata *vsd, + enum vsd_dtype voi_dtype, enum vsd_dtype vsd_dtype, size_t vsd_sz, + enum sb_str_fmt fmt, struct sbuf *buf, int objdump); + +static inline struct voistatdata_numeric +stats_ctor_vsd_numeric(uint64_t val) +{ + struct voistatdata_numeric tmp; + + tmp.int64.u64 = val; + + return (tmp); +} + +static inline int +stats_tpl_alloc(const char *name, uint32_t flags) +{ + + return (stats_v1_tpl_alloc(name, flags)); +} + +static inline int +stats_tpl_add_voistats(uint32_t tpl_id, int32_t voi_id, const char *voi_name, + enum vsd_dtype voi_dtype, uint32_t nvss, struct voistatspec *vss, + uint32_t flags) +{ + int ret; + + if ((ret = stats_vss_hlpr_init(voi_dtype, nvss, vss)) == 0) { + ret = stats_v1_tpl_add_voistats(tpl_id, voi_id, voi_name, + voi_dtype, nvss, vss, flags); + } + stats_vss_hlpr_cleanup(nvss, vss); + + return (ret); +} + +static inline int +stats_blob_init(struct statsblob *sb, uint32_t tpl_id, uint32_t flags) +{ + + return (stats_v1_blob_init((struct statsblobv1 *)sb, tpl_id, flags)); +} + +static inline struct statsblob * +stats_blob_alloc(uint32_t tpl_id, uint32_t flags) +{ + + return ((struct statsblob *)stats_v1_blob_alloc(tpl_id, flags)); +} + +static inline int +stats_blob_clone(struct statsblob **dst, size_t dstmaxsz, struct statsblob *src, + uint32_t flags) +{ + + return (stats_v1_blob_clone((struct statsblobv1 **)dst, dstmaxsz, + (struct statsblobv1 *)src, flags)); +} + +static inline void +stats_blob_destroy(struct statsblob *sb) +{ + + stats_v1_blob_destroy((struct statsblobv1 *)sb); +} + +static inline int +stats_blob_visit(struct statsblob *sb, stats_blob_visitcb_t func, void *usrctx) +{ + + return (stats_v1_blob_visit((struct statsblobv1 *)sb, func, usrctx)); +} + +static inline int +stats_blob_tostr(struct statsblob *sb, struct sbuf *buf, + enum sb_str_fmt fmt, uint32_t flags) +{ + + return (stats_v1_blob_tostr((struct statsblobv1 *)sb, buf, fmt, flags)); +} + +static inline int +stats_voistat_fetch_dptr(struct statsblob *sb, int32_t voi_id, + enum voi_stype stype, enum vsd_dtype *retdtype, struct voistatdata **retvsd, + size_t *retvsdsz) +{ + + return (stats_v1_voistat_fetch_dptr((struct statsblobv1 *)sb, + voi_id, stype, retdtype, retvsd, retvsdsz)); +} + +static inline int +stats_voistat_fetch_s64(struct statsblob *sb, int32_t voi_id, + enum voi_stype stype, int64_t *ret) +{ + struct voistatdata *vsd; + enum vsd_dtype vs_dtype; + int error; + + if ((error = stats_voistat_fetch_dptr(sb, voi_id, stype, &vs_dtype, &vsd, + NULL))) + return (error); + else if (VSD_DTYPE_INT_S64 != vs_dtype) + return (EFTYPE); + + *ret = vsd->int64.s64; + return (0); +} + +static inline int +stats_voistat_fetch_u64(struct statsblob *sb, int32_t voi_id, + enum voi_stype stype, uint64_t *ret) +{ + struct voistatdata *vsd; + enum vsd_dtype vs_dtype; + int error; + + if ((error = stats_voistat_fetch_dptr(sb, voi_id, stype, &vs_dtype, &vsd, + NULL))) + return (error); + else if (VSD_DTYPE_INT_U64 != vs_dtype) + return (EFTYPE); + + *ret = vsd->int64.u64; + return (0); +} + +static inline int +stats_voistat_fetch_s32(struct statsblob *sb, int32_t voi_id, + enum voi_stype stype, int32_t *ret) +{ + struct voistatdata *vsd; + enum vsd_dtype vs_dtype; + int error; + + if ((error = stats_voistat_fetch_dptr(sb, voi_id, stype, &vs_dtype, &vsd, + NULL))) + return (error); + else if (VSD_DTYPE_INT_S32 != vs_dtype) + return (EFTYPE); + + *ret = vsd->int32.s32; + return (0); +} + +static inline int +stats_voistat_fetch_u32(struct statsblob *sb, int32_t voi_id, + enum voi_stype stype, uint32_t *ret) +{ + struct voistatdata *vsd; + enum vsd_dtype vs_dtype; + int error; + + if ((error = stats_voistat_fetch_dptr(sb, voi_id, stype, &vs_dtype, &vsd, + NULL))) + return (error); + else if (VSD_DTYPE_INT_U32 != vs_dtype) + return (EFTYPE); + + *ret = vsd->int32.u32; + return (0); +} + +static inline int +stats_voistat_fetch_slong(struct statsblob *sb, int32_t voi_id, + enum voi_stype stype, long *ret) +{ + struct voistatdata *vsd; + enum vsd_dtype vs_dtype; + int error; + + if ((error = stats_voistat_fetch_dptr(sb, voi_id, stype, &vs_dtype, &vsd, + NULL))) + return (error); + else if (VSD_DTYPE_INT_SLONG != vs_dtype) + return (EFTYPE); + + *ret = vsd->intlong.slong; + return (0); +} + +static inline int +stats_voistat_fetch_ulong(struct statsblob *sb, int32_t voi_id, + enum voi_stype stype, unsigned long *ret) +{ + struct voistatdata *vsd; + enum vsd_dtype vs_dtype; + int error; + + if ((error = stats_voistat_fetch_dptr(sb, voi_id, stype, &vs_dtype, &vsd, + NULL))) + return (error); + else if (VSD_DTYPE_INT_ULONG != vs_dtype) + return (EFTYPE); + + *ret = vsd->intlong.ulong; + return (0); +} + +static inline int +stats_blob_snapshot(struct statsblob **dst, size_t dstmaxsz, + struct statsblob *src, uint32_t flags) +{ + + return (stats_v1_blob_snapshot((struct statsblobv1 **)dst, dstmaxsz, + (struct statsblobv1 *)src, flags)); +} + +static inline int +stats_voi_update_abs_s32(struct statsblob *sb, int32_t voi_id, int32_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int32.s32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_S32, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_s32(struct statsblob *sb, int32_t voi_id, int32_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int32.s32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_S32, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_u32(struct statsblob *sb, int32_t voi_id, uint32_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int32.u32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_U32, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_u32(struct statsblob *sb, int32_t voi_id, uint32_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int32.u32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_U32, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_s64(struct statsblob *sb, int32_t voi_id, int64_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int64.s64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_S64, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_s64(struct statsblob *sb, int32_t voi_id, int64_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int64.s64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_S64, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_u64(struct statsblob *sb, int32_t voi_id, uint64_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int64.u64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_U64, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_u64(struct statsblob *sb, int32_t voi_id, uint64_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.int64.u64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_U64, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_slong(struct statsblob *sb, int32_t voi_id, long voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.intlong.slong = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_SLONG, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_slong(struct statsblob *sb, int32_t voi_id, long voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.intlong.slong = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_SLONG, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_ulong(struct statsblob *sb, int32_t voi_id, + unsigned long voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.intlong.ulong = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_ULONG, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_ulong(struct statsblob *sb, int32_t voi_id, + unsigned long voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.intlong.ulong = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_INT_ULONG, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_sq32(struct statsblob *sb, int32_t voi_id, s32q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q32.sq32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_S32, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_sq32(struct statsblob *sb, int32_t voi_id, s32q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q32.sq32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_S32, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_uq32(struct statsblob *sb, int32_t voi_id, u32q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q32.uq32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_U32, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_uq32(struct statsblob *sb, int32_t voi_id, u32q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q32.uq32 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_U32, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_sq64(struct statsblob *sb, int32_t voi_id, s64q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q64.sq64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_S64, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_sq64(struct statsblob *sb, int32_t voi_id, s64q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q64.sq64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_S64, &tmp, SB_VOI_RELUPDATE)); +} + +static inline int +stats_voi_update_abs_uq64(struct statsblob *sb, int32_t voi_id, u64q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q64.uq64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_U64, &tmp, 0)); +} + +static inline int +stats_voi_update_rel_uq64(struct statsblob *sb, int32_t voi_id, u64q_t voival) +{ + + if (sb == NULL) + return (0); + + struct voistatdata tmp; + tmp.q64.uq64 = voival; + + return (stats_v1_voi_update((struct statsblobv1 *)sb, voi_id, + VSD_DTYPE_Q_U64, &tmp, SB_VOI_RELUPDATE)); +} + +/* End ABI agnostic functions. */ + +#endif /* _SYS_STATS_H_ */