diff --git a/usr.bin/ctlstat/Makefile b/usr.bin/ctlstat/Makefile --- a/usr.bin/ctlstat/Makefile +++ b/usr.bin/ctlstat/Makefile @@ -5,4 +5,6 @@ SDIR= ${SRCTOP}/sys CFLAGS+= -I${SDIR} +LIBADD= sbuf bsdxml + .include diff --git a/usr.bin/ctlstat/ctlstat.8 b/usr.bin/ctlstat/ctlstat.8 --- a/usr.bin/ctlstat/ctlstat.8 +++ b/usr.bin/ctlstat/ctlstat.8 @@ -34,7 +34,7 @@ .\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $ .\" $FreeBSD$ .\" -.Dd January 9, 2017 +.Dd April 22, 2021 .Dt CTLSTAT 8 .Os .Sh NAME @@ -48,6 +48,7 @@ .Op Fl d .Op Fl D .Op Fl j +.Op Fl P .Op Fl l Ar lun .Op Fl n Ar numdevs .Op Fl p Ar port @@ -83,6 +84,19 @@ JSON dump mode. Dump statistics every 30 seconds in JavaScript Object Notation (JSON) format. No statistics are computed in this mode, only raw numbers are displayed. +.It Fl P +Prometheus dump mode. +Dump statistics in a format suitable for ingestion into Prometheus. +When invoked with this option, +.Nm +dumps once, regardless of the +.Fl t +option. +This option is especially useful when invoked by +.Xr inetd 8 . +See the comments in +.Pa /etc/inetd.conf +for an example configuration. .It Fl l Ar lun Request statistics for the specified LUN. .It Fl n Ar numdevs @@ -116,7 +130,13 @@ .Xr camcontrol 8 , .Xr ctladm 8 , .Xr ctld 8 , -.Xr iostat 8 +.Xr iostat 8 , +.Lk +Prometheus project: +.Pa https://prometheus.io/ . +.Pp +Prometheus exposition formats: +.Lk https://prometheus.io/docs/instrumenting/exposition_formats/ . .Sh AUTHORS .An Ken Merry Aq Mt ken@FreeBSD.org .An Will Andrews Aq Mt will@FreeBSD.org diff --git a/usr.bin/ctlstat/ctlstat.c b/usr.bin/ctlstat/ctlstat.c --- a/usr.bin/ctlstat/ctlstat.c +++ b/usr.bin/ctlstat/ctlstat.c @@ -41,19 +41,23 @@ #include __FBSDID("$FreeBSD$"); -#include -#include #include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -76,8 +80,8 @@ static int ctl_stat_bits; -static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:"; -static const char *ctlstat_usage = "Usage: ctlstat [-CDdjht] [-l lunnum]" +static const char *ctlstat_opts = "Cc:DPdhjl:n:p:tw:"; +static const char *ctlstat_usage = "Usage: ctlstat [-CDPdjht] [-l lunnum]" "[-c count] [-n numdevs] [-w wait]\n"; struct ctl_cpu_stats { @@ -92,6 +96,7 @@ CTLSTAT_MODE_STANDARD, CTLSTAT_MODE_DUMP, CTLSTAT_MODE_JSON, + CTLSTAT_MODE_PROMETHEUS, } ctlstat_mode_types; #define CTLSTAT_FLAG_CPU (1 << 0) @@ -129,6 +134,15 @@ int header_interval; }; +struct cctl_portlist_data { + int level; + struct sbuf *cur_sb[32]; + int lun; + int ntargets; + char *target; + char **targets; +}; + #ifndef min #define min(x,y) (((x) < (y)) ? (x) : (y)) #endif @@ -381,6 +395,200 @@ printf("]}"); } +#define CTLSTAT_PROMETHEUS_LOOP(field) \ + for (i = n = 0; i < ctx->cur_items; i++) { \ + if (F_MASK(ctx) && bit_test(ctx->item_mask, \ + (int)stats[i].item) == 0) \ + continue; \ + for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \ + int lun = stats[i].item; \ + if (lun >= targdata.ntargets) \ + errx(1, "LUN %u out of range", lun); \ + printf("iscsi_target_" #field "{" \ + "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \ + "\n", \ + lun, targdata.targets[lun], iotypes[iotype], \ + stats[i].field[iotype]); \ + } \ + } \ + +#define CTLSTAT_PROMETHEUS_TIMELOOP(field) \ + for (i = n = 0; i < ctx->cur_items; i++) { \ + if (F_MASK(ctx) && bit_test(ctx->item_mask, \ + (int)stats[i].item) == 0) \ + continue; \ + for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \ + uint64_t us; \ + struct timespec ts; \ + int lun = stats[i].item; \ + if (lun >= targdata.ntargets) \ + errx(1, "LUN %u out of range", lun); \ + bintime2timespec(&stats[i].field[iotype], &ts); \ + us = ts.tv_sec * 1000000 + ts.tv_nsec / 1000; \ + printf("iscsi_target_" #field "{" \ + "lun=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \ + "\n", \ + lun, targdata.targets[lun], iotypes[iotype], us); \ + } \ + } \ + +static void +cctl_start_pelement(void *user_data, const char *name, const char **attr __unused) +{ + struct cctl_portlist_data* targdata = user_data; + + targdata->level++; + if ((u_int)targdata->level >= (sizeof(targdata->cur_sb) / + sizeof(targdata->cur_sb[0]))) + errx(1, "%s: too many nesting levels, %zd max", __func__, + sizeof(targdata->cur_sb) / sizeof(targdata->cur_sb[0])); + + targdata->cur_sb[targdata->level] = sbuf_new_auto(); + if (targdata->cur_sb[targdata->level] == NULL) + err(1, "%s: Unable to allocate sbuf", __func__); + + if (strcmp(name, "targ_port") == 0) { + targdata->lun = -1; + free(targdata->target); + targdata->target = NULL; + } +} + +static void +cctl_char_phandler(void *user_data, const XML_Char *str, int len) +{ + struct cctl_portlist_data *targdata = user_data; + + sbuf_bcat(targdata->cur_sb[targdata->level], str, len); +} + +static void +cctl_end_pelement(void *user_data, const char *name) +{ + struct cctl_portlist_data* targdata = user_data; + char *str; + + if (targdata->cur_sb[targdata->level] == NULL) + errx(1, "%s: no valid sbuf at level %d (name %s)", __func__, + targdata->level, name); + + if (sbuf_finish(targdata->cur_sb[targdata->level]) != 0) + err(1, "%s: sbuf_finish", __func__); + str = strdup(sbuf_data(targdata->cur_sb[targdata->level])); + if (str == NULL) + err(1, "%s can't allocate %zd bytes for string", __func__, + sbuf_len(targdata->cur_sb[targdata->level])); + + sbuf_delete(targdata->cur_sb[targdata->level]); + targdata->cur_sb[targdata->level] = NULL; + targdata->level--; + + if (strcmp(name, "target") == 0) { + free(targdata->target); + targdata->target = str; + } else if (strcmp(name, "lun") == 0) { + targdata->lun = atoi(str); + free(str); + } else if (strcmp(name, "targ_port") == 0) { + if (targdata->lun >= 0 && targdata->target != NULL) { + if (targdata->lun >= targdata->ntargets) { + /* + * This can happen for example if there are + * holes in CTL's lunlist. + */ + targdata->ntargets = MAX(targdata->ntargets * 2, + targdata->lun + 1); + size_t newsize = targdata->ntargets * + sizeof(char*); + targdata->targets = rallocx(targdata->targets, + newsize, MALLOCX_ZERO); + } + free(targdata->targets[targdata->lun]); + targdata->targets[targdata->lun] = targdata->target; + targdata->target = NULL; + } + free(str); + } else { + free(str); + } +} + +static void +ctlstat_prometheus(int fd, struct ctlstat_context *ctx) { + struct ctl_io_stats *stats = ctx->cur_stats; + struct ctl_lun_list list; + struct cctl_portlist_data targdata; + XML_Parser parser; + char *port_str = NULL; + int iotype, i, n, retval; + int port_len = 4096; + + bzero(&targdata, sizeof(targdata)); + targdata.ntargets = ctx->cur_items; + targdata.targets = calloc(targdata.ntargets, sizeof(char*)); +retry: + port_str = (char *)realloc(port_str, port_len); + bzero(&list, sizeof(list)); + list.alloc_len = port_len; + list.status = CTL_LUN_LIST_NONE; + list.lun_xml = port_str; + if (ioctl(fd, CTL_PORT_LIST, &list) == -1) + err(1, "%s: error issuing CTL_PORT_LIST ioctl", __func__); + if (list.status == CTL_LUN_LIST_ERROR) { + warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s", + __func__, list.error_str); + } else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) { + port_len <<= 1; + goto retry; + } + + parser = XML_ParserCreate(NULL); + if (parser == NULL) + err(1, "%s: Unable to create XML parser", __func__); + XML_SetUserData(parser, &targdata); + XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement); + XML_SetCharacterDataHandler(parser, cctl_char_phandler); + + retval = XML_Parse(parser, port_str, strlen(port_str), 1); + if (retval != 1) { + errx(1, "%s: Unable to parse XML: Error %d", __func__, + XML_GetErrorCode(parser)); + } + XML_ParserFree(parser); + + /* + * NB: Some clients will print a warning if we don't set Content-Length, + * but they still work. And the data still gets into Prometheus. + */ + printf("HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Type: text/plain; version=0.0.4\r\n" + "\r\n"); + + printf("# HELP iscsi_target_bytes Number of bytes\n" + "# TYPE iscsi_target_bytes counter\n"); + CTLSTAT_PROMETHEUS_LOOP(bytes); + printf("# HELP iscsi_target_dmas Number of DMA\n" + "# TYPE iscsi_target_dmas counter\n"); + CTLSTAT_PROMETHEUS_LOOP(dmas); + printf("# HELP iscsi_target_operations Number of operations\n" + "# TYPE iscsi_target_operations counter\n"); + CTLSTAT_PROMETHEUS_LOOP(operations); + printf("# HELP iscsi_target_time Cumulative operation time in us\n" + "# TYPE iscsi_target_time counter\n"); + CTLSTAT_PROMETHEUS_TIMELOOP(time); + printf("# HELP iscsi_target_dma_time Cumulative DMA time in us\n" + "# TYPE iscsi_target_dma_time counter\n"); + CTLSTAT_PROMETHEUS_TIMELOOP(dma_time); + + for (i = 0; i < targdata.ntargets; i++) + free(targdata.targets[i]); + free(targdata.target); + free(targdata.targets); + + fflush(stdout); +} + static void ctlstat_standard(struct ctlstat_context *ctx) { long double etime; @@ -659,6 +867,9 @@ ctx.flags |= CTLSTAT_FLAG_PORTS; break; } + case 'P': + ctx.mode = CTLSTAT_MODE_PROMETHEUS; + break; case 't': ctx.flags |= CTLSTAT_FLAG_TOTALS; break; @@ -676,6 +887,17 @@ if (F_LUNS(&ctx) && F_PORTS(&ctx)) errx(1, "Options -p and -l are exclusive."); + if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) { + if ((count != -1) || + (waittime != 1) || + /* NB: -P could be compatible with -t in the future */ + (ctx.flags & CTLSTAT_FLAG_TOTALS)) + { + errx(1, "Option -P is exclusive with -c, -w, and -t"); + } + count = 1; + } + if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) { if (F_TOTALS(&ctx)) ctx.flags |= CTLSTAT_FLAG_PORTS; @@ -712,6 +934,9 @@ case CTLSTAT_MODE_JSON: ctlstat_json(&ctx); break; + case CTLSTAT_MODE_PROMETHEUS: + ctlstat_prometheus(fd, &ctx); + break; default: break; } diff --git a/usr.sbin/inetd/inetd.conf b/usr.sbin/inetd/inetd.conf --- a/usr.sbin/inetd/inetd.conf +++ b/usr.sbin/inetd/inetd.conf @@ -120,6 +120,9 @@ # #prom-sysctl stream tcp nowait nobody /usr/sbin/prometheus_sysctl_exporter prometheus_sysctl_exporter -dgh # +# Example entry for the CTL exporter +#prom-ctl stream tcp nowait root /usr/bin/ctlstat ctlstat -P +# # Example entry for insecure rsync server # This is best combined with encrypted virtual tunnel interfaces, which can be # found with: apropos if_ | grep tunnel diff --git a/usr.sbin/services_mkdb/services b/usr.sbin/services_mkdb/services --- a/usr.sbin/services_mkdb/services +++ b/usr.sbin/services_mkdb/services @@ -2000,6 +2000,7 @@ prom-sysctl 9124/tcp #prometheus_sysctl_exporter(8) git 9418/tcp #git pack transfer service git 9418/udp #git pack transfer service +prom-ctl 9572/tcp #CTL prometheus odbcpathway 9628/tcp #ODBC Pathway Service odbcpathway 9628/udp #ODBC Pathway Service davsrc 9800/tcp #WebDav Source Port