Index: sys/fs/nfs/nfsport.h =================================================================== --- sys/fs/nfs/nfsport.h +++ sys/fs/nfs/nfsport.h @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include @@ -412,6 +413,13 @@ int cllocalopens; int cllocallockowners; int cllocallocks; + uint64_t srvstartcnt; + uint64_t srvdonecnt; + uint64_t srvbytes[NFSV4OP_NOPS + NFSV4OP_FAKENOPS]; + uint64_t srvops[NFSV4OP_NOPS + NFSV4OP_FAKENOPS]; + struct bintime srvduration[NFSV4OP_NOPS + NFSV4OP_FAKENOPS]; + struct bintime busyfrom; + struct bintime busytime; }; #ifdef _KERNEL Index: sys/fs/nfsserver/nfs_nfsdport.c =================================================================== --- sys/fs/nfsserver/nfs_nfsdport.c +++ sys/fs/nfsserver/nfs_nfsdport.c @@ -58,6 +58,7 @@ extern void (*nfsd_call_servertimer)(void); extern SVCPOOL *nfsrvd_pool; extern struct nfsv4lock nfsd_suspend_lock; +extern struct nfsstats newnfsstats; extern struct nfssessionhash nfssessionhash[NFSSESSIONHASHSIZE]; struct vfsoptlist nfsv4root_opt, nfsv4root_newopt; NFSDLOCKMUTEX; @@ -683,6 +684,8 @@ uiop->uio_td = NULL; nh = nfsrv_sequential_heuristic(uiop, vp); ioflag |= nh->nh_seqcount << IO_SEQSHIFT; + /* XXX KDM make this more systematic? */ + newnfsstats.srvbytes[NFSV4OP_READ] += uiop->uio_resid; error = VOP_READ(vp, uiop, IO_NODELOCKED | ioflag, cred); FREE((caddr_t)iv2, M_TEMP); if (error) { @@ -755,6 +758,8 @@ uiop->uio_offset = off; nh = nfsrv_sequential_heuristic(uiop, vp); ioflags |= nh->nh_seqcount << IO_SEQSHIFT; + /* XXX KDM make this more systematic? */ + newnfsstats.srvbytes[NFSV4OP_WRITE] += uiop->uio_resid; error = VOP_WRITE(vp, uiop, ioflags, cred); if (error == 0) nh->nh_nextoff = uiop->uio_offset; Index: sys/fs/nfsserver/nfs_nfsdsocket.c =================================================================== --- sys/fs/nfsserver/nfs_nfsdsocket.c +++ sys/fs/nfsserver/nfs_nfsdsocket.c @@ -399,6 +399,68 @@ NFSV4OP_COMMIT, }; +static struct mtx nfsrvd_statmtx; +MTX_SYSINIT(nfsst, &nfsrvd_statmtx, "NFSstat", MTX_DEF); + +static void +nfsrvd_statstart(int procnum, struct bintime *now) +{ + if (procnum > (NFSV4OP_NOPS + NFSV4OP_FAKENOPS)) { + printf("%s: procnum %d invalid\n", __func__, procnum); + return; + } + + mtx_lock(&nfsrvd_statmtx); + if (newnfsstats.srvstartcnt == newnfsstats.srvdonecnt) { + if (now != NULL) + newnfsstats.busyfrom = *now; + else + binuptime(&newnfsstats.busyfrom); + + } + newnfsstats.srvrpccnt[procnum]++; + newnfsstats.srvstartcnt++; + mtx_unlock(&nfsrvd_statmtx); + +} + +static void +nfsrvd_statend(int procnum, uint64_t bytes, struct bintime *now, + struct bintime *then) +{ + struct bintime dt, lnow; + + if (procnum > (NFSV4OP_NOPS + NFSV4OP_FAKENOPS)) { + printf("%s: procnum %d invalid\n", __func__, procnum); + return; + } + + if (now == NULL) { + now = &lnow; + binuptime(now); + } + + mtx_lock(&nfsrvd_statmtx); + + newnfsstats.srvbytes[procnum] += bytes; + newnfsstats.srvops[procnum]++; + + if (then != NULL) { + dt = *now; + bintime_sub(&dt, then); + bintime_add(&newnfsstats.srvduration[procnum], &dt); + } + + dt = *now; + bintime_sub(&dt, &newnfsstats.busyfrom); + bintime_add(&newnfsstats.busytime, &dt); + newnfsstats.busyfrom = *now; + + newnfsstats.srvdonecnt++; + + mtx_unlock(&nfsrvd_statmtx); +} + /* * Do an RPC. Basically, get the file handles translated to vnode pointers * and then call the appropriate server routine. The server routines are @@ -472,7 +534,9 @@ */ if (nd->nd_repstat && (nd->nd_flag & ND_NFSV2)) { *nd->nd_errp = nfsd_errmap(nd); - NFSINCRGLOBAL(newnfsstats.srvrpccnt[nfsv3to4op[nd->nd_procnum]]); + nfsrvd_statstart(nfsv3to4op[nd->nd_procnum], /*now*/ NULL); + nfsrvd_statend(nfsv3to4op[nd->nd_procnum], /*bytes*/ 0, + /*now*/ NULL, /*then*/ NULL); if (mp != NULL && nfs_writerpc[nd->nd_procnum] != 0) vn_finished_write(mp); goto out; @@ -487,6 +551,11 @@ if (nd->nd_flag & ND_NFSV4) { nfsrvd_compound(nd, isdgram, tag, taglen, minorvers, p); } else { + struct bintime start_time; + + binuptime(&start_time); + nfsrvd_statstart(nfsv3to4op[nd->nd_procnum], &start_time); + if (nfs_retfh[nd->nd_procnum] == 1) { if (vp) NFSVOPUNLOCK(vp, 0); @@ -501,7 +570,9 @@ } if (mp != NULL && nfs_writerpc[nd->nd_procnum] != 0) vn_finished_write(mp); - NFSINCRGLOBAL(newnfsstats.srvrpccnt[nfsv3to4op[nd->nd_procnum]]); + + nfsrvd_statend(nfsv3to4op[nd->nd_procnum], /*bytes*/ 0, + /*now*/ NULL, /*then*/ &start_time); } if (error) { if (error != EBADRPC) @@ -555,6 +626,7 @@ struct nfsexstuff nes, vpnes, savevpnes; fsid_t cur_fsid, save_fsid; static u_int64_t compref = 0; + struct bintime start_time; NFSVNO_EXINIT(&vpnes); NFSVNO_EXINIT(&savevpnes); @@ -767,12 +839,18 @@ } if (nfsv4_opflag[op].savereply) nd->nd_flag |= ND_SAVEREPLY; - NFSINCRGLOBAL(newnfsstats.srvrpccnt[nd->nd_procnum]); + + binuptime(&start_time); + nfsrvd_statstart(nd->nd_procnum, &start_time); + switch (op) { case NFSV4OP_PUTFH: error = nfsrv_mtofh(nd, &fh); - if (error) + if (error) { + nfsrvd_statend(nd->nd_procnum, /*bytes*/ 0, + /*now*/ NULL, /*then*/ &start_time); goto nfsmout; + } if (!nd->nd_repstat) nfsd_fhtovp(nd, &fh, LK_SHARED, &nvp, &nes, NULL, 0, p); @@ -985,6 +1063,7 @@ } } }; + if (error) { if (error == EBADRPC || error == NFSERR_BADXDR) { nd->nd_repstat = NFSERR_BADXDR; @@ -994,6 +1073,10 @@ } error = 0; } + + nfsrvd_statend(nd->nd_procnum, /*bytes*/ 0, + /*now*/ NULL, /*then*/ &start_time); + retops++; if (nd->nd_repstat) { *repp = nfsd_errmap(nd); Index: usr.bin/nfsstat/Makefile =================================================================== --- usr.bin/nfsstat/Makefile +++ usr.bin/nfsstat/Makefile @@ -4,7 +4,7 @@ PROG= nfsstat CFLAGS+=-DNFS -LIBADD= kvm +LIBADD= devstat kvm WARNS?= 3 Index: usr.bin/nfsstat/nfsstat.1 =================================================================== --- usr.bin/nfsstat/nfsstat.1 +++ usr.bin/nfsstat/nfsstat.1 @@ -28,7 +28,7 @@ .\" From: @(#)nfsstat.1 8.1 (Berkeley) 6/6/93 .\" $FreeBSD$ .\" -.Dd May 1, 2013 +.Dd November 13, 2014 .Dt NFSSTAT 1 .Os .Sh NAME @@ -57,6 +57,21 @@ .Bl -tag -width indent .It Fl c Only display client side statistics. +.It Fl d +Display statistics for the new NFS server that are similar to those +displayed by +.Xr iostat 8 . +This includes kilobytes per transfer, transfers per second, and megabytes per +second for read, write and all operations. +It also includes the current queue depth, the busy percentage, and latency +for all operations. +If the +.Fl W +flag is added, commits per second, commit latency, read latency and write +latency are also added to the display. +The busy percentage may exceed 100 at times. +That is not a bug, but is rather caused when a long running operation +completes. .It Fl e Report the extra statistics collected by the new NFS client and server for NFSv4. @@ -85,7 +100,8 @@ Use wide format with interval short summary. This option is especially useful when combined with -.Fl c +.Fl c , +.Fl d , or .Fl s and a time delay. Index: usr.bin/nfsstat/nfsstat.c =================================================================== --- usr.bin/nfsstat/nfsstat.c +++ usr.bin/nfsstat/nfsstat.c @@ -29,7 +29,37 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +/*- + * Copyright (c) 2004, 2008, 2009 Silicon Graphics International Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * substantially similar to the "NO WARRANTY" disclaimer below + * ("Disclaimer") and any redistribution must be conditioned upon + * including a substantially similar Disclaimer requirement for further + * binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGES. + */ + #ifndef lint static const char copyright[] = "@(#) Copyright (c) 1983, 1989, 1993\n\ @@ -68,6 +98,7 @@ #include #include #include +#include #include struct nlist nl[] = { @@ -88,16 +119,37 @@ static int extra_output = 0; void intpr(int, int); -void printhdr(int, int); +void printhdr(int, int, int); void sidewaysintpr(u_int, int, int); void usage(void); char *sperc1(int, int); char *sperc2(int, int); void exp_intpr(int, int); -void exp_sidewaysintpr(u_int, int, int); +void exp_sidewaysintpr(u_int, int, int, int); +void compute_new_stats(struct ext_nfsstats *cur_stats, + struct ext_nfsstats *prev_stats, int curop, long double etime, + long double *mbsec, long double *kb_per_transfer, + long double *transfers_per_second, long double *ms_per_transfer, + uint64_t *queue_len, long double *busy_pct); #define DELTA(field) (nfsstats.field - lastst.field) +#define STAT_TYPE_READ 0 +#define STAT_TYPE_WRITE 1 +#define STAT_TYPE_COMMIT 2 +#define NUM_STAT_TYPES 3 + +struct stattypes { + int stat_type; + int nfs_type; +} statstruct[] = { + {STAT_TYPE_READ, NFSV4OP_READ}, + {STAT_TYPE_WRITE, NFSV4OP_WRITE}, + {STAT_TYPE_COMMIT, NFSV4OP_COMMIT} +}; + +#define STAT_TYPE_TO_NFS(stat_type) statstruct[stat_type].nfs_type + int main(int argc, char **argv) { @@ -104,6 +156,7 @@ u_int interval; int clientOnly = -1; int serverOnly = -1; + int newStats = 0; int ch; char *memf, *nlistf; char errbuf[_POSIX2_LINE_MAX]; @@ -114,7 +167,7 @@ interval = 0; memf = nlistf = NULL; - while ((ch = getopt(argc, argv, "cesWM:mN:ow:z")) != -1) + while ((ch = getopt(argc, argv, "cdesWM:mN:ow:z")) != -1) switch(ch) { case 'M': memf = optarg; @@ -154,6 +207,9 @@ if (serverOnly < 0) serverOnly = 0; break; + case 'd': + newStats = 1; + break; case 's': serverOnly = 1; if (clientOnly < 0) @@ -207,7 +263,8 @@ if (interval) { if (run_v4 > 0) - exp_sidewaysintpr(interval, clientOnly, serverOnly); + exp_sidewaysintpr(interval, clientOnly, serverOnly, + newStats); else sidewaysintpr(interval, clientOnly, serverOnly); } else { @@ -578,7 +635,9 @@ nfsstatsp = &lastst; nfsrvstatsp = &lastsrvst; + readstats(&nfsstatsp, &nfsrvstatsp, 0); + if (clientOnly && !nfsstatsp) { printf("Client not present!\n"); clientOnly = 0; @@ -595,7 +654,7 @@ readstats(&nfsstatsp, &nfsrvstatsp, 0); if (--hdrcnt == 0) { - printhdr(clientOnly, serverOnly); + printhdr(clientOnly, serverOnly, 0); if (clientOnly && serverOnly) hdrcnt = 10; else @@ -655,14 +714,29 @@ } void -printhdr(int clientOnly, int serverOnly) +printhdr(int clientOnly, int serverOnly, int newStats) { - printf("%s%6.6s %6.6s %6.6s %6.6s %6.6s %6.6s %6.6s %6.6s", - ((serverOnly && clientOnly) ? " " : " "), - "GtAttr", "Lookup", "Rdlink", "Read", "Write", "Rename", - "Access", "Rddir"); - if (widemode && clientOnly) { - printf(" Attr Lkup BioR BioW Accs BioD"); + if (newStats) { + printf( +" [%s Read %s] [%s Write %s] %s[=========== Total ============]\n" +" KB/t tps MB/s%s KB/t tps MB/s%s %sKB/t tps MB/s ms ql %%b", + widemode ? "========" : "=====", + widemode ? "========" : "=====", + widemode ? "========" : "=====", + widemode ? "=======" : "====", + widemode ? "[Commit ] " : "", + widemode ? " ms" : "", + widemode ? " ms" : "", + widemode ? "tps ms " : ""); + + } else { + printf("%s%6.6s %6.6s %6.6s %6.6s %6.6s %6.6s %6.6s %6.6s", + ((serverOnly && clientOnly) ? " " : " "), + "GtAttr", "Lookup", "Rdlink", "Read", "Write", "Rename", + "Access", "Rddir"); + if (widemode && clientOnly) { + printf(" Attr Lkup BioR BioW Accs BioD"); + } } printf("\n"); fflush(stdout); @@ -672,7 +746,7 @@ usage(void) { (void)fprintf(stderr, - "usage: nfsstat [-cemoszW] [-M core] [-N system] [-w wait]\n"); + "usage: nfsstat [-cdemoszW] [-M core] [-N system] [-w wait]\n"); exit(1); } @@ -709,7 +783,76 @@ return(p); } +#define DELTA_T(field) \ + devstat_compute_etime(&cur_stats->field, \ + (prev_stats ? &prev_stats->field : NULL)) + /* + * XXX KDM mostly copied from ctlstat. We should commonize the code (and + * the devstat code) somehow. + */ +void +compute_new_stats(struct ext_nfsstats *cur_stats, + struct ext_nfsstats *prev_stats, int curop, + long double etime, long double *mbsec, + long double *kb_per_transfer, + long double *transfers_per_second, + long double *ms_per_transfer, uint64_t *queue_len, + long double *busy_pct) +{ + uint64_t total_bytes = 0, total_operations = 0; + struct bintime total_time_bt; + struct timespec total_time_ts; + + bzero(&total_time_bt, sizeof(total_time_bt)); + bzero(&total_time_ts, sizeof(total_time_ts)); + + total_bytes = cur_stats->srvbytes[curop]; + total_operations = cur_stats->srvops[curop]; + if (prev_stats != NULL) { + total_bytes -= prev_stats->srvbytes[curop]; + total_operations -= prev_stats->srvops[curop]; + } + + *mbsec = total_bytes; + *mbsec /= 1024 * 1024; + if (etime > 0.0) { + *busy_pct = DELTA_T(busytime); + if (*busy_pct < 0) + *busy_pct = 0; + *busy_pct /= etime; + *busy_pct *= 100; + if (*busy_pct < 0) + *busy_pct = 0; + *mbsec /= etime; + } else { + *busy_pct = 0; + *mbsec = 0; + } + *kb_per_transfer = total_bytes; + *kb_per_transfer /= 1024; + if (total_operations > 0) + *kb_per_transfer /= total_operations; + else + *kb_per_transfer = 0; + if (etime > 0.0) { + *transfers_per_second = total_operations; + *transfers_per_second /= etime; + } else { + *transfers_per_second = 0.0; + } + + if (total_operations > 0) { + *ms_per_transfer = DELTA_T(srvduration[curop]); + *ms_per_transfer /= total_operations; + *ms_per_transfer *= 1000; + } else + *ms_per_transfer = 0.0; + + *queue_len = cur_stats->srvstartcnt - cur_stats->srvdonecnt; +} + +/* * Print a description of the nfs stats for the experimental client/server. */ void @@ -961,6 +1104,25 @@ } } +static void +compute_totals(struct ext_nfsstats *total_stats, struct ext_nfsstats *cur_stats) +{ + int i; + + bzero(total_stats, sizeof(*total_stats)); + for (i = 0; i < (NFSV4OP_NOPS + NFSV4OP_FAKENOPS); i++) { + total_stats->srvbytes[0] += cur_stats->srvbytes[i]; + total_stats->srvops[0] += cur_stats->srvops[i]; + bintime_add(&total_stats->srvduration[0], + &cur_stats->srvduration[i]); + total_stats->srvrpccnt[i] = cur_stats->srvrpccnt[i]; + } + total_stats->srvstartcnt = cur_stats->srvstartcnt; + total_stats->srvdonecnt = cur_stats->srvdonecnt; + total_stats->busytime = cur_stats->busytime; + +} + /* * Print a running summary of nfs statistics for the experimental client and/or * server. @@ -969,14 +1131,19 @@ * First line printed at top of screen is always cumulative. */ void -exp_sidewaysintpr(u_int interval, int clientOnly, int serverOnly) +exp_sidewaysintpr(u_int interval, int clientOnly, int serverOnly, + int newStats) { struct ext_nfsstats nfsstats, lastst, *ext_nfsstatsp; + struct ext_nfsstats curtotal, lasttotal; + struct timespec ts, lastts; int hdrcnt = 1; ext_nfsstatsp = &lastst; if (nfssvc(NFSSVC_GETSTATS, ext_nfsstatsp) < 0) err(1, "Can't get stats"); + clock_gettime(CLOCK_MONOTONIC, &lastts); + compute_totals(&lasttotal, ext_nfsstatsp); sleep(interval); for (;;) { @@ -983,15 +1150,18 @@ ext_nfsstatsp = &nfsstats; if (nfssvc(NFSSVC_GETSTATS, ext_nfsstatsp) < 0) err(1, "Can't get stats"); + clock_gettime(CLOCK_MONOTONIC, &ts); if (--hdrcnt == 0) { - printhdr(clientOnly, serverOnly); - if (clientOnly && serverOnly) + printhdr(clientOnly, serverOnly, newStats); + if (newStats) + hdrcnt = 20; + else if (clientOnly && serverOnly) hdrcnt = 10; else hdrcnt = 20; } - if (clientOnly) { + if (clientOnly && newStats == 0) { printf("%s %6d %6d %6d %6d %6d %6d %6d %6d", ((clientOnly && serverOnly) ? "Client:" : ""), DELTA(rpccnt[NFSPROC_GETATTR]), @@ -1022,7 +1192,58 @@ } printf("\n"); } - if (serverOnly) { + + if (serverOnly && newStats) { + long double cur_secs, last_secs, etime; + long double mbsec; + long double kb_per_transfer; + long double transfers_per_second; + long double ms_per_transfer; + uint64_t queue_len; + long double busy_pct; + int i; + + cur_secs = ts.tv_sec + + ((long double)ts.tv_nsec / 1000000000); + last_secs = lastts.tv_sec + + ((long double)lastts.tv_nsec / 1000000000); + etime = cur_secs - last_secs; + + compute_totals(&curtotal, &nfsstats); + + for (i = 0; i < NUM_STAT_TYPES; i++) { + compute_new_stats(&nfsstats, &lastst, + STAT_TYPE_TO_NFS(i), etime, &mbsec, + &kb_per_transfer, + &transfers_per_second, + &ms_per_transfer, &queue_len, + &busy_pct); + + if (i == STAT_TYPE_COMMIT) { + if (widemode == 0) + continue; + + printf("%2.0Lf %7.2Lf ", + transfers_per_second, + ms_per_transfer); + } else { + printf("%5.2Lf %5.0Lf %7.2Lf ", + kb_per_transfer, + transfers_per_second, mbsec); + if (widemode) + printf("%5.2Lf ", + ms_per_transfer); + } + } + + compute_new_stats(&curtotal, &lasttotal, 0, etime, + &mbsec, &kb_per_transfer, &transfers_per_second, + &ms_per_transfer, &queue_len, &busy_pct); + + printf("%5.2Lf %5.0Lf %7.2Lf %5.2Lf %3ju %3.0Lf\n", + kb_per_transfer, transfers_per_second, mbsec, + ms_per_transfer, queue_len, busy_pct); + } else if (serverOnly) { printf("%s %6d %6d %6d %6d %6d %6d %6d %6d", ((clientOnly && serverOnly) ? "Server:" : ""), DELTA(srvrpccnt[NFSV4OP_GETATTR]), @@ -1036,7 +1257,9 @@ DELTA(srvrpccnt[NFSV4OP_READDIRPLUS])); printf("\n"); } - lastst = nfsstats; + bcopy(&nfsstats, &lastst, sizeof(lastst)); + bcopy(&curtotal, &lasttotal, sizeof(lasttotal)); + lastts = ts; fflush(stdout); sleep(interval); }