Index: head/lib/libpmc/pmclog.3 =================================================================== --- head/lib/libpmc/pmclog.3 (revision 157143) +++ head/lib/libpmc/pmclog.3 (revision 157144) @@ -1,302 +1,320 @@ -.\" Copyright (c) 2005 Joseph Koshy. All rights reserved. +.\" Copyright (c) 2005-2006 Joseph Koshy. All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" This software is provided by Joseph Koshy ``as is'' and .\" any express or implied warranties, including, but not limited to, the .\" implied warranties of merchantability and fitness for a particular purpose .\" are disclaimed. in no event shall Joseph Koshy be liable .\" for any direct, indirect, incidental, special, exemplary, or consequential .\" damages (including, but not limited to, procurement of substitute goods .\" or services; loss of use, data, or profits; or business interruption) .\" however caused and on any theory of liability, whether in contract, strict .\" liability, or tort (including negligence or otherwise) arising in any way .\" out of the use of this software, even if advised of the possibility of .\" such damage. .\" .\" $FreeBSD$ .\" -.Dd June 1, 2005 +.Dd March 26, 2006 .Os .Dt PMCLOG 3 .Sh NAME .Nm pmclog_open , .Nm pmclog_close , .Nm pmclog_read , .Nm pmclog_feed .Nd parse event log data generated by .Xr hwpmc 4 .Sh LIBRARY .Lb libpmc .Sh SYNOPSIS .In pmclog.h .Ft "void *" .Fn pmclog_open "int fd" .Ft void .Fn pmclog_close "void *cookie" .Ft int .Fn pmclog_read "void *cookie" "struct pmclog_ev *ev" .Ft int .Fn pmclog_feed "void *cookie" "char *data" "int len" .Sh DESCRIPTION These functions provide a way for application programs to extract events from an event stream generated by .Xr hwpmc 4 . .Pp A new event log parser is allocated using .Fn pmclog_open . Argument .Fa fd may be a file descriptor opened for reading if the event stream is present in a file, or the constant .Dv PMCLOG_FD_NONE for an event stream present in memory. This function returns a cookie that is passed into the other functions in this API set. .Pp Function .Fn pmclog_read returns the next available event in the event stream associated with argument .Fa cookie . Argument .Fa ev points to an event descriptor that which will contain the result of a successfully parsed event. .Pp An event descriptor returned by .Fn pmclog_read has the following structure: .Bd -literal struct pmclog_ev { enum pmclog_state pl_state; /* parser state after 'get_event()' */ off_t pl_offset; /* byte offset in stream */ size_t pl_count; /* count of records so far */ struct timespec pl_ts; /* log entry timestamp */ enum pmclog_type pl_type; /* log entry kind */ union { /* log entry data */ - struct pmclog_ev_allocate pl_a; - struct pmclog_ev_proccsw pl_c; - struct pmclog_ev_dropnotify pl_d; - struct pmclog_ev_procexit pl_e; - struct pmclog_ev_initialize pl_i; - struct pmclog_ev_pcsample pl_s; - struct pmclog_ev_pmcattach pl_t; - struct pmclog_ev_userdata pl_u; - struct pmclog_ev_procexec pl_x; + struct pmclog_ev_closelog pl_cl; + struct pmclog_ev_dropnotify pl_d; + struct pmclog_ev_initialize pl_i; + struct pmclog_ev_map_in pl_mi; + struct pmclog_ev_map_out pl_mo; + struct pmclog_ev_pcsample pl_s; + struct pmclog_ev_pmcallocate pl_a; + struct pmclog_ev_pmcattach pl_t; + struct pmclog_ev_pmcdetach pl_d; + struct pmclog_ev_proccsw pl_c; + struct pmclog_ev_procexec pl_x; + struct pmclog_ev_procexit pl_e; + struct pmclog_ev_procfork pl_f; + struct pmclog_ev_sysexit pl_e; + struct pmclog_ev_userdata pl_u; } pl_u; }; .Ed .Pp The current state of the parser is recorded in .Va pl_state . This field can take on the following values: .Bl -tag -width ".Dv PMCLOG_REQUIRE_DATA" .It Dv PMCLOG_EOF (For file based parsers only) An end-of-file condition was encountered on the configured file descriptor. .It Dv PMCLOG_ERROR An error occurred during parsing. .It Dv PMCLOG_OK A complete event record was read into .Fa *ev . .It Dv PMCLOG_REQUIRE_DATA There was insufficient data in the event stream to assemble a complete event record. For memory based parsers, more data can be fed to the parser using function .Fn pmclog_feed . For file based parsers, function .Fn pmclog_read may be retried when data is available on the configured file descriptor. .El .Pp The rest of the event structure is valid only if field .Va pl_state contains .Dv PMCLOG_OK . Field .Va pl_offset contains the offset of the current record in the byte stream. Field .Va pl_count contains the serial number of this event. Field .Va pl_ts contains a timestamp with the system time when the event occurred. Field .Va pl_type denotes the kind of the event returned in argument .Fa *ev and is one of the following: .Bl -tag -width ".Dv PMCLOG_TYPE_PMCALLOCATE" .It Dv PMCLOG_TYPE_CLOSELOG A marker indicating a successful close of a log file. This record will be the last record of a log file. .It Dv PMCLOG_TYPE_DROPNOTIFY A marker indicating that .Xr hwpmc 4 had to drop data due to a resource constraint. .It Dv PMCLOG_TYPE_INITIALIZE An initialization record. This is the first record in a log file. -.It Dv PMCLOG_TYPE_MAPPINGCHANGE -A record describing an address space change for a process. +.It Dv PMCLOG_TYPE_MAP_IN +A record describing the introduction of a mapping to an executable +object by a +.Xr kldload 2 +or +.Xr mmap 2 +system call. +.It Dv PMCLOG_TYPE_MAP_OUT +A record describing the removal of a mapping to an executable +object by a +.Xr kldunload 2 +or +.Xr munmap 2 +system call. .It Dv PMCLOG_TYPE_PCSAMPLE A record containing an instruction pointer sample. .It Dv PMCLOG_TYPE_PMCALLOCATE A record describing a PMC allocation operation. .It Dv PMCLOG_TYPE_PMCATTACH A record describing a PMC attach operation. .It Dv PMCLOG_TYPE_PMCDETACH A record describing a PMC detach operation. .It Dv PMCLOG_TYPE_PROCCSW A record describing a PMC reading at the time of a process context switch. .It Dv PMCLOG_TYPE_PROCEXEC A record describing an .Xr execve 2 by a target process. .It Dv PMCLOG_TYPE_PROCEXIT A record describing the accumulated PMC reading for a process at the time of .Xr _exit 2 . .It Dv PMCLOG_TYPE_PROCFORK A record describing a .Xr fork 2 by a target process. .It Dv PMCLOG_TYPE_SYSEXIT A record describing a process exit, sent to processes owning system-wide sampling PMCs. .It Dv PMCLOG_TYPE_USERDATA A record containing user data. .El .Pp Function .Fn pmclog_feed is used with parsers configured to parse memory based event streams. It is intended to be called when function .Fn pmclog_read indicates the need for more data by a returning .Dv PMCLOG_REQUIRE_DATA in field .Va pl_state of its event structure argument. Argument .Fa data points to the start of a memory buffer containing fresh event data. Argument .Fa len indicates the number of data bytes available. The memory range .Bq Fa data , Fa data No + Fa len must remain valid till the next time .Fn pmclog_read returns an error. It is an error to use .Fn pmclog_feed on a parser configured to parse file data. .Pp Function .Fn pmclog_close releases the internal state allocated by a prior call to .Fn pmclog_open . .Sh RETURN VALUES Function .Fn pmclog_open will return a .No non- Ns Dv NULL value if successful or .Dv NULL otherwise. .Pp Function .Fn pmclog_read will return 0 in case a complete event record was successfully read, or will return \-1 and will set the .Va pl_state field of the event record to the appropriate code in case of an error. .Pp Function .Fn pmclog_feed will return 0 on success or \-1 in case of failure. .Sh EXAMPLES A template for using the log file parsing API is shown below in pseudocode: .Bd -literal void *parser; /* cookie */ struct pmclog_ev ev; /* parsed event */ int fd; /* file descriptor */ fd = open(filename, O_RDONLY); /* open log file */ parser = pmclog_open(fd); /* initialize parser */ if (parser == NULL) --handle an out of memory error--; /* read and parse data */ while (pmclog_read(parser, &ev) == 0) { assert(ev.pl_state == PMCLOG_OK); /* process the event */ switch (ev.pl_type) { case PMCLOG_TYPE_ALLOCATE: --process a pmc allocation record-- break; case PMCLOG_TYPE_PROCCSW: --process a thread context switch record-- break; case PMCLOG_TYPE_PCSAMPLE: --process a PC sample-- break; --and so on-- } } /* examine parser state */ switch (ev.pl_state) { case PMCLOG_EOF: --normal termination-- break; case PMCLOG_ERROR: --look at errno here-- break; case PMCLOG_REQUIRE_DATA: --arrange for more data to be available for parsing-- break; default: assert(0); /*NOTREACHED*/ } pmclog_close(parser); /* cleanup */ .Ed .Sh ERRORS A call to .Fn pmclog_init_parser may fail with any of the errors returned by .Xr malloc 3 . .Pp A call to .Fn pmclog_read for a file based parser may fail with any of the errors returned by .Xr read 2 . .Sh SEE ALSO .Xr read 2 , .Xr malloc 3 , .Xr pmc 3 , .Xr hwpmc 4 , .Xr pmcstat 8 .Sh HISTORY The .Nm pmclog API .Ud It first appeared in .Fx 6.0 . Index: head/lib/libpmc/pmclog.c =================================================================== --- head/lib/libpmc/pmclog.c (revision 157143) +++ head/lib/libpmc/pmclog.c (revision 157144) @@ -1,550 +1,553 @@ /*- - * Copyright (c) 2005 Joseph Koshy + * Copyright (c) 2005-2006 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #define PMCLOG_BUFFER_SIZE 4096 /* * API NOTES * * The pmclog(3) API is oriented towards parsing an event stream in * "realtime", i.e., from an data source that may or may not preserve * record boundaries -- for example when the data source is elsewhere * on a network. The API allows data to be fed into the parser zero * or more bytes at a time. * * The state for a log file parser is maintained in a 'struct * pmclog_parse_state'. Parser invocations are done by calling * 'pmclog_read()'; this function will inform the caller when a * complete event is parsed. * * The parser first assembles a complete log file event in an internal * work area (see "ps_saved" below). Once a complete log file event * is read, the parser then parses it and converts it to an event * descriptor usable by the client. We could possibly avoid this two * step process by directly parsing the input log to set fields in the * event record. However the parser's state machine would get * insanely complicated, and this code is unlikely to be used in * performance critical paths. */ enum pmclog_parser_state { PL_STATE_NEW_RECORD, /* in-between records */ PL_STATE_EXPECTING_HEADER, /* header being read */ PL_STATE_PARTIAL_RECORD, /* header present but not the record */ PL_STATE_ERROR /* parsing error encountered */ }; struct pmclog_parse_state { enum pmclog_parser_state ps_state; enum pmc_cputype ps_arch; /* log file architecture */ uint32_t ps_version; /* hwpmc version */ int ps_initialized; /* whether initialized */ int ps_count; /* count of records processed */ off_t ps_offset; /* stream byte offset */ union pmclog_entry ps_saved; /* saved partial log entry */ int ps_svcount; /* #bytes saved */ int ps_fd; /* active fd or -1 */ char *ps_buffer; /* scratch buffer if fd != -1 */ char *ps_data; /* current parse pointer */ size_t ps_len; /* length of buffered data */ }; #define PMCLOG_HEADER_FROM_SAVED_STATE(PS) \ (* ((uint32_t *) &(PS)->ps_saved)) #define PMCLOG_INITIALIZE_READER(LE,A) LE = (uint32_t *) &(A) #define PMCLOG_READ32(LE,V) do { \ (V) = *(LE)++; \ } while (0) #define PMCLOG_READ64(LE,V) do { \ uint64_t _v; \ _v = (uint64_t) *(LE)++; \ _v |= ((uint64_t) *(LE)++) << 32; \ (V) = _v; \ } while (0) #define PMCLOG_READSTRING(LE,DST,LEN) strlcpy((DST), (char *) (LE), (LEN)) /* * Assemble a log record from '*len' octets starting from address '*data'. * Update 'data' and 'len' to reflect the number of bytes consumed. * * '*data' is potentially an unaligned address and '*len' octets may * not be enough to complete a event record. */ static enum pmclog_parser_state pmclog_get_record(struct pmclog_parse_state *ps, char **data, ssize_t *len) { int avail, copylen, recordsize, used; uint32_t h; const int HEADERSIZE = sizeof(uint32_t); char *src, *dst; if ((avail = *len) <= 0) return (ps->ps_state = PL_STATE_ERROR); src = *data; h = used = 0; if (ps->ps_state == PL_STATE_NEW_RECORD) ps->ps_svcount = 0; dst = (char *) &ps->ps_saved + ps->ps_svcount; switch (ps->ps_state) { case PL_STATE_NEW_RECORD: /* * Transitions: * * Case A: avail < headersize * -> 'expecting header' * * Case B: avail >= headersize * B.1: avail < recordsize * -> 'partial record' * B.2: avail >= recordsize * -> 'new record' */ copylen = avail < HEADERSIZE ? avail : HEADERSIZE; bcopy(src, dst, copylen); ps->ps_svcount = used = copylen; if (copylen < HEADERSIZE) { ps->ps_state = PL_STATE_EXPECTING_HEADER; goto done; } src += copylen; dst += copylen; h = PMCLOG_HEADER_FROM_SAVED_STATE(ps); recordsize = PMCLOG_HEADER_TO_LENGTH(h); if (recordsize <= 0) goto error; if (recordsize <= avail) { /* full record available */ bcopy(src, dst, recordsize - copylen); ps->ps_svcount = used = recordsize; goto done; } /* header + a partial record is available */ bcopy(src, dst, avail - copylen); ps->ps_svcount = used = avail; ps->ps_state = PL_STATE_PARTIAL_RECORD; break; case PL_STATE_EXPECTING_HEADER: /* * Transitions: * * Case C: avail+saved < headersize * -> 'expecting header' * * Case D: avail+saved >= headersize * D.1: avail+saved < recordsize * -> 'partial record' * D.2: avail+saved >= recordsize * -> 'new record' * (see PARTIAL_RECORD handling below) */ if (avail + ps->ps_svcount < HEADERSIZE) { bcopy(src, dst, avail); ps->ps_svcount += avail; used = avail; break; } used = copylen = HEADERSIZE - ps->ps_svcount; bcopy(src, dst, copylen); src += copylen; dst += copylen; avail -= copylen; ps->ps_svcount += copylen; /*FALLTHROUGH*/ case PL_STATE_PARTIAL_RECORD: /* * Transitions: * * Case E: avail+saved < recordsize * -> 'partial record' * * Case F: avail+saved >= recordsize * -> 'new record' */ h = PMCLOG_HEADER_FROM_SAVED_STATE(ps); recordsize = PMCLOG_HEADER_TO_LENGTH(h); if (recordsize <= 0) goto error; if (avail + ps->ps_svcount < recordsize) { copylen = avail; ps->ps_state = PL_STATE_PARTIAL_RECORD; } else { copylen = recordsize - ps->ps_svcount; ps->ps_state = PL_STATE_NEW_RECORD; } bcopy(src, dst, copylen); ps->ps_svcount += copylen; used += copylen; break; default: goto error; } done: *data += used; *len -= used; return ps->ps_state; error: ps->ps_state = PL_STATE_ERROR; return ps->ps_state; } /* * Get an event from the stream pointed to by '*data'. '*len' * indicates the number of bytes available to parse. Arguments * '*data' and '*len' are updated to indicate the number of bytes * consumed. */ static int pmclog_get_event(void *cookie, char **data, ssize_t *len, struct pmclog_ev *ev) { int evlen, pathlen; uint32_t h, *le; enum pmclog_parser_state e; struct pmclog_parse_state *ps; ps = (struct pmclog_parse_state *) cookie; assert(ps->ps_state != PL_STATE_ERROR); if ((e = pmclog_get_record(ps,data,len)) == PL_STATE_ERROR) { ev->pl_state = PMCLOG_ERROR; return -1; } if (e != PL_STATE_NEW_RECORD) { ev->pl_state = PMCLOG_REQUIRE_DATA; return -1; } PMCLOG_INITIALIZE_READER(le, ps->ps_saved); PMCLOG_READ32(le,h); if (!PMCLOG_HEADER_CHECK_MAGIC(h)) { ps->ps_state = PL_STATE_ERROR; ev->pl_state = PMCLOG_ERROR; return -1; } /* copy out the time stamp */ PMCLOG_READ32(le,ev->pl_ts.tv_sec); PMCLOG_READ32(le,ev->pl_ts.tv_nsec); evlen = PMCLOG_HEADER_TO_LENGTH(h); #define PMCLOG_GET_PATHLEN(P,E,TYPE) do { \ (P) = (E) - offsetof(struct TYPE, pl_pathname); \ if ((P) > PATH_MAX || (P) < 0) \ goto error; \ } while (0) switch (ev->pl_type = PMCLOG_HEADER_TO_TYPE(h)) { case PMCLOG_TYPE_CLOSELOG: case PMCLOG_TYPE_DROPNOTIFY: /* nothing to do */ break; case PMCLOG_TYPE_INITIALIZE: PMCLOG_READ32(le,ev->pl_u.pl_i.pl_version); PMCLOG_READ32(le,ev->pl_u.pl_i.pl_arch); ps->ps_version = ev->pl_u.pl_i.pl_version; ps->ps_arch = ev->pl_u.pl_i.pl_arch; ps->ps_initialized = 1; break; - case PMCLOG_TYPE_MAPPINGCHANGE: - PMCLOG_GET_PATHLEN(pathlen,evlen,pmclog_mappingchange); - PMCLOG_READ32(le,ev->pl_u.pl_m.pl_type); - PMCLOG_READADDR(le,ev->pl_u.pl_m.pl_start); - PMCLOG_READADDR(le,ev->pl_u.pl_m.pl_end); - PMCLOG_READ32(le,ev->pl_u.pl_m.pl_pid); - PMCLOG_READSTRING(le, ev->pl_u.pl_m.pl_pathname, pathlen); + case PMCLOG_TYPE_MAP_IN: + PMCLOG_GET_PATHLEN(pathlen,evlen,pmclog_map_in); + PMCLOG_READ32(le,ev->pl_u.pl_mi.pl_pid); + PMCLOG_READADDR(le,ev->pl_u.pl_mi.pl_start); + PMCLOG_READSTRING(le, ev->pl_u.pl_mi.pl_pathname, pathlen); + break; + case PMCLOG_TYPE_MAP_OUT: + PMCLOG_READ32(le,ev->pl_u.pl_mo.pl_pid); + PMCLOG_READADDR(le,ev->pl_u.pl_mo.pl_start); + PMCLOG_READADDR(le,ev->pl_u.pl_mo.pl_end); break; case PMCLOG_TYPE_PCSAMPLE: PMCLOG_READ32(le,ev->pl_u.pl_s.pl_pid); PMCLOG_READADDR(le,ev->pl_u.pl_s.pl_pc); PMCLOG_READ32(le,ev->pl_u.pl_s.pl_pmcid); PMCLOG_READ32(le,ev->pl_u.pl_s.pl_usermode); break; case PMCLOG_TYPE_PMCALLOCATE: PMCLOG_READ32(le,ev->pl_u.pl_a.pl_pmcid); PMCLOG_READ32(le,ev->pl_u.pl_a.pl_event); PMCLOG_READ32(le,ev->pl_u.pl_a.pl_flags); if ((ev->pl_u.pl_a.pl_evname = pmc_name_of_event(ev->pl_u.pl_a.pl_event)) == NULL) goto error; break; case PMCLOG_TYPE_PMCATTACH: PMCLOG_GET_PATHLEN(pathlen,evlen,pmclog_pmcattach); PMCLOG_READ32(le,ev->pl_u.pl_t.pl_pmcid); PMCLOG_READ32(le,ev->pl_u.pl_t.pl_pid); PMCLOG_READSTRING(le,ev->pl_u.pl_t.pl_pathname,pathlen); break; case PMCLOG_TYPE_PMCDETACH: PMCLOG_READ32(le,ev->pl_u.pl_d.pl_pmcid); PMCLOG_READ32(le,ev->pl_u.pl_d.pl_pid); break; case PMCLOG_TYPE_PROCCSW: PMCLOG_READ32(le,ev->pl_u.pl_c.pl_pmcid); PMCLOG_READ64(le,ev->pl_u.pl_c.pl_value); PMCLOG_READ32(le,ev->pl_u.pl_c.pl_pid); break; case PMCLOG_TYPE_PROCEXEC: PMCLOG_GET_PATHLEN(pathlen,evlen,pmclog_procexec); PMCLOG_READ32(le,ev->pl_u.pl_x.pl_pid); PMCLOG_READADDR(le,ev->pl_u.pl_x.pl_entryaddr); PMCLOG_READ32(le,ev->pl_u.pl_x.pl_pmcid); PMCLOG_READSTRING(le,ev->pl_u.pl_x.pl_pathname,pathlen); break; case PMCLOG_TYPE_PROCEXIT: PMCLOG_READ32(le,ev->pl_u.pl_e.pl_pmcid); PMCLOG_READ64(le,ev->pl_u.pl_e.pl_value); PMCLOG_READ32(le,ev->pl_u.pl_e.pl_pid); break; case PMCLOG_TYPE_PROCFORK: PMCLOG_READ32(le,ev->pl_u.pl_f.pl_oldpid); PMCLOG_READ32(le,ev->pl_u.pl_f.pl_newpid); break; case PMCLOG_TYPE_SYSEXIT: PMCLOG_READ32(le,ev->pl_u.pl_se.pl_pid); break; case PMCLOG_TYPE_USERDATA: PMCLOG_READ32(le,ev->pl_u.pl_u.pl_userdata); break; default: /* unknown record type */ ps->ps_state = PL_STATE_ERROR; ev->pl_state = PMCLOG_ERROR; return -1; } ev->pl_offset = (ps->ps_offset += evlen); ev->pl_count = (ps->ps_count += 1); ev->pl_state = PMCLOG_OK; return 0; error: ev->pl_state = PMCLOG_ERROR; ps->ps_state = PL_STATE_ERROR; return -1; } /* * Extract and return the next event from the byte stream. * * Returns 0 and sets the event's state to PMCLOG_OK in case an event * was successfully parsed. Otherwise this function returns -1 and * sets the event's state to one of PMCLOG_REQUIRE_DATA (if more data * is needed) or PMCLOG_EOF (if an EOF was seen) or PMCLOG_ERROR if * a parse error was encountered. */ int pmclog_read(void *cookie, struct pmclog_ev *ev) { int retval; ssize_t nread; struct pmclog_parse_state *ps; ps = (struct pmclog_parse_state *) cookie; if (ps->ps_state == PL_STATE_ERROR) { ev->pl_state = PMCLOG_ERROR; return -1; } /* * If there isn't enough data left for a new event try and get * more data. */ if (ps->ps_len == 0) { ev->pl_state = PMCLOG_REQUIRE_DATA; /* * If we have a valid file descriptor to read from, attempt * to read from that. This read may return with an error, * (which may be EAGAIN or other recoverable error), or * can return EOF. */ if (ps->ps_fd != PMCLOG_FD_NONE) { refill: nread = read(ps->ps_fd, ps->ps_buffer, PMCLOG_BUFFER_SIZE); if (nread <= 0) { if (nread == 0) ev->pl_state = PMCLOG_EOF; else if (errno != EAGAIN) /* not restartable */ ev->pl_state = PMCLOG_ERROR; return -1; } ps->ps_len = nread; ps->ps_data = ps->ps_buffer; } else return -1; } assert(ps->ps_len > 0); /* Retrieve one event from the byte stream. */ retval = pmclog_get_event(ps, &ps->ps_data, &ps->ps_len, ev); /* * If we need more data and we have a configured fd, try read * from it. */ if (retval < 0 && ev->pl_state == PMCLOG_REQUIRE_DATA && ps->ps_fd != -1) { assert(ps->ps_len == 0); goto refill; } return retval; } /* * Feed data to a memory based parser. * * The memory area pointed to by 'data' needs to be valid till the * next error return from pmclog_next_event(). */ int pmclog_feed(void *cookie, char *data, int len) { struct pmclog_parse_state *ps; ps = (struct pmclog_parse_state *) cookie; if (len < 0 || /* invalid length */ ps->ps_buffer || /* called for a file parser */ ps->ps_len != 0) /* unnecessary call */ return -1; ps->ps_data = data; ps->ps_len = len; return 0; } /* * Allocate and initialize parser state. */ void * pmclog_open(int fd) { struct pmclog_parse_state *ps; if ((ps = (struct pmclog_parse_state *) malloc(sizeof(*ps))) == NULL) return NULL; ps->ps_state = PL_STATE_NEW_RECORD; ps->ps_arch = -1; ps->ps_initialized = 0; ps->ps_count = 0; ps->ps_offset = (off_t) 0; bzero(&ps->ps_saved, sizeof(ps->ps_saved)); ps->ps_svcount = 0; ps->ps_fd = fd; ps->ps_data = NULL; ps->ps_buffer = NULL; ps->ps_len = 0; /* allocate space for a work area */ if (ps->ps_fd != PMCLOG_FD_NONE) { if ((ps->ps_buffer = malloc(PMCLOG_BUFFER_SIZE)) == NULL) return NULL; } return ps; } /* * Free up parser state. */ void pmclog_close(void *cookie) { struct pmclog_parse_state *ps; ps = (struct pmclog_parse_state *) cookie; if (ps->ps_buffer) free(ps->ps_buffer); free(ps); } Index: head/lib/libpmc/pmclog.h =================================================================== --- head/lib/libpmc/pmclog.h (revision 157143) +++ head/lib/libpmc/pmclog.h (revision 157144) @@ -1,149 +1,154 @@ /*- - * Copyright (c) 2005 Joseph Koshy + * Copyright (c) 2005-2006 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _PMCLOG_H_ #define _PMCLOG_H_ #include enum pmclog_state { PMCLOG_OK, PMCLOG_EOF, PMCLOG_REQUIRE_DATA, PMCLOG_ERROR }; struct pmclog_ev_dropnotify { }; struct pmclog_ev_closelog { }; struct pmclog_ev_initialize { uint32_t pl_version; uint32_t pl_arch; }; -struct pmclog_ev_mappingchange { - uint32_t pl_type; +struct pmclog_ev_map_in { pid_t pl_pid; uintfptr_t pl_start; - uintfptr_t pl_end; char pl_pathname[PATH_MAX]; }; +struct pmclog_ev_map_out { + pid_t pl_pid; + uintfptr_t pl_start; + uintfptr_t pl_end; +}; + struct pmclog_ev_pcsample { uintfptr_t pl_pc; pid_t pl_pid; pmc_id_t pl_pmcid; uint32_t pl_usermode; }; struct pmclog_ev_pmcallocate { uint32_t pl_event; const char * pl_evname; uint32_t pl_flags; pmc_id_t pl_pmcid; }; struct pmclog_ev_pmcattach { pmc_id_t pl_pmcid; pid_t pl_pid; char pl_pathname[PATH_MAX]; }; struct pmclog_ev_pmcdetach { pmc_id_t pl_pmcid; pid_t pl_pid; }; struct pmclog_ev_proccsw { pid_t pl_pid; pmc_id_t pl_pmcid; pmc_value_t pl_value; }; struct pmclog_ev_procexec { pid_t pl_pid; pmc_id_t pl_pmcid; uintfptr_t pl_entryaddr; char pl_pathname[PATH_MAX]; }; struct pmclog_ev_procexit { uint32_t pl_pid; pmc_id_t pl_pmcid; pmc_value_t pl_value; }; struct pmclog_ev_procfork { pid_t pl_oldpid; pid_t pl_newpid; }; struct pmclog_ev_sysexit { pid_t pl_pid; }; struct pmclog_ev_userdata { uint32_t pl_userdata; }; struct pmclog_ev { enum pmclog_state pl_state; /* state after 'get_event()' */ off_t pl_offset; /* byte offset in stream */ size_t pl_count; /* count of records so far */ struct timespec pl_ts; /* log entry timestamp */ enum pmclog_type pl_type; /* type of log entry */ union { /* log entry data */ struct pmclog_ev_closelog pl_cl; struct pmclog_ev_dropnotify pl_dn; struct pmclog_ev_initialize pl_i; - struct pmclog_ev_mappingchange pl_m; + struct pmclog_ev_map_in pl_mi; + struct pmclog_ev_map_out pl_mo; struct pmclog_ev_pcsample pl_s; struct pmclog_ev_pmcallocate pl_a; struct pmclog_ev_pmcattach pl_t; struct pmclog_ev_pmcdetach pl_d; struct pmclog_ev_proccsw pl_c; struct pmclog_ev_procexec pl_x; struct pmclog_ev_procexit pl_e; struct pmclog_ev_procfork pl_f; struct pmclog_ev_sysexit pl_se; struct pmclog_ev_userdata pl_u; } pl_u; }; #define PMCLOG_FD_NONE (-1) void *pmclog_open(int _fd); int pmclog_feed(void *_cookie, char *_data, int _len); int pmclog_read(void *_cookie, struct pmclog_ev *_ev); void pmclog_close(void *_cookie); #endif Index: head/sys/dev/hwpmc/hwpmc_logging.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_logging.c (revision 157143) +++ head/sys/dev/hwpmc/hwpmc_logging.c (revision 157144) @@ -1,985 +1,998 @@ /*- * Copyright (c) 2005 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * Logging code for hwpmc(4) */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Sysctl tunables */ SYSCTL_DECL(_kern_hwpmc); /* * kern.hwpmc.logbuffersize -- size of the per-cpu owner buffers. */ static int pmclog_buffer_size = PMC_LOG_BUFFER_SIZE; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "logbuffersize", &pmclog_buffer_size); SYSCTL_INT(_kern_hwpmc, OID_AUTO, logbuffersize, CTLFLAG_TUN|CTLFLAG_RD, &pmclog_buffer_size, 0, "size of log buffers in kilobytes"); /* * kern.hwpmc.nbuffer -- number of global log buffers */ static int pmc_nlogbuffers = PMC_NLOGBUFFERS; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nbuffers", &pmc_nlogbuffers); SYSCTL_INT(_kern_hwpmc, OID_AUTO, nbuffers, CTLFLAG_TUN|CTLFLAG_RD, &pmc_nlogbuffers, 0, "number of global log buffers"); /* * Global log buffer list and associated spin lock. */ TAILQ_HEAD(, pmclog_buffer) pmc_bufferlist = TAILQ_HEAD_INITIALIZER(pmc_bufferlist); static struct mtx pmc_bufferlist_mtx; /* spin lock */ static struct mtx pmc_kthread_mtx; /* sleep lock */ #define PMCLOG_INIT_BUFFER_DESCRIPTOR(D) do { \ const int __roundup = roundup(sizeof(*D), \ sizeof(uint32_t)); \ (D)->plb_fence = ((char *) (D)) + \ 1024*pmclog_buffer_size; \ (D)->plb_base = (D)->plb_ptr = ((char *) (D)) + \ __roundup; \ } while (0) /* * Log file record constructors. */ #define _PMCLOG_TO_HEADER(T,L) \ ((PMCLOG_HEADER_MAGIC << 24) | \ (PMCLOG_TYPE_ ## T << 16) | \ ((L) & 0xFFFF)) /* reserve LEN bytes of space and initialize the entry header */ #define _PMCLOG_RESERVE(PO,TYPE,LEN,ACTION) do { \ uint32_t *_le; \ int _len = roundup((LEN), sizeof(uint32_t)); \ if ((_le = pmclog_reserve((PO), _len)) == NULL) { \ ACTION; \ } \ *_le = _PMCLOG_TO_HEADER(TYPE,_len); \ _le += 3 /* skip over timestamp */ #define PMCLOG_RESERVE(P,T,L) _PMCLOG_RESERVE(P,T,L,return) #define PMCLOG_RESERVE_WITH_ERROR(P,T,L) _PMCLOG_RESERVE(P,T,L, \ error=ENOMEM;goto error) #define PMCLOG_EMIT32(V) do { *_le++ = (V); } while (0) #define PMCLOG_EMIT64(V) do { \ *_le++ = (uint32_t) ((V) & 0xFFFFFFFF); \ *_le++ = (uint32_t) (((V) >> 32) & 0xFFFFFFFF); \ } while (0) /* Emit a string. Caution: does NOT update _le, so needs to be last */ #define PMCLOG_EMITSTRING(S,L) do { bcopy((S), _le, (L)); } while (0) #define PMCLOG_DESPATCH(PO) \ pmclog_release((PO)); \ } while (0) /* * Assertions about the log file format. */ CTASSERT(sizeof(struct pmclog_closelog) == 3*4); CTASSERT(sizeof(struct pmclog_dropnotify) == 3*4); -CTASSERT(sizeof(struct pmclog_mappingchange) == PATH_MAX + - 5*4 + 2*sizeof(uintfptr_t)); -CTASSERT(offsetof(struct pmclog_mappingchange,pl_pathname) == - 5*4 + 2*sizeof(uintfptr_t)); +CTASSERT(sizeof(struct pmclog_map_in) == PATH_MAX + + 4*4 + sizeof(uintfptr_t)); +CTASSERT(offsetof(struct pmclog_map_in,pl_pathname) == + 4*4 + sizeof(uintfptr_t)); +CTASSERT(sizeof(struct pmclog_map_out) == 4*4 + 2*sizeof(uintfptr_t)); CTASSERT(sizeof(struct pmclog_pcsample) == 6*4 + sizeof(uintfptr_t)); CTASSERT(sizeof(struct pmclog_pmcallocate) == 6*4); CTASSERT(sizeof(struct pmclog_pmcattach) == 5*4 + PATH_MAX); CTASSERT(offsetof(struct pmclog_pmcattach,pl_pathname) == 5*4); CTASSERT(sizeof(struct pmclog_pmcdetach) == 5*4); CTASSERT(sizeof(struct pmclog_proccsw) == 5*4 + 8); CTASSERT(sizeof(struct pmclog_procexec) == 5*4 + PATH_MAX + sizeof(uintfptr_t)); CTASSERT(offsetof(struct pmclog_procexec,pl_pathname) == 5*4 + sizeof(uintfptr_t)); CTASSERT(sizeof(struct pmclog_procexit) == 5*4 + 8); CTASSERT(sizeof(struct pmclog_procfork) == 5*4); CTASSERT(sizeof(struct pmclog_sysexit) == 4*4); CTASSERT(sizeof(struct pmclog_userdata) == 4*4); /* * Log buffer structure */ struct pmclog_buffer { TAILQ_ENTRY(pmclog_buffer) plb_next; char *plb_base; char *plb_ptr; char *plb_fence; }; /* * Prototypes */ static int pmclog_get_buffer(struct pmc_owner *po); static void pmclog_loop(void *arg); static void pmclog_release(struct pmc_owner *po); static uint32_t *pmclog_reserve(struct pmc_owner *po, int length); static void pmclog_schedule_io(struct pmc_owner *po); static void pmclog_stop_kthread(struct pmc_owner *po); /* * Helper functions */ /* * Get a log buffer */ static int pmclog_get_buffer(struct pmc_owner *po) { struct pmclog_buffer *plb; mtx_assert(&po->po_mtx, MA_OWNED); KASSERT(po->po_curbuf == NULL, ("[pmc,%d] po=%p current buffer still valid", __LINE__, po)); mtx_lock_spin(&pmc_bufferlist_mtx); if ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL) TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next); mtx_unlock_spin(&pmc_bufferlist_mtx); PMCDBG(LOG,GTB,1, "po=%p plb=%p", po, plb); #ifdef DEBUG if (plb) KASSERT(plb->plb_ptr == plb->plb_base && plb->plb_base < plb->plb_fence, ("[pmc,%d] po=%p buffer invariants: ptr=%p " "base=%p fence=%p", __LINE__, po, plb->plb_ptr, plb->plb_base, plb->plb_fence)); #endif po->po_curbuf = plb; /* update stats */ atomic_add_int(&pmc_stats.pm_buffer_requests, 1); if (plb == NULL) atomic_add_int(&pmc_stats.pm_buffer_requests_failed, 1); return plb ? 0 : ENOMEM; } /* * Log handler loop. * * This function is executed by each pmc owner's helper thread. */ static void pmclog_loop(void *arg) { int error; struct pmc_owner *po; struct pmclog_buffer *lb; struct ucred *ownercred; struct ucred *mycred; struct thread *td; struct uio auio; struct iovec aiov; size_t nbytes; po = (struct pmc_owner *) arg; td = curthread; mycred = td->td_ucred; PROC_LOCK(po->po_owner); ownercred = crhold(po->po_owner->p_ucred); PROC_UNLOCK(po->po_owner); PMCDBG(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread); KASSERT(po->po_kthread == curthread->td_proc, ("[pmc,%d] proc mismatch po=%p po/kt=%p curproc=%p", __LINE__, po, po->po_kthread, curthread->td_proc)); lb = NULL; /* * Loop waiting for I/O requests to be added to the owner * struct's queue. The loop is exited when the log file * is deconfigured. */ mtx_lock(&pmc_kthread_mtx); for (;;) { /* check if we've been asked to exit */ if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) break; if (lb == NULL) { /* look for a fresh buffer to write */ mtx_lock_spin(&po->po_mtx); if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) { mtx_unlock_spin(&po->po_mtx); /* wakeup any processes waiting for a FLUSH */ if (po->po_flags & PMC_PO_IN_FLUSH) { po->po_flags &= ~PMC_PO_IN_FLUSH; wakeup_one(po->po_kthread); } (void) msleep(po, &pmc_kthread_mtx, PWAIT, "pmcloop", 0); continue; } TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next); mtx_unlock_spin(&po->po_mtx); } mtx_unlock(&pmc_kthread_mtx); /* process the request */ PMCDBG(LOG,WRI,2, "po=%p base=%p ptr=%p", po, lb->plb_base, lb->plb_ptr); /* change our thread's credentials before issuing the I/O */ aiov.iov_base = lb->plb_base; aiov.iov_len = nbytes = lb->plb_ptr - lb->plb_base; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = -1; auio.uio_resid = nbytes; auio.uio_rw = UIO_WRITE; auio.uio_segflg = UIO_SYSSPACE; auio.uio_td = td; /* switch thread credentials -- see kern_ktrace.c */ td->td_ucred = ownercred; error = fo_write(po->po_file, &auio, ownercred, 0, td); td->td_ucred = mycred; mtx_lock(&pmc_kthread_mtx); if (error) { /* XXX some errors are recoverable */ /* XXX also check for SIGPIPE if a socket */ /* send a SIGIO to the owner and exit */ PROC_LOCK(po->po_owner); psignal(po->po_owner, SIGIO); PROC_UNLOCK(po->po_owner); po->po_error = error; /* save for flush log */ PMCDBG(LOG,WRI,2, "po=%p error=%d", po, error); break; } /* put the used buffer back into the global pool */ PMCLOG_INIT_BUFFER_DESCRIPTOR(lb); mtx_lock_spin(&pmc_bufferlist_mtx); TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next); mtx_unlock_spin(&pmc_bufferlist_mtx); lb = NULL; } po->po_kthread = NULL; mtx_unlock(&pmc_kthread_mtx); /* return the current I/O buffer to the global pool */ if (lb) { PMCLOG_INIT_BUFFER_DESCRIPTOR(lb); mtx_lock_spin(&pmc_bufferlist_mtx); TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next); mtx_unlock_spin(&pmc_bufferlist_mtx); } /* * Exit this thread, signalling the waiter */ crfree(ownercred); kthread_exit(0); } /* * Release and log entry and schedule an I/O if needed. */ static void pmclog_release(struct pmc_owner *po) { KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base, ("[pmc,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base)); KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, ("[pmc,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence)); /* schedule an I/O if we've filled a buffer */ if (po->po_curbuf->plb_ptr >= po->po_curbuf->plb_fence) pmclog_schedule_io(po); mtx_unlock_spin(&po->po_mtx); PMCDBG(LOG,REL,1, "po=%p", po); } /* * Attempt to reserve 'length' bytes of space in an owner's log * buffer. The function returns a pointer to 'length' bytes of space * if there was enough space or returns NULL if no space was * available. Non-null returns do so with the po mutex locked. The * caller must invoke pmclog_release() on the pmc owner structure * when done. */ static uint32_t * pmclog_reserve(struct pmc_owner *po, int length) { uintptr_t newptr, oldptr; uint32_t *lh; struct timespec ts; PMCDBG(LOG,ALL,1, "po=%p len=%d", po, length); KASSERT(length % sizeof(uint32_t) == 0, ("[pmclog,%d] length not a multiple of word size", __LINE__)); mtx_lock_spin(&po->po_mtx); if (po->po_curbuf == NULL) if (pmclog_get_buffer(po) != 0) { mtx_unlock_spin(&po->po_mtx); return NULL; } KASSERT(po->po_curbuf != NULL, ("[pmc,%d] po=%p no current buffer", __LINE__, po)); KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base && po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, ("[pmc,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p", __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base, po->po_curbuf->plb_fence)); oldptr = (uintptr_t) po->po_curbuf->plb_ptr; newptr = oldptr + length; KASSERT(oldptr != (uintptr_t) NULL, ("[pmc,%d] po=%p Null log buffer pointer", __LINE__, po)); /* * If we have space in the current buffer, return a pointer to * available space with the PO structure locked. */ if (newptr <= (uintptr_t) po->po_curbuf->plb_fence) { po->po_curbuf->plb_ptr = (char *) newptr; goto done; } /* * Otherwise, schedule the current buffer for output and get a * fresh buffer. */ pmclog_schedule_io(po); if (pmclog_get_buffer(po) != 0) { mtx_unlock_spin(&po->po_mtx); return NULL; } KASSERT(po->po_curbuf != NULL, ("[pmc,%d] po=%p no current buffer", __LINE__, po)); KASSERT(po->po_curbuf->plb_ptr != NULL, ("[pmc,%d] null return from pmc_get_log_buffer", __LINE__)); KASSERT(po->po_curbuf->plb_ptr == po->po_curbuf->plb_base && po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, ("[pmc,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p", __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base, po->po_curbuf->plb_fence)); oldptr = (uintptr_t) po->po_curbuf->plb_ptr; done: lh = (uint32_t *) oldptr; lh++; /* skip header */ getnanotime(&ts); /* fill in the timestamp */ *lh++ = ts.tv_sec & 0xFFFFFFFF; *lh++ = ts.tv_nsec & 0xFFFFFFF; return (uint32_t *) oldptr; } /* * Schedule an I/O. * * Transfer the current buffer to the helper kthread. */ static void pmclog_schedule_io(struct pmc_owner *po) { KASSERT(po->po_curbuf != NULL, ("[pmc,%d] schedule_io with null buffer po=%p", __LINE__, po)); KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base, ("[pmc,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base)); KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence, ("[pmc,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence)); PMCDBG(LOG,SIO, 1, "po=%p", po); mtx_assert(&po->po_mtx, MA_OWNED); /* * Add the current buffer to the tail of the buffer list and * wakeup the helper. */ TAILQ_INSERT_TAIL(&po->po_logbuffers, po->po_curbuf, plb_next); po->po_curbuf = NULL; wakeup_one(po); } /* * Stop the helper kthread. */ static void pmclog_stop_kthread(struct pmc_owner *po) { /* * Unset flag, wakeup the helper thread, * wait for it to exit */ mtx_assert(&pmc_kthread_mtx, MA_OWNED); po->po_flags &= ~PMC_PO_OWNS_LOGFILE; wakeup_one(po); if (po->po_kthread) msleep(po->po_kthread, &pmc_kthread_mtx, PPAUSE, "pmckstp", 0); } /* * Public functions */ /* * Configure a log file for pmc owner 'po'. * * Parameter 'logfd' is a file handle referencing an open file in the * owner process. This file needs to have been opened for writing. */ int pmclog_configure_log(struct pmc_owner *po, int logfd) { int error; struct proc *p; PMCDBG(LOG,CFG,1, "config po=%p logfd=%d", po, logfd); p = po->po_owner; /* return EBUSY if a log file was already present */ if (po->po_flags & PMC_PO_OWNS_LOGFILE) return EBUSY; KASSERT(po->po_kthread == NULL, ("[pmc,%d] po=%p kthread (%p) already present", __LINE__, po, po->po_kthread)); KASSERT(po->po_file == NULL, ("[pmc,%d] po=%p file (%p) already present", __LINE__, po, po->po_file)); /* get a reference to the file state */ error = fget_write(curthread, logfd, &po->po_file); if (error) goto error; /* mark process as owning a log file */ po->po_flags |= PMC_PO_OWNS_LOGFILE; error = kthread_create(pmclog_loop, po, &po->po_kthread, RFHIGHPID, 0, "hwpmc: proc(%d)", p->p_pid); if (error) goto error; /* mark process as using HWPMCs */ PROC_LOCK(p); p->p_flag |= P_HWPMC; PROC_UNLOCK(p); /* create a log initialization entry */ PMCLOG_RESERVE_WITH_ERROR(po, INITIALIZE, sizeof(struct pmclog_initialize)); PMCLOG_EMIT32(PMC_VERSION); PMCLOG_EMIT32(md->pmd_cputype); PMCLOG_DESPATCH(po); return 0; error: /* shutdown the thread */ mtx_lock(&pmc_kthread_mtx); if (po->po_kthread) pmclog_stop_kthread(po); mtx_unlock(&pmc_kthread_mtx); KASSERT(po->po_kthread == NULL, ("[pmc,%d] po=%p kthread not stopped", __LINE__, po)); if (po->po_file) (void) fdrop(po->po_file, curthread); po->po_file = NULL; /* clear file and error state */ po->po_error = 0; return error; } /* * De-configure a log file. This will throw away any buffers queued * for this owner process. */ int pmclog_deconfigure_log(struct pmc_owner *po) { int error; struct pmclog_buffer *lb; PMCDBG(LOG,CFG,1, "de-config po=%p", po); if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) return EINVAL; KASSERT(po->po_sscount == 0, ("[pmc,%d] po=%p still owning SS PMCs", __LINE__, po)); KASSERT(po->po_file != NULL, ("[pmc,%d] po=%p no log file", __LINE__, po)); /* stop the kthread, this will reset the 'OWNS_LOGFILE' flag */ mtx_lock(&pmc_kthread_mtx); if (po->po_kthread) pmclog_stop_kthread(po); mtx_unlock(&pmc_kthread_mtx); KASSERT(po->po_kthread == NULL, ("[pmc,%d] po=%p kthread not stopped", __LINE__, po)); /* return all queued log buffers to the global pool */ while ((lb = TAILQ_FIRST(&po->po_logbuffers)) != NULL) { TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next); PMCLOG_INIT_BUFFER_DESCRIPTOR(lb); mtx_lock_spin(&pmc_bufferlist_mtx); TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next); mtx_unlock_spin(&pmc_bufferlist_mtx); } /* return the 'current' buffer to the global pool */ if ((lb = po->po_curbuf) != NULL) { PMCLOG_INIT_BUFFER_DESCRIPTOR(lb); mtx_lock_spin(&pmc_bufferlist_mtx); TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next); mtx_unlock_spin(&pmc_bufferlist_mtx); } /* drop a reference to the fd */ error = fdrop(po->po_file, curthread); po->po_file = NULL; po->po_error = 0; return error; } /* * Flush a process' log buffer. */ int pmclog_flush(struct pmc_owner *po) { int error, has_pending_buffers; PMCDBG(LOG,FLS,1, "po=%p", po); /* * If there is a pending error recorded by the logger thread, * return that. */ if (po->po_error) return po->po_error; error = 0; /* * Check that we do have an active log file. */ mtx_lock(&pmc_kthread_mtx); if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) { error = EINVAL; goto error; } /* * Schedule the current buffer if any. */ mtx_lock_spin(&po->po_mtx); if (po->po_curbuf) pmclog_schedule_io(po); has_pending_buffers = !TAILQ_EMPTY(&po->po_logbuffers); mtx_unlock_spin(&po->po_mtx); if (has_pending_buffers) { po->po_flags |= PMC_PO_IN_FLUSH; /* ask for a wakeup */ error = msleep(po->po_kthread, &pmc_kthread_mtx, PWAIT, "pmcflush", 0); } error: mtx_unlock(&pmc_kthread_mtx); return error; } /* * Send a 'close log' event to the log file. */ void pmclog_process_closelog(struct pmc_owner *po) { PMCLOG_RESERVE(po,CLOSELOG,sizeof(struct pmclog_closelog)); PMCLOG_DESPATCH(po); } void pmclog_process_dropnotify(struct pmc_owner *po) { PMCLOG_RESERVE(po,DROPNOTIFY,sizeof(struct pmclog_dropnotify)); PMCLOG_DESPATCH(po); } void -pmclog_process_mappingchange(struct pmc_owner *po, pid_t pid, int type, - uintfptr_t start, uintfptr_t end, char *path) +pmclog_process_map_in(struct pmc_owner *po, pid_t pid, uintfptr_t start, + const char *path) { int pathlen, recordlen; + KASSERT(path != NULL, ("[pmclog,%d] map-in, null path", __LINE__)); + pathlen = strlen(path) + 1; /* #bytes for path name */ - recordlen = offsetof(struct pmclog_mappingchange, pl_pathname) + + recordlen = offsetof(struct pmclog_map_in, pl_pathname) + pathlen; - PMCLOG_RESERVE(po,MAPPINGCHANGE,recordlen); - PMCLOG_EMIT32(type); - PMCLOG_EMITADDR(start); - PMCLOG_EMITADDR(end); + PMCLOG_RESERVE(po, MAP_IN, recordlen); PMCLOG_EMIT32(pid); + PMCLOG_EMITADDR(start); PMCLOG_EMITSTRING(path,pathlen); PMCLOG_DESPATCH(po); } +void +pmclog_process_map_out(struct pmc_owner *po, pid_t pid, uintfptr_t start, + uintfptr_t end) +{ + KASSERT(start <= end, ("[pmclog,%d] start > end", __LINE__)); + + PMCLOG_RESERVE(po, MAP_OUT, sizeof(struct pmclog_map_out)); + PMCLOG_EMIT32(pid); + PMCLOG_EMITADDR(start); + PMCLOG_EMITADDR(end); + PMCLOG_DESPATCH(po); +} void pmclog_process_pcsample(struct pmc *pm, struct pmc_sample *ps) { struct pmc_owner *po; PMCDBG(LOG,SAM,1,"pm=%p pid=%d pc=%p", pm, ps->ps_pid, (void *) ps->ps_pc); po = pm->pm_owner; PMCLOG_RESERVE(po, PCSAMPLE, sizeof(struct pmclog_pcsample)); PMCLOG_EMIT32(ps->ps_pid); PMCLOG_EMITADDR(ps->ps_pc); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT32(ps->ps_usermode); PMCLOG_DESPATCH(po); } void pmclog_process_pmcallocate(struct pmc *pm) { struct pmc_owner *po; po = pm->pm_owner; PMCDBG(LOG,ALL,1, "pm=%p", pm); PMCLOG_RESERVE(po, PMCALLOCATE, sizeof(struct pmclog_pmcallocate)); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT32(pm->pm_event); PMCLOG_EMIT32(pm->pm_flags); PMCLOG_DESPATCH(po); } void pmclog_process_pmcattach(struct pmc *pm, pid_t pid, char *path) { int pathlen, recordlen; struct pmc_owner *po; PMCDBG(LOG,ATT,1,"pm=%p pid=%d", pm, pid); po = pm->pm_owner; pathlen = strlen(path) + 1; /* #bytes for the string */ recordlen = offsetof(struct pmclog_pmcattach, pl_pathname) + pathlen; PMCLOG_RESERVE(po, PMCATTACH, recordlen); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT32(pid); PMCLOG_EMITSTRING(path, pathlen); PMCLOG_DESPATCH(po); } void pmclog_process_pmcdetach(struct pmc *pm, pid_t pid) { struct pmc_owner *po; PMCDBG(LOG,ATT,1,"!pm=%p pid=%d", pm, pid); po = pm->pm_owner; PMCLOG_RESERVE(po, PMCDETACH, sizeof(struct pmclog_pmcdetach)); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT32(pid); PMCLOG_DESPATCH(po); } /* * Log a context switch event to the log file. */ void pmclog_process_proccsw(struct pmc *pm, struct pmc_process *pp, pmc_value_t v) { struct pmc_owner *po; KASSERT(pm->pm_flags & PMC_F_LOG_PROCCSW, ("[pmclog,%d] log-process-csw called gratuitously", __LINE__)); PMCDBG(LOG,SWO,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid, v); po = pm->pm_owner; PMCLOG_RESERVE(po, PROCCSW, sizeof(struct pmclog_proccsw)); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT64(v); PMCLOG_EMIT32(pp->pp_proc->p_pid); PMCLOG_DESPATCH(po); } void pmclog_process_procexec(struct pmc_owner *po, pmc_id_t pmid, pid_t pid, uintfptr_t startaddr, char *path) { int pathlen, recordlen; PMCDBG(LOG,EXC,1,"po=%p pid=%d path=\"%s\"", po, pid, path); pathlen = strlen(path) + 1; /* #bytes for the path */ recordlen = offsetof(struct pmclog_procexec, pl_pathname) + pathlen; PMCLOG_RESERVE(po, PROCEXEC, recordlen); PMCLOG_EMIT32(pid); PMCLOG_EMITADDR(startaddr); PMCLOG_EMIT32(pmid); PMCLOG_EMITSTRING(path,pathlen); PMCLOG_DESPATCH(po); } /* * Log a process exit event (and accumulated pmc value) to the log file. */ void pmclog_process_procexit(struct pmc *pm, struct pmc_process *pp) { int ri; struct pmc_owner *po; ri = PMC_TO_ROWINDEX(pm); PMCDBG(LOG,EXT,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid, pp->pp_pmcs[ri].pp_pmcval); po = pm->pm_owner; PMCLOG_RESERVE(po, PROCEXIT, sizeof(struct pmclog_procexit)); PMCLOG_EMIT32(pm->pm_id); PMCLOG_EMIT64(pp->pp_pmcs[ri].pp_pmcval); PMCLOG_EMIT32(pp->pp_proc->p_pid); PMCLOG_DESPATCH(po); } /* * Log a fork event. */ void pmclog_process_procfork(struct pmc_owner *po, pid_t oldpid, pid_t newpid) { PMCLOG_RESERVE(po, PROCFORK, sizeof(struct pmclog_procfork)); PMCLOG_EMIT32(oldpid); PMCLOG_EMIT32(newpid); PMCLOG_DESPATCH(po); } /* * Log a process exit event of the form suitable for system-wide PMCs. */ void pmclog_process_sysexit(struct pmc_owner *po, pid_t pid) { PMCLOG_RESERVE(po, SYSEXIT, sizeof(struct pmclog_sysexit)); PMCLOG_EMIT32(pid); PMCLOG_DESPATCH(po); } /* * Write a user log entry. */ int pmclog_process_userlog(struct pmc_owner *po, struct pmc_op_writelog *wl) { int error; PMCDBG(LOG,WRI,1, "writelog po=%p ud=0x%x", po, wl->pm_userdata); error = 0; PMCLOG_RESERVE_WITH_ERROR(po, USERDATA, sizeof(struct pmclog_userdata)); PMCLOG_EMIT32(wl->pm_userdata); PMCLOG_DESPATCH(po); error: return error; } /* * Initialization. * * Create a pool of log buffers and initialize mutexes. */ void pmclog_initialize() { int n; struct pmclog_buffer *plb; if (pmclog_buffer_size <= 0) { (void) printf("hwpmc: tunable logbuffersize=%d must be greater " "than zero.\n", pmclog_buffer_size); pmclog_buffer_size = PMC_LOG_BUFFER_SIZE; } if (pmc_nlogbuffers <= 0) { (void) printf("hwpmc: tunable nlogbuffers=%d must be greater " "than zero.\n", pmc_nlogbuffers); pmc_nlogbuffers = PMC_NLOGBUFFERS; } /* create global pool of log buffers */ for (n = 0; n < pmc_nlogbuffers; n++) { MALLOC(plb, struct pmclog_buffer *, 1024 * pmclog_buffer_size, M_PMC, M_ZERO|M_WAITOK); PMCLOG_INIT_BUFFER_DESCRIPTOR(plb); TAILQ_INSERT_HEAD(&pmc_bufferlist, plb, plb_next); } mtx_init(&pmc_bufferlist_mtx, "pmc-buffer-list", "pmc", MTX_SPIN); mtx_init(&pmc_kthread_mtx, "pmc-kthread", "pmc", MTX_DEF); } /* * Shutdown logging. * * Destroy mutexes and release memory back the to free pool. */ void pmclog_shutdown() { struct pmclog_buffer *plb; mtx_destroy(&pmc_kthread_mtx); mtx_destroy(&pmc_bufferlist_mtx); while ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL) { TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next); FREE(plb, M_PMC); } } Index: head/sys/dev/hwpmc/hwpmc_mod.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_mod.c (revision 157143) +++ head/sys/dev/hwpmc/hwpmc_mod.c (revision 157144) @@ -1,4241 +1,4401 @@ /*- - * Copyright (c) 2003-2005 Joseph Koshy + * Copyright (c) 2003-2006 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include /* needs to be after */ + #include #include /* * Types */ enum pmc_flags { PMC_FLAG_NONE = 0x00, /* do nothing */ PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */ PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */ }; /* * The offset in sysent where the syscall is allocated. */ static int pmc_syscall_num = NO_SYSCALL; struct pmc_cpu **pmc_pcpu; /* per-cpu state */ pmc_value_t *pmc_pcpu_saved; /* saved PMC values: CSW handling */ #define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)] struct mtx_pool *pmc_mtxpool; static int *pmc_pmcdisp; /* PMC row dispositions */ #define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0) #define PMC_ROW_DISP_IS_THREAD(R) (pmc_pmcdisp[(R)] > 0) #define PMC_ROW_DISP_IS_STANDALONE(R) (pmc_pmcdisp[(R)] < 0) #define PMC_MARK_ROW_FREE(R) do { \ pmc_pmcdisp[(R)] = 0; \ } while (0) #define PMC_MARK_ROW_STANDALONE(R) do { \ KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ atomic_add_int(&pmc_pmcdisp[(R)], -1); \ KASSERT(pmc_pmcdisp[(R)] >= (-mp_ncpus), ("[pmc,%d] row " \ "disposition error", __LINE__)); \ } while (0) #define PMC_UNMARK_ROW_STANDALONE(R) do { \ atomic_add_int(&pmc_pmcdisp[(R)], 1); \ KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ } while (0) #define PMC_MARK_ROW_THREAD(R) do { \ KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ atomic_add_int(&pmc_pmcdisp[(R)], 1); \ } while (0) #define PMC_UNMARK_ROW_THREAD(R) do { \ atomic_add_int(&pmc_pmcdisp[(R)], -1); \ KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ } while (0) /* various event handlers */ static eventhandler_tag pmc_exit_tag, pmc_fork_tag; /* Module statistics */ struct pmc_op_getdriverstats pmc_stats; /* Machine/processor dependent operations */ struct pmc_mdep *md; /* * Hash tables mapping owner processes and target threads to PMCs. */ struct mtx pmc_processhash_mtx; /* spin mutex */ static u_long pmc_processhashmask; static LIST_HEAD(pmc_processhash, pmc_process) *pmc_processhash; /* * Hash table of PMC owner descriptors. This table is protected by * the shared PMC "sx" lock. */ static u_long pmc_ownerhashmask; static LIST_HEAD(pmc_ownerhash, pmc_owner) *pmc_ownerhash; /* * List of PMC owners with system-wide sampling PMCs. */ static LIST_HEAD(, pmc_owner) pmc_ss_owners; /* * Prototypes */ #ifdef DEBUG static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS); static int pmc_debugflags_parse(char *newstr, char *fence); #endif static int load(struct module *module, int cmd, void *arg); static int pmc_attach_process(struct proc *p, struct pmc *pm); static struct pmc *pmc_allocate_pmc_descriptor(void); static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p); static int pmc_attach_one_process(struct proc *p, struct pmc *pm); static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu); static int pmc_can_attach(struct pmc *pm, struct proc *p); static void pmc_cleanup(void); static int pmc_detach_process(struct proc *p, struct pmc *pm); static int pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags); static void pmc_destroy_owner_descriptor(struct pmc_owner *po); static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmc); static struct pmc_process *pmc_find_process_descriptor(struct proc *p, uint32_t mode); static void pmc_force_context_switch(void); static void pmc_link_target_process(struct pmc *pm, struct pmc_process *pp); static void pmc_maybe_remove_owner(struct pmc_owner *po); static void pmc_process_csw_in(struct thread *td); static void pmc_process_csw_out(struct thread *td); static void pmc_process_exit(void *arg, struct proc *p); static void pmc_process_fork(void *arg, struct proc *p1, struct proc *p2, int n); static void pmc_process_samples(int cpu); static void pmc_release_pmc_descriptor(struct pmc *pmc); static void pmc_remove_owner(struct pmc_owner *po); static void pmc_remove_process_descriptor(struct pmc_process *pp); static void pmc_restore_cpu_binding(struct pmc_binding *pb); static void pmc_save_cpu_binding(struct pmc_binding *pb); static void pmc_select_cpu(int cpu); static int pmc_start(struct pmc *pm); static int pmc_stop(struct pmc *pm); static int pmc_syscall_handler(struct thread *td, void *syscall_args); static void pmc_unlink_target_process(struct pmc *pmc, struct pmc_process *pp); /* * Kernel tunables and sysctl(8) interface. */ SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); #ifdef DEBUG struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS; char pmc_debugstr[PMC_DEBUG_STRSIZE]; TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, sizeof(pmc_debugstr)); SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags, CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN, 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags"); #endif /* * kern.hwpmc.hashrows -- determines the number of rows in the * of the hash table used to look up threads */ static int pmc_hashsize = PMC_HASH_SIZE; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize); SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD, &pmc_hashsize, 0, "rows in hash tables"); /* * kern.hwpmc.nsamples --- number of PC samples per CPU */ static int pmc_nsamples = PMC_NSAMPLES; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples); SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD, &pmc_nsamples, 0, "number of PC samples per CPU"); /* * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool. */ static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size); SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD, &pmc_mtxpool_size, 0, "size of spin mutex pool"); /* * security.bsd.unprivileged_syspmcs -- allow non-root processes to * allocate system-wide PMCs. * * Allowing unprivileged processes to allocate system PMCs is convenient * if system-wide measurements need to be taken concurrently with other * per-process measurements. This feature is turned off by default. */ SYSCTL_DECL(_security_bsd); static int pmc_unprivileged_syspmcs = 0; TUNABLE_INT("security.bsd.unprivileged_syspmcs", &pmc_unprivileged_syspmcs); SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RW, &pmc_unprivileged_syspmcs, 0, "allow unprivileged process to allocate system PMCs"); /* * Hash function. Discard the lower 2 bits of the pointer since * these are always zero for our uses. The hash multiplier is * round((2^LONG_BIT) * ((sqrt(5)-1)/2)). */ #if LONG_BIT == 64 #define _PMC_HM 11400714819323198486u #elif LONG_BIT == 32 #define _PMC_HM 2654435769u #else #error Must know the size of 'long' to compile #endif #define PMC_HASH_PTR(P,M) ((((unsigned long) (P) >> 2) * _PMC_HM) & (M)) /* * Syscall structures */ /* The `sysent' for the new syscall */ static struct sysent pmc_sysent = { 2, /* sy_narg */ pmc_syscall_handler /* sy_call */ }; static struct syscall_module_data pmc_syscall_mod = { load, NULL, &pmc_syscall_num, &pmc_sysent, { 0, NULL } }; static moduledata_t pmc_mod = { PMC_MODULE_NAME, syscall_module_handler, &pmc_syscall_mod }; DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY); MODULE_VERSION(pmc, PMC_VERSION); #ifdef DEBUG enum pmc_dbgparse_state { PMCDS_WS, /* in whitespace */ PMCDS_MAJOR, /* seen a major keyword */ PMCDS_MINOR }; static int pmc_debugflags_parse(char *newstr, char *fence) { char c, *p, *q; struct pmc_debugflags *tmpflags; int error, found, *newbits, tmp; size_t kwlen; MALLOC(tmpflags, struct pmc_debugflags *, sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO); p = newstr; error = 0; for (; p < fence && (c = *p); p++) { /* skip white space */ if (c == ' ' || c == '\t') continue; /* look for a keyword followed by "=" */ for (q = p; p < fence && (c = *p) && c != '='; p++) ; if (c != '=') { error = EINVAL; goto done; } kwlen = p - q; newbits = NULL; /* lookup flag group name */ #define DBG_SET_FLAG_MAJ(S,F) \ if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \ newbits = &tmpflags->pdb_ ## F; DBG_SET_FLAG_MAJ("cpu", CPU); DBG_SET_FLAG_MAJ("csw", CSW); DBG_SET_FLAG_MAJ("logging", LOG); DBG_SET_FLAG_MAJ("module", MOD); DBG_SET_FLAG_MAJ("md", MDP); DBG_SET_FLAG_MAJ("owner", OWN); DBG_SET_FLAG_MAJ("pmc", PMC); DBG_SET_FLAG_MAJ("process", PRC); DBG_SET_FLAG_MAJ("sampling", SAM); if (newbits == NULL) { error = EINVAL; goto done; } p++; /* skip the '=' */ /* Now parse the individual flags */ tmp = 0; newflag: for (q = p; p < fence && (c = *p); p++) if (c == ' ' || c == '\t' || c == ',') break; /* p == fence or c == ws or c == "," or c == 0 */ if ((kwlen = p - q) == 0) { *newbits = tmp; continue; } found = 0; #define DBG_SET_FLAG_MIN(S,F) \ if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \ tmp |= found = (1 << PMC_DEBUG_MIN_ ## F) /* a '*' denotes all possible flags in the group */ if (kwlen == 1 && *q == '*') tmp = found = ~0; /* look for individual flag names */ DBG_SET_FLAG_MIN("allocaterow", ALR); DBG_SET_FLAG_MIN("allocate", ALL); DBG_SET_FLAG_MIN("attach", ATT); DBG_SET_FLAG_MIN("bind", BND); DBG_SET_FLAG_MIN("config", CFG); DBG_SET_FLAG_MIN("exec", EXC); DBG_SET_FLAG_MIN("exit", EXT); DBG_SET_FLAG_MIN("find", FND); DBG_SET_FLAG_MIN("flush", FLS); DBG_SET_FLAG_MIN("fork", FRK); DBG_SET_FLAG_MIN("getbuf", GTB); DBG_SET_FLAG_MIN("hook", PMH); DBG_SET_FLAG_MIN("init", INI); DBG_SET_FLAG_MIN("intr", INT); DBG_SET_FLAG_MIN("linktarget", TLK); DBG_SET_FLAG_MIN("mayberemove", OMR); DBG_SET_FLAG_MIN("ops", OPS); DBG_SET_FLAG_MIN("read", REA); DBG_SET_FLAG_MIN("register", REG); DBG_SET_FLAG_MIN("release", REL); DBG_SET_FLAG_MIN("remove", ORM); DBG_SET_FLAG_MIN("sample", SAM); DBG_SET_FLAG_MIN("scheduleio", SIO); DBG_SET_FLAG_MIN("select", SEL); DBG_SET_FLAG_MIN("signal", SIG); DBG_SET_FLAG_MIN("swi", SWI); DBG_SET_FLAG_MIN("swo", SWO); DBG_SET_FLAG_MIN("start", STA); DBG_SET_FLAG_MIN("stop", STO); DBG_SET_FLAG_MIN("syscall", PMS); DBG_SET_FLAG_MIN("unlinktarget", TUL); DBG_SET_FLAG_MIN("write", WRI); if (found == 0) { /* unrecognized flag name */ error = EINVAL; goto done; } if (c == 0 || c == ' ' || c == '\t') { /* end of flag group */ *newbits = tmp; continue; } p++; goto newflag; } /* save the new flag set */ bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags)); done: FREE(tmpflags, M_PMC); return error; } static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS) { char *fence, *newstr; int error; unsigned int n; (void) arg1; (void) arg2; /* unused parameters */ n = sizeof(pmc_debugstr); MALLOC(newstr, char *, n, M_PMC, M_ZERO|M_WAITOK); (void) strlcpy(newstr, pmc_debugstr, n); error = sysctl_handle_string(oidp, newstr, n, req); /* if there is a new string, parse and copy it */ if (error == 0 && req->newptr != NULL) { fence = newstr + (n < req->newlen ? n : req->newlen + 1); if ((error = pmc_debugflags_parse(newstr, fence)) == 0) (void) strlcpy(pmc_debugstr, newstr, sizeof(pmc_debugstr)); } FREE(newstr, M_PMC); return error; } #endif /* * Concurrency Control * * The driver manages the following data structures: * * - target process descriptors, one per target process * - owner process descriptors (and attached lists), one per owner process * - lookup hash tables for owner and target processes * - PMC descriptors (and attached lists) * - per-cpu hardware state * - the 'hook' variable through which the kernel calls into * this module * - the machine hardware state (managed by the MD layer) * * These data structures are accessed from: * * - thread context-switch code * - interrupt handlers (possibly on multiple cpus) * - kernel threads on multiple cpus running on behalf of user * processes doing system calls * - this driver's private kernel threads * * = Locks and Locking strategy = * * The driver uses four locking strategies for its operation: * * - There is a 'global' SX lock "pmc_sx" that is used to protect * the its 'meta-data'. * * Calls into the module (via syscall() or by the kernel) start with * this lock being held in exclusive mode. Depending on the requested * operation, the lock may be downgraded to 'shared' mode to allow * more concurrent readers into the module. * * This SX lock is held in exclusive mode for any operations that * modify the linkages between the driver's internal data structures. * * The 'pmc_hook' function pointer is also protected by this lock. * It is only examined with the sx lock held in exclusive mode. The * kernel module is allowed to be unloaded only with the sx lock * held in exclusive mode. In normal syscall handling, after * acquiring the pmc_sx lock we first check that 'pmc_hook' is * non-null before proceeding. This prevents races between the * thread unloading the module and other threads seeking to use the * module. * * - Lookups of target process structures and owner process structures * cannot use the global "pmc_sx" SX lock because these lookups need * to happen during context switches and in other critical sections * where sleeping is not allowed. We protect these lookup tables * with their own private spin-mutexes, "pmc_processhash_mtx" and * "pmc_ownerhash_mtx". These are 'leaf' mutexes, in that no other * lock is acquired with these locks held. * * - Interrupt handlers work in a lock free manner. At interrupt * time, handlers look at the PMC pointer (phw->phw_pmc) configured * when the PMC was started. If this pointer is NULL, the interrupt * is ignored after updating driver statistics. We ensure that this * pointer is set (using an atomic operation if necessary) before the * PMC hardware is started. Conversely, this pointer is unset atomically * only after the PMC hardware is stopped. * * We ensure that everything needed for the operation of an * interrupt handler is available without it needing to acquire any * locks. We also ensure that a PMC's software state is destroyed only * after the PMC is taken off hardware (on all CPUs). * * - Context-switch handling with process-private PMCs needs more * care. * * A given process may be the target of multiple PMCs. For example, * PMCATTACH and PMCDETACH may be requested by a process on one CPU * while the target process is running on another. A PMC could also * be getting released because its owner is exiting. We tackle * these situations in the following manner: * * - each target process structure 'pmc_process' has an array * of 'struct pmc *' pointers, one for each hardware PMC. * * - At context switch IN time, each "target" PMC in RUNNING state * gets started on hardware and a pointer to each PMC is copied into * the per-cpu phw array. The 'runcount' for the PMC is * incremented. * * - At context switch OUT time, all process-virtual PMCs are stopped * on hardware. The saved value is added to the PMCs value field * only if the PMC is in a non-deleted state (the PMCs state could * have changed during the current time slice). * * Note that since in-between a switch IN on a processor and a switch * OUT, the PMC could have been released on another CPU. Therefore * context switch OUT always looks at the hardware state to turn * OFF PMCs and will update a PMC's saved value only if reachable * from the target process record. * * - OP PMCRELEASE could be called on a PMC at any time (the PMC could * be attached to many processes at the time of the call and could * be active on multiple CPUs). * * We prevent further scheduling of the PMC by marking it as in * state 'DELETED'. If the runcount of the PMC is non-zero then * this PMC is currently running on a CPU somewhere. The thread * doing the PMCRELEASE operation waits by repeatedly doing an * tsleep() till the runcount comes to zero. * */ /* * save the cpu binding of the current kthread */ static void pmc_save_cpu_binding(struct pmc_binding *pb) { PMCDBG(CPU,BND,2, "%s", "save-cpu"); mtx_lock_spin(&sched_lock); pb->pb_bound = sched_is_bound(curthread); pb->pb_cpu = curthread->td_oncpu; mtx_unlock_spin(&sched_lock); PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu); } /* * restore the cpu binding of the current thread */ static void pmc_restore_cpu_binding(struct pmc_binding *pb) { PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d", curthread->td_oncpu, pb->pb_cpu); mtx_lock_spin(&sched_lock); if (pb->pb_bound) sched_bind(curthread, pb->pb_cpu); else sched_unbind(curthread); mtx_unlock_spin(&sched_lock); PMCDBG(CPU,BND,2, "%s", "restore-cpu done"); } /* * move execution over the specified cpu and bind it there. */ static void pmc_select_cpu(int cpu) { KASSERT(cpu >= 0 && cpu < mp_ncpus, ("[pmc,%d] bad cpu number %d", __LINE__, cpu)); /* never move to a disabled CPU */ KASSERT(pmc_cpu_is_disabled(cpu) == 0, ("[pmc,%d] selecting " "disabled CPU %d", __LINE__, cpu)); PMCDBG(CPU,SEL,2, "select-cpu cpu=%d", cpu); mtx_lock_spin(&sched_lock); sched_bind(curthread, cpu); mtx_unlock_spin(&sched_lock); KASSERT(curthread->td_oncpu == cpu, ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__, cpu, curthread->td_oncpu)); PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu); } /* * Force a context switch. * * We do this by tsleep'ing for 1 tick -- invoking mi_switch() is not * guaranteed to force a context switch. */ static void pmc_force_context_switch(void) { u_char curpri; mtx_lock_spin(&sched_lock); curpri = curthread->td_priority; mtx_unlock_spin(&sched_lock); (void) tsleep((void *) pmc_force_context_switch, curpri, "pmcctx", 1); } /* * Get the file name for an executable. This is a simple wrapper * around vn_fullpath(9). */ static void pmc_getfilename(struct vnode *v, char **fullpath, char **freepath) { struct thread *td; td = curthread; *fullpath = "unknown"; *freepath = NULL; vn_lock(v, LK_CANRECURSE | LK_EXCLUSIVE | LK_RETRY, td); vn_fullpath(td, v, fullpath, freepath); VOP_UNLOCK(v, 0, td); } /* * remove an process owning PMCs */ void pmc_remove_owner(struct pmc_owner *po) { struct pmc *pm, *tmp; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG(OWN,ORM,1, "remove-owner po=%p", po); /* Remove descriptor from the owner hash table */ LIST_REMOVE(po, po_next); /* release all owned PMC descriptors */ LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) { PMCDBG(OWN,ORM,2, "pmc=%p", pm); KASSERT(pm->pm_owner == po, ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po)); pmc_release_pmc_descriptor(pm); /* will unlink from the list */ } KASSERT(po->po_sscount == 0, ("[pmc,%d] SS count not zero", __LINE__)); KASSERT(LIST_EMPTY(&po->po_pmcs), ("[pmc,%d] PMC list not empty", __LINE__)); /* de-configure the log file if present */ if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_deconfigure_log(po); } /* * remove an owner process record if all conditions are met. */ static void pmc_maybe_remove_owner(struct pmc_owner *po) { PMCDBG(OWN,OMR,1, "maybe-remove-owner po=%p", po); /* * Remove owner record if * - this process does not own any PMCs * - this process has not allocated a system-wide sampling buffer */ if (LIST_EMPTY(&po->po_pmcs) && ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) { pmc_remove_owner(po); pmc_destroy_owner_descriptor(po); } } /* * Add an association between a target process and a PMC. */ static void pmc_link_target_process(struct pmc *pm, struct pmc_process *pp) { int ri; struct pmc_target *pt; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm != NULL && pp != NULL, ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d", __LINE__, pm, pp->pp_proc->p_pid)); KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1), ("[pmc,%d] Illegal reference count %d for process record %p", __LINE__, pp->pp_refcnt, (void *) pp)); ri = PMC_TO_ROWINDEX(pm); PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p", pm, ri, pp); #ifdef DEBUG LIST_FOREACH(pt, &pm->pm_targets, pt_next) if (pt->pt_process == pp) KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", __LINE__, pp, pm)); #endif MALLOC(pt, struct pmc_target *, sizeof(struct pmc_target), M_PMC, M_ZERO|M_WAITOK); pt->pt_process = pp; LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc, (uintptr_t)pm); if (pm->pm_owner->po_owner == pp->pp_proc) pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER; /* * Initialize the per-process values at this row index. */ pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ? pm->pm_sc.pm_reloadcount : 0; pp->pp_refcnt++; } /* * Removes the association between a target process and a PMC. */ static void pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) { int ri; struct proc *p; struct pmc_target *ptgt; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm != NULL && pp != NULL, ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc, ("[pmc,%d] Illegal ref count %d on process record %p", __LINE__, pp->pp_refcnt, (void *) pp)); ri = PMC_TO_ROWINDEX(pm); PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p", pm, ri, pp); KASSERT(pp->pp_pmcs[ri].pp_pmc == pm, ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__, ri, pm, pp->pp_pmcs[ri].pp_pmc)); pp->pp_pmcs[ri].pp_pmc = NULL; pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0; /* Remove owner-specific flags */ if (pm->pm_owner->po_owner == pp->pp_proc) { pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS; pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER; } pp->pp_refcnt--; /* Remove the target process from the PMC structure */ LIST_FOREACH(ptgt, &pm->pm_targets, pt_next) if (ptgt->pt_process == pp) break; KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found " "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); LIST_REMOVE(ptgt, pt_next); FREE(ptgt, M_PMC); /* if the PMC now lacks targets, send the owner a SIGIO */ if (LIST_EMPTY(&pm->pm_targets)) { p = pm->pm_owner->po_owner; PROC_LOCK(p); psignal(p, SIGIO); PROC_UNLOCK(p); PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p, SIGIO); } } /* * Check if PMC 'pm' may be attached to target process 't'. */ static int pmc_can_attach(struct pmc *pm, struct proc *t) { struct proc *o; /* pmc owner */ struct ucred *oc, *tc; /* owner, target credentials */ int decline_attach, i; /* * A PMC's owner can always attach that PMC to itself. */ if ((o = pm->pm_owner->po_owner) == t) return 0; PROC_LOCK(o); oc = o->p_ucred; crhold(oc); PROC_UNLOCK(o); PROC_LOCK(t); tc = t->p_ucred; crhold(tc); PROC_UNLOCK(t); /* * The effective uid of the PMC owner should match at least one * of the {effective,real,saved} uids of the target process. */ decline_attach = oc->cr_uid != tc->cr_uid && oc->cr_uid != tc->cr_svuid && oc->cr_uid != tc->cr_ruid; /* * Every one of the target's group ids, must be in the owner's * group list. */ for (i = 0; !decline_attach && i < tc->cr_ngroups; i++) decline_attach = !groupmember(tc->cr_groups[i], oc); /* check the read and saved gids too */ if (decline_attach == 0) decline_attach = !groupmember(tc->cr_rgid, oc) || !groupmember(tc->cr_svgid, oc); crfree(tc); crfree(oc); return !decline_attach; } /* * Attach a process to a PMC. */ static int pmc_attach_one_process(struct proc *p, struct pmc *pm) { int ri; char *fullpath, *freepath; struct pmc_process *pp; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); /* * Locate the process descriptor corresponding to process 'p', * allocating space as needed. * * Verify that rowindex 'pm_rowindex' is free in the process * descriptor. * * If not, allocate space for a descriptor and link the * process descriptor and PMC. */ ri = PMC_TO_ROWINDEX(pm); if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) return ENOMEM; if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */ return EEXIST; if (pp->pp_pmcs[ri].pp_pmc != NULL) return EBUSY; pmc_link_target_process(pm, pp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) && (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0) pm->pm_flags |= PMC_F_NEEDS_LOGFILE; pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */ /* issue an attach event to a configured log file */ if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) { pmc_getfilename(p->p_textvp, &fullpath, &freepath); pmclog_process_pmcattach(pm, p->p_pid, fullpath); if (freepath) FREE(freepath, M_TEMP); } /* mark process as using HWPMCs */ PROC_LOCK(p); p->p_flag |= P_HWPMC; PROC_UNLOCK(p); return 0; } /* * Attach a process and optionally its children */ static int pmc_attach_process(struct proc *p, struct pmc *pm) { int error; struct proc *top; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); /* * If this PMC successfully allowed a GETMSR operation * in the past, disallow further ATTACHes. */ if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0) return EPERM; if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) return pmc_attach_one_process(p, pm); /* * Traverse all child processes, attaching them to * this PMC. */ sx_slock(&proctree_lock); top = p; for (;;) { if ((error = pmc_attach_one_process(p, pm)) != 0) break; if (!LIST_EMPTY(&p->p_children)) p = LIST_FIRST(&p->p_children); else for (;;) { if (p == top) goto done; if (LIST_NEXT(p, p_sibling)) { p = LIST_NEXT(p, p_sibling); break; } p = p->p_pptr; } } if (error) (void) pmc_detach_process(top, pm); done: sx_sunlock(&proctree_lock); return error; } /* * Detach a process from a PMC. If there are no other PMCs tracking * this process, remove the process structure from its hash table. If * 'flags' contains PMC_FLAG_REMOVE, then free the process structure. */ static int pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags) { int ri; struct pmc_process *pp; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm != NULL, ("[pmc,%d] null pm pointer", __LINE__)); ri = PMC_TO_ROWINDEX(pm); PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x", pm, ri, p, p->p_pid, p->p_comm, flags); if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) return ESRCH; if (pp->pp_pmcs[ri].pp_pmc != pm) return EINVAL; pmc_unlink_target_process(pm, pp); /* Issue a detach entry if a log file is configured */ if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_pmcdetach(pm, p->p_pid); /* * If there are no PMCs targetting this process, we remove its * descriptor from the target hash table and unset the P_HWPMC * flag in the struct proc. */ KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc, ("[pmc,%d] Illegal refcnt %d for process struct %p", __LINE__, pp->pp_refcnt, pp)); if (pp->pp_refcnt != 0) /* still a target of some PMC */ return 0; pmc_remove_process_descriptor(pp); if (flags & PMC_FLAG_REMOVE) FREE(pp, M_PMC); PROC_LOCK(p); p->p_flag &= ~P_HWPMC; PROC_UNLOCK(p); return 0; } /* * Detach a process and optionally its descendants from a PMC. */ static int pmc_detach_process(struct proc *p, struct pmc *pm) { struct proc *top; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); /* * Traverse all children, detaching them from this PMC. We * ignore errors since we could be detaching a PMC from a * partially attached proc tree. */ sx_slock(&proctree_lock); top = p; for (;;) { (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); if (!LIST_EMPTY(&p->p_children)) p = LIST_FIRST(&p->p_children); else for (;;) { if (p == top) goto done; if (LIST_NEXT(p, p_sibling)) { p = LIST_NEXT(p, p_sibling); break; } p = p->p_pptr; } } done: sx_sunlock(&proctree_lock); if (LIST_EMPTY(&pm->pm_targets)) pm->pm_flags &= ~PMC_F_ATTACH_DONE; return 0; } /* * Thread context switch IN */ static void pmc_process_csw_in(struct thread *td) { int cpu; unsigned int ri; struct pmc *pm; struct proc *p; struct pmc_cpu *pc; struct pmc_hw *phw; struct pmc_process *pp; pmc_value_t newvalue; p = td->td_proc; if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL) return; KASSERT(pp->pp_proc == td->td_proc, ("[pmc,%d] not my thread state", __LINE__)); critical_enter(); /* no preemption from this point */ cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, p->p_pid, p->p_comm, pp); KASSERT(cpu >= 0 && cpu < mp_ncpus, ("[pmc,%d] wierd CPU id %d", __LINE__, cpu)); pc = pmc_pcpu[cpu]; for (ri = 0; ri < md->pmd_npmc; ri++) { if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) continue; KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] Target PMC in non-virtual mode (%d)", __LINE__, PMC_TO_MODE(pm))); KASSERT(PMC_TO_ROWINDEX(pm) == ri, ("[pmc,%d] Row index mismatch pmc %d != ri %d", __LINE__, PMC_TO_ROWINDEX(pm), ri)); /* * Only PMCs that are marked as 'RUNNING' need * be placed on hardware. */ if (pm->pm_state != PMC_STATE_RUNNING) continue; /* increment PMC runcount */ atomic_add_rel_32(&pm->pm_runcount, 1); /* configure the HWPMC we are going to use. */ md->pmd_config_pmc(cpu, ri, pm); phw = pc->pc_hwpmcs[ri]; KASSERT(phw != NULL, ("[pmc,%d] null hw pointer", __LINE__)); KASSERT(phw->phw_pmc == pm, ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__, phw->phw_pmc, pm)); /* * Write out saved value and start the PMC. * * Sampling PMCs use a per-process value, while * counting mode PMCs use a per-pmc value that is * inherited across descendants. */ if (PMC_TO_MODE(pm) == PMC_MODE_TS) { mtx_pool_lock_spin(pmc_mtxpool, pm); newvalue = PMC_PCPU_SAVED(cpu,ri) = pp->pp_pmcs[ri].pp_pmcval; mtx_pool_unlock_spin(pmc_mtxpool, pm); } else { KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, ("[pmc,%d] illegal mode=%d", __LINE__, PMC_TO_MODE(pm))); mtx_pool_lock_spin(pmc_mtxpool, pm); newvalue = PMC_PCPU_SAVED(cpu, ri) = pm->pm_gv.pm_savedvalue; mtx_pool_unlock_spin(pmc_mtxpool, pm); } PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue); md->pmd_write_pmc(cpu, ri, newvalue); md->pmd_start_pmc(cpu, ri); } /* * perform any other architecture/cpu dependent thread * switch-in actions. */ (void) (*md->pmd_switch_in)(pc, pp); critical_exit(); } /* * Thread context switch OUT. */ static void pmc_process_csw_out(struct thread *td) { int cpu; enum pmc_mode mode; unsigned int ri; struct pmc *pm; struct proc *p; struct pmc_cpu *pc; struct pmc_process *pp; int64_t tmp; pmc_value_t newvalue; /* * Locate our process descriptor; this may be NULL if * this process is exiting and we have already removed * the process from the target process table. * * Note that due to kernel preemption, multiple * context switches may happen while the process is * exiting. * * Note also that if the target process cannot be * found we still need to deconfigure any PMCs that * are currently running on hardware. */ p = td->td_proc; pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE); /* * save PMCs */ critical_enter(); cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, p->p_pid, p->p_comm, pp); KASSERT(cpu >= 0 && cpu < mp_ncpus, ("[pmc,%d wierd CPU id %d", __LINE__, cpu)); pc = pmc_pcpu[cpu]; /* * When a PMC gets unlinked from a target PMC, it will * be removed from the target's pp_pmc[] array. * * However, on a MP system, the target could have been * executing on another CPU at the time of the unlink. * So, at context switch OUT time, we need to look at * the hardware to determine if a PMC is scheduled on * it. */ for (ri = 0; ri < md->pmd_npmc; ri++) { pm = NULL; (void) (*md->pmd_get_config)(cpu, ri, &pm); if (pm == NULL) /* nothing at this row index */ continue; mode = PMC_TO_MODE(pm); if (!PMC_IS_VIRTUAL_MODE(mode)) continue; /* not a process virtual PMC */ KASSERT(PMC_TO_ROWINDEX(pm) == ri, ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", __LINE__, PMC_TO_ROWINDEX(pm), ri)); /* Stop hardware if not already stopped */ if (pm->pm_stalled == 0) md->pmd_stop_pmc(cpu, ri); /* reduce this PMC's runcount */ atomic_subtract_rel_32(&pm->pm_runcount, 1); /* * If this PMC is associated with this process, * save the reading. */ if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) { KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); KASSERT(pp->pp_refcnt > 0, ("[pmc,%d] pp refcnt = %d", __LINE__, pp->pp_refcnt)); md->pmd_read_pmc(cpu, ri, &newvalue); tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri, tmp); if (mode == PMC_MODE_TS) { /* * For sampling process-virtual PMCs, * we expect the count to be * decreasing as the 'value' * programmed into the PMC is the * number of events to be seen till * the next sampling interrupt. */ if (tmp < 0) tmp += pm->pm_sc.pm_reloadcount; mtx_pool_lock_spin(pmc_mtxpool, pm); pp->pp_pmcs[ri].pp_pmcval -= tmp; if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0) pp->pp_pmcs[ri].pp_pmcval += pm->pm_sc.pm_reloadcount; mtx_pool_unlock_spin(pmc_mtxpool, pm); } else { /* * For counting process-virtual PMCs, * we expect the count to be * increasing monotonically, modulo a 64 * bit wraparound. */ KASSERT((int64_t) tmp >= 0, ("[pmc,%d] negative increment cpu=%d " "ri=%d newvalue=%jx saved=%jx " "incr=%jx", __LINE__, cpu, ri, newvalue, PMC_PCPU_SAVED(cpu,ri), tmp)); mtx_pool_lock_spin(pmc_mtxpool, pm); pm->pm_gv.pm_savedvalue += tmp; pp->pp_pmcs[ri].pp_pmcval += tmp; mtx_pool_unlock_spin(pmc_mtxpool, pm); if (pm->pm_flags & PMC_F_LOG_PROCCSW) pmclog_process_proccsw(pm, pp, tmp); } } /* mark hardware as free */ md->pmd_config_pmc(cpu, ri, NULL); } /* * perform any other architecture/cpu dependent thread * switch out functions. */ (void) (*md->pmd_switch_out)(pc, pp); critical_exit(); } /* + * Log a KLD operation. + */ + +static void +pmc_process_kld_load(struct pmckern_map_in *pkm) +{ + struct pmc_owner *po; + + sx_assert(&pmc_sx, SX_LOCKED); + + /* + * Notify owners of system sampling PMCs about KLD operations. + */ + + LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) + if (po->po_flags & PMC_PO_OWNS_LOGFILE) + pmclog_process_map_in(po, (pid_t) -1, pkm->pm_address, + (char *) pkm->pm_file); + + /* + * TODO: Notify owners of (all) process-sampling PMCs too. + */ + + return; +} + +static void +pmc_process_kld_unload(struct pmckern_map_out *pkm) +{ + struct pmc_owner *po; + + sx_assert(&pmc_sx, SX_LOCKED); + + LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) + if (po->po_flags & PMC_PO_OWNS_LOGFILE) + pmclog_process_map_out(po, (pid_t) -1, + pkm->pm_address, pkm->pm_address + pkm->pm_size); + + /* + * TODO: Notify owners of process-sampling PMCs. + */ +} + +/* + * A mapping change for a process. + */ + +static void +pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm) +{ + int ri; + pid_t pid; + char *fullpath, *freepath; + const struct pmc *pm; + struct pmc_owner *po; + const struct pmc_process *pp; + + freepath = fullpath = NULL; + pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath); + + pid = td->td_proc->p_pid; + + /* Inform owners of all system-wide sampling PMCs. */ + LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) + if (po->po_flags & PMC_PO_OWNS_LOGFILE) + pmclog_process_map_in(po, pid, pkm->pm_address, fullpath); + + if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) + goto done; + + /* + * Inform sampling PMC owners tracking this process. + */ + for (ri = 0; ri < md->pmd_npmc; ri++) + if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && + PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + pmclog_process_map_in(pm->pm_owner, + pid, pkm->pm_address, fullpath); + + done: + if (freepath) + FREE(freepath, M_TEMP); +} + + +/* + * Log an munmap request. + */ + +static void +pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm) +{ + int ri; + pid_t pid; + struct pmc_owner *po; + const struct pmc *pm; + const struct pmc_process *pp; + + pid = td->td_proc->p_pid; + + LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) + if (po->po_flags & PMC_PO_OWNS_LOGFILE) + pmclog_process_map_out(po, pid, pkm->pm_address, + pkm->pm_address + pkm->pm_size); + + if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) + return; + + for (ri = 0; ri < md->pmd_npmc; ri++) + if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && + PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) + pmclog_process_map_out(po, pid, pkm->pm_address, + pkm->pm_address + pkm->pm_size); +} + +/* * The 'hook' invoked from the kernel proper */ #ifdef DEBUG const char *pmc_hooknames[] = { + /* these strings correspond to PMC_FN_* in */ "", - "EXIT", "EXEC", - "FORK", "CSW-IN", "CSW-OUT", - "SAMPLE" + "SAMPLE", + "KLDLOAD", + "KLDUNLOAD", + "MMAP", + "MUNMAP" }; #endif static int pmc_hook_handler(struct thread *td, int function, void *arg) { PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function, pmc_hooknames[function], arg); switch (function) { /* * Process exec() */ case PMC_FN_PROCESS_EXEC: { char *fullpath, *freepath; unsigned int ri; int is_using_hwpmcs; struct pmc *pm; struct proc *p; struct pmc_owner *po; struct pmc_process *pp; struct pmckern_procexec *pk; sx_assert(&pmc_sx, SX_XLOCKED); p = td->td_proc; pmc_getfilename(p->p_textvp, &fullpath, &freepath); pk = (struct pmckern_procexec *) arg; /* Inform owners of SS mode PMCs of the exec event. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procexec(po, PMC_ID_INVALID, p->p_pid, pk->pm_entryaddr, fullpath); PROC_LOCK(p); is_using_hwpmcs = p->p_flag & P_HWPMC; PROC_UNLOCK(p); if (!is_using_hwpmcs) { if (freepath) FREE(freepath, M_TEMP); break; } /* * PMCs are not inherited across an exec(): remove any * PMCs that this process is the owner of. */ if ((po = pmc_find_owner_descriptor(p)) != NULL) { pmc_remove_owner(po); pmc_destroy_owner_descriptor(po); } /* * If the process being exec'ed is not the target of any * PMC, we are done. */ if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) { if (freepath) FREE(freepath, M_TEMP); break; } /* * Log the exec event to all monitoring owners. Skip * owners who have already recieved the event because * they had system sampling PMCs active. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { po = pm->pm_owner; if (po->po_sscount == 0 && po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procexec(po, pm->pm_id, p->p_pid, pk->pm_entryaddr, fullpath); } if (freepath) FREE(freepath, M_TEMP); PMCDBG(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d", p, p->p_pid, p->p_comm, pk->pm_credentialschanged); if (pk->pm_credentialschanged == 0) /* no change */ break; /* * If the newly exec()'ed process has a different credential * than before, allow it to be the target of a PMC only if * the PMC's owner has sufficient priviledge. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) if (pmc_can_attach(pm, td->td_proc) != 0) pmc_detach_one_process(td->td_proc, pm, PMC_FLAG_NONE); KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc, ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__, pp->pp_refcnt, pp)); /* * If this process is no longer the target of any * PMCs, we can remove the process entry and free * up space. */ if (pp->pp_refcnt == 0) { pmc_remove_process_descriptor(pp); FREE(pp, M_PMC); break; } } break; case PMC_FN_CSW_IN: pmc_process_csw_in(td); break; case PMC_FN_CSW_OUT: pmc_process_csw_out(td); break; /* * Process accumulated PC samples. * * This function is expected to be called by hardclock() for * each CPU that has accumulated PC samples. * * This function is to be executed on the CPU whose samples * are being processed. */ case PMC_FN_DO_SAMPLES: /* * Clear the cpu specific bit in the CPU mask before * do the rest of the processing. If the NMI handler * gets invoked after the "atomic_clear_int()" call * below but before "pmc_process_samples()" gets * around to processing the interrupt, then we will * come back here at the next hardclock() tick (and * may find nothing to do if "pmc_process_samples()" * had already processed the interrupt). We don't * lose the interrupt sample. */ atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid))); pmc_process_samples(PCPU_GET(cpuid)); break; + + case PMC_FN_KLD_LOAD: + sx_assert(&pmc_sx, SX_LOCKED); + pmc_process_kld_load((struct pmckern_map_in *) arg); + break; + + case PMC_FN_KLD_UNLOAD: + sx_assert(&pmc_sx, SX_LOCKED); + pmc_process_kld_unload((struct pmckern_map_out *) arg); + break; + + case PMC_FN_MMAP: + sx_assert(&pmc_sx, SX_LOCKED); + pmc_process_mmap(td, (struct pmckern_map_in *) arg); + break; + + case PMC_FN_MUNMAP: + sx_assert(&pmc_sx, SX_LOCKED); + pmc_process_munmap(td, (struct pmckern_map_out *) arg); + break; + default: #ifdef DEBUG KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); #endif break; } return 0; } /* * allocate a 'struct pmc_owner' descriptor in the owner hash table. */ static struct pmc_owner * pmc_allocate_owner_descriptor(struct proc *p) { uint32_t hindex; struct pmc_owner *po; struct pmc_ownerhash *poh; hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); poh = &pmc_ownerhash[hindex]; /* allocate space for N pointers and one descriptor struct */ MALLOC(po, struct pmc_owner *, sizeof(struct pmc_owner), M_PMC, M_ZERO|M_WAITOK); po->po_sscount = po->po_error = po->po_flags = 0; po->po_file = NULL; po->po_owner = p; po->po_kthread = NULL; LIST_INIT(&po->po_pmcs); LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */ TAILQ_INIT(&po->po_logbuffers); mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc", MTX_SPIN); PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p", p, p->p_pid, p->p_comm, po); return po; } static void pmc_destroy_owner_descriptor(struct pmc_owner *po) { PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)", po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm); mtx_destroy(&po->po_mtx); FREE(po, M_PMC); } /* * find the descriptor corresponding to process 'p', adding or removing it * as specified by 'mode'. */ static struct pmc_process * pmc_find_process_descriptor(struct proc *p, uint32_t mode) { uint32_t hindex; struct pmc_process *pp, *ppnew; struct pmc_processhash *pph; hindex = PMC_HASH_PTR(p, pmc_processhashmask); pph = &pmc_processhash[hindex]; ppnew = NULL; /* * Pre-allocate memory in the FIND_ALLOCATE case since we * cannot call malloc(9) once we hold a spin lock. */ if (mode & PMC_FLAG_ALLOCATE) { /* allocate additional space for 'n' pmc pointers */ MALLOC(ppnew, struct pmc_process *, sizeof(struct pmc_process) + md->pmd_npmc * sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK); } mtx_lock_spin(&pmc_processhash_mtx); LIST_FOREACH(pp, pph, pp_next) if (pp->pp_proc == p) break; if ((mode & PMC_FLAG_REMOVE) && pp != NULL) LIST_REMOVE(pp, pp_next); if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL && ppnew != NULL) { ppnew->pp_proc = p; LIST_INSERT_HEAD(pph, ppnew, pp_next); pp = ppnew; ppnew = NULL; } mtx_unlock_spin(&pmc_processhash_mtx); if (pp != NULL && ppnew != NULL) FREE(ppnew, M_PMC); return pp; } /* * remove a process descriptor from the process hash table. */ static void pmc_remove_process_descriptor(struct pmc_process *pp) { KASSERT(pp->pp_refcnt == 0, ("[pmc,%d] Removing process descriptor %p with count %d", __LINE__, pp, pp->pp_refcnt)); mtx_lock_spin(&pmc_processhash_mtx); LIST_REMOVE(pp, pp_next); mtx_unlock_spin(&pmc_processhash_mtx); } /* * find an owner descriptor corresponding to proc 'p' */ static struct pmc_owner * pmc_find_owner_descriptor(struct proc *p) { uint32_t hindex; struct pmc_owner *po; struct pmc_ownerhash *poh; hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); poh = &pmc_ownerhash[hindex]; po = NULL; LIST_FOREACH(po, poh, po_next) if (po->po_owner == p) break; PMCDBG(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> " "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po); return po; } /* * pmc_allocate_pmc_descriptor * * Allocate a pmc descriptor and initialize its * fields. */ static struct pmc * pmc_allocate_pmc_descriptor(void) { struct pmc *pmc; MALLOC(pmc, struct pmc *, sizeof(struct pmc), M_PMC, M_ZERO|M_WAITOK); if (pmc != NULL) { pmc->pm_owner = NULL; LIST_INIT(&pmc->pm_targets); } PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc); return pmc; } /* * Destroy a pmc descriptor. */ static void pmc_destroy_pmc_descriptor(struct pmc *pm) { (void) pm; #ifdef DEBUG KASSERT(pm->pm_state == PMC_STATE_DELETED || pm->pm_state == PMC_STATE_FREE, ("[pmc,%d] destroying non-deleted PMC", __LINE__)); KASSERT(LIST_EMPTY(&pm->pm_targets), ("[pmc,%d] destroying pmc with targets", __LINE__)); KASSERT(pm->pm_owner == NULL, ("[pmc,%d] destroying pmc attached to an owner", __LINE__)); KASSERT(pm->pm_runcount == 0, ("[pmc,%d] pmc has non-zero run count %d", __LINE__, pm->pm_runcount)); #endif } static void pmc_wait_for_pmc_idle(struct pmc *pm) { #ifdef DEBUG volatile int maxloop; maxloop = 100 * mp_ncpus; #endif /* * Loop (with a forced context switch) till the PMC's runcount * comes down to zero. */ while (atomic_load_acq_32(&pm->pm_runcount) > 0) { #ifdef DEBUG maxloop--; KASSERT(maxloop > 0, ("[pmc,%d] (ri%d, rc%d) waiting too long for " "pmc to be free", __LINE__, PMC_TO_ROWINDEX(pm), pm->pm_runcount)); #endif pmc_force_context_switch(); } } /* * This function does the following things: * * - detaches the PMC from hardware * - unlinks all target threads that were attached to it * - removes the PMC from its owner's list * - destroy's the PMC private mutex * * Once this function completes, the given pmc pointer can be safely * FREE'd by the caller. */ static void pmc_release_pmc_descriptor(struct pmc *pm) { u_int ri, cpu; enum pmc_mode mode; struct pmc_hw *phw; struct pmc_owner *po; struct pmc_process *pp; struct pmc_target *ptgt, *tmp; struct pmc_binding pb; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); ri = PMC_TO_ROWINDEX(pm); mode = PMC_TO_MODE(pm); PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, mode); /* * First, we take the PMC off hardware. */ cpu = 0; if (PMC_IS_SYSTEM_MODE(mode)) { /* * A system mode PMC runs on a specific CPU. Switch * to this CPU and turn hardware off. */ pmc_save_cpu_binding(&pb); cpu = PMC_TO_CPU(pm); pmc_select_cpu(cpu); /* switch off non-stalled CPUs */ if (pm->pm_state == PMC_STATE_RUNNING && pm->pm_stalled == 0) { phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; KASSERT(phw->phw_pmc == pm, ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)", __LINE__, ri, phw->phw_pmc, pm)); PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri); critical_enter(); md->pmd_stop_pmc(cpu, ri); critical_exit(); } PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri); critical_enter(); md->pmd_config_pmc(cpu, ri, NULL); critical_exit(); /* adjust the global and process count of SS mode PMCs */ if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) { po = pm->pm_owner; po->po_sscount--; if (po->po_sscount == 0) { atomic_subtract_rel_int(&pmc_ss_count, 1); LIST_REMOVE(po, po_ssnext); } } pm->pm_state = PMC_STATE_DELETED; pmc_restore_cpu_binding(&pb); /* * We could have references to this PMC structure in * the per-cpu sample queues. Wait for the queue to * drain. */ pmc_wait_for_pmc_idle(pm); } else if (PMC_IS_VIRTUAL_MODE(mode)) { /* * A virtual PMC could be running on multiple CPUs at * a given instant. * * By marking its state as DELETED, we ensure that * this PMC is never further scheduled on hardware. * * Then we wait till all CPUs are done with this PMC. */ pm->pm_state = PMC_STATE_DELETED; /* Wait for the PMCs runcount to come to zero. */ pmc_wait_for_pmc_idle(pm); /* * At this point the PMC is off all CPUs and cannot be * freshly scheduled onto a CPU. It is now safe to * unlink all targets from this PMC. If a * process-record's refcount falls to zero, we remove * it from the hash table. The module-wide SX lock * protects us from races. */ LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) { pp = ptgt->pt_process; pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */ PMCDBG(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt); /* * If the target process record shows that no * PMCs are attached to it, reclaim its space. */ if (pp->pp_refcnt == 0) { pmc_remove_process_descriptor(pp); FREE(pp, M_PMC); } } cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */ } /* * Release any MD resources */ (void) md->pmd_release_pmc(cpu, ri, pm); /* * Update row disposition */ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) PMC_UNMARK_ROW_STANDALONE(ri); else PMC_UNMARK_ROW_THREAD(ri); /* unlink from the owner's list */ if (pm->pm_owner) { LIST_REMOVE(pm, pm_next); pm->pm_owner = NULL; } pmc_destroy_pmc_descriptor(pm); } /* * Register an owner and a pmc. */ static int pmc_register_owner(struct proc *p, struct pmc *pmc) { struct pmc_owner *po; sx_assert(&pmc_sx, SX_XLOCKED); if ((po = pmc_find_owner_descriptor(p)) == NULL) if ((po = pmc_allocate_owner_descriptor(p)) == NULL) return ENOMEM; KASSERT(pmc->pm_owner == NULL, ("[pmc,%d] attempting to own an initialized PMC", __LINE__)); pmc->pm_owner = po; LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next); PROC_LOCK(p); p->p_flag |= P_HWPMC; PROC_UNLOCK(p); if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_pmcallocate(pmc); PMCDBG(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p", po, pmc); return 0; } /* * Return the current row disposition: * == 0 => FREE * > 0 => PROCESS MODE * < 0 => SYSTEM MODE */ int pmc_getrowdisp(int ri) { return pmc_pmcdisp[ri]; } /* * Check if a PMC at row index 'ri' can be allocated to the current * process. * * Allocation can fail if: * - the current process is already being profiled by a PMC at index 'ri', * attached to it via OP_PMCATTACH. * - the current process has already allocated a PMC at index 'ri' * via OP_ALLOCATE. */ static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu) { enum pmc_mode mode; struct pmc *pm; struct pmc_owner *po; struct pmc_process *pp; PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d " "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu); /* * We shouldn't have already allocated a process-mode PMC at * row index 'ri'. * * We shouldn't have allocated a system-wide PMC on the same * CPU and same RI. */ if ((po = pmc_find_owner_descriptor(p)) != NULL) LIST_FOREACH(pm, &po->po_pmcs, pm_next) { if (PMC_TO_ROWINDEX(pm) == ri) { mode = PMC_TO_MODE(pm); if (PMC_IS_VIRTUAL_MODE(mode)) return EEXIST; if (PMC_IS_SYSTEM_MODE(mode) && (int) PMC_TO_CPU(pm) == cpu) return EEXIST; } } /* * We also shouldn't be the target of any PMC at this index * since otherwise a PMC_ATTACH to ourselves will fail. */ if ((pp = pmc_find_process_descriptor(p, 0)) != NULL) if (pp->pp_pmcs[ri].pp_pmc) return EEXIST; PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok", p, p->p_pid, p->p_comm, ri); return 0; } /* * Check if a given PMC at row index 'ri' can be currently used in * mode 'mode'. */ static int pmc_can_allocate_row(int ri, enum pmc_mode mode) { enum pmc_disp disp; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode); if (PMC_IS_SYSTEM_MODE(mode)) disp = PMC_DISP_STANDALONE; else disp = PMC_DISP_THREAD; /* * check disposition for PMC row 'ri': * * Expected disposition Row-disposition Result * * STANDALONE STANDALONE or FREE proceed * STANDALONE THREAD fail * THREAD THREAD or FREE proceed * THREAD STANDALONE fail */ if (!PMC_ROW_DISP_IS_FREE(ri) && !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) && !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri))) return EBUSY; /* * All OK */ PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode); return 0; } /* * Find a PMC descriptor with user handle 'pmcid' for thread 'td'. */ static struct pmc * pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid) { struct pmc *pm; KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc, ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc)); LIST_FOREACH(pm, &po->po_pmcs, pm_next) if (pm->pm_id == pmcid) return pm; return NULL; } static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc) { struct pmc *pm; struct pmc_owner *po; PMCDBG(PMC,FND,1, "find-pmc id=%d", pmcid); if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) return ESRCH; if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL) return EINVAL; PMCDBG(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm); *pmc = pm; return 0; } /* * Start a PMC. */ static int pmc_start(struct pmc *pm) { int error, cpu, ri; enum pmc_mode mode; struct pmc_owner *po; struct pmc_binding pb; KASSERT(pm != NULL, ("[pmc,%d] null pm", __LINE__)); mode = PMC_TO_MODE(pm); ri = PMC_TO_ROWINDEX(pm); error = 0; PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); po = pm->pm_owner; if (PMC_IS_VIRTUAL_MODE(mode)) { /* * If a PMCATTACH has never been done on this PMC, * attach it to its owner process. */ if (LIST_EMPTY(&pm->pm_targets)) error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH : pmc_attach_process(po->po_owner, pm); /* * Disallow PMCSTART if a logfile is required but has not * been configured yet. */ if (error == 0 && (pm->pm_flags & PMC_F_NEEDS_LOGFILE) && (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) error = EDOOFUS; /* * If the PMC is attached to its owner, then force a context * switch to ensure that the MD state gets set correctly. */ if (error == 0) { pm->pm_state = PMC_STATE_RUNNING; if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) pmc_force_context_switch(); } return error; } /* * A system-wide PMC. */ if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) return EDOOFUS; /* programming error */ /* * Add the owner to the global list if this is a system-wide * sampling PMC. */ if (mode == PMC_MODE_SS) { if (po->po_sscount == 0) { LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext); atomic_add_rel_int(&pmc_ss_count, 1); PMCDBG(PMC,OPS,1, "po=%p in global list", po); } po->po_sscount++; } + /* TODO: dump system wide process mappings to the log? */ + /* * Move to the CPU associated with this * PMC, and start the hardware. */ pmc_save_cpu_binding(&pb); cpu = PMC_TO_CPU(pm); if (pmc_cpu_is_disabled(cpu)) return ENXIO; pmc_select_cpu(cpu); /* * global PMCs are configured at allocation time * so write out the initial value and start the PMC. */ pm->pm_state = PMC_STATE_RUNNING; critical_enter(); if ((error = md->pmd_write_pmc(cpu, ri, PMC_IS_SAMPLING_MODE(mode) ? pm->pm_sc.pm_reloadcount : pm->pm_sc.pm_initial)) == 0) error = md->pmd_start_pmc(cpu, ri); critical_exit(); pmc_restore_cpu_binding(&pb); return error; } /* * Stop a PMC. */ static int pmc_stop(struct pmc *pm) { int cpu, error, ri; struct pmc_owner *po; struct pmc_binding pb; KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm)); pm->pm_state = PMC_STATE_STOPPED; /* * If the PMC is a virtual mode one, changing the state to * non-RUNNING is enough to ensure that the PMC never gets * scheduled. * * If this PMC is current running on a CPU, then it will * handled correctly at the time its target process is context * switched out. */ if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) return 0; /* * A system-mode PMC. Move to the CPU associated with * this PMC, and stop the hardware. We update the * 'initial count' so that a subsequent PMCSTART will * resume counting from the current hardware count. */ pmc_save_cpu_binding(&pb); cpu = PMC_TO_CPU(pm); KASSERT(cpu >= 0 && cpu < mp_ncpus, ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); if (pmc_cpu_is_disabled(cpu)) return ENXIO; pmc_select_cpu(cpu); ri = PMC_TO_ROWINDEX(pm); critical_enter(); if ((error = md->pmd_stop_pmc(cpu, ri)) == 0) error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial); critical_exit(); pmc_restore_cpu_binding(&pb); po = pm->pm_owner; /* remove this owner from the global list of SS PMC owners */ if (PMC_TO_MODE(pm) == PMC_MODE_SS) { po->po_sscount--; if (po->po_sscount == 0) { atomic_subtract_rel_int(&pmc_ss_count, 1); LIST_REMOVE(po, po_ssnext); PMCDBG(PMC,OPS,2,"po=%p removed from global list", po); } } return error; } #ifdef DEBUG static const char *pmc_op_to_name[] = { #undef __PMC_OP #define __PMC_OP(N, D) #N , __PMC_OPS() NULL }; #endif /* * The syscall interface */ #define PMC_GET_SX_XLOCK(...) do { \ sx_xlock(&pmc_sx); \ if (pmc_hook == NULL) { \ sx_xunlock(&pmc_sx); \ return __VA_ARGS__; \ } \ } while (0) #define PMC_DOWNGRADE_SX() do { \ sx_downgrade(&pmc_sx); \ is_sx_downgraded = 1; \ } while (0) static int pmc_syscall_handler(struct thread *td, void *syscall_args) { int error, is_sx_downgraded, op; struct pmc_syscall_args *c; void *arg; PMC_GET_SX_XLOCK(ENOSYS); DROP_GIANT(); is_sx_downgraded = 0; c = (struct pmc_syscall_args *) syscall_args; op = c->pmop_code; arg = c->pmop_data; PMCDBG(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op, pmc_op_to_name[op], arg); error = 0; atomic_add_int(&pmc_stats.pm_syscalls, 1); switch(op) { /* * Configure a log file. * * XXX This OP will be reworked. */ case PMC_OP_CONFIGURELOG: { + struct proc *p; struct pmc *pm; struct pmc_owner *po; + struct pmckern_map_in *km, *kmbase; struct pmc_op_configurelog cl; - struct proc *p; sx_assert(&pmc_sx, SX_XLOCKED); if ((error = copyin(arg, &cl, sizeof(cl))) != 0) break; /* mark this process as owning a log file */ p = td->td_proc; if ((po = pmc_find_owner_descriptor(p)) == NULL) if ((po = pmc_allocate_owner_descriptor(p)) == NULL) { error = ENOMEM; break; } /* * If a valid fd was passed in, try to configure that, * otherwise if 'fd' was less than zero and there was * a log file configured, flush its buffers and * de-configure it. */ if (cl.pm_logfd >= 0) error = pmclog_configure_log(po, cl.pm_logfd); else if (po->po_flags & PMC_PO_OWNS_LOGFILE) { pmclog_process_closelog(po); error = pmclog_flush(po); if (error == 0) { LIST_FOREACH(pm, &po->po_pmcs, pm_next) if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && pm->pm_state == PMC_STATE_RUNNING) pmc_stop(pm); error = pmclog_deconfigure_log(po); } } else error = EINVAL; + + if (error) + break; + + /* + * Log the current set of kernel modules. + */ + kmbase = linker_hwpmc_list_objects(); + for (km = kmbase; km->pm_file != NULL; km++) { + PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file, + (void *) km->pm_address); + pmclog_process_map_in(po, (pid_t) -1, km->pm_address, + km->pm_file); + } + FREE(kmbase, M_LINKER); } break; /* * Flush a log file. */ case PMC_OP_FLUSHLOG: { struct pmc_owner *po; sx_assert(&pmc_sx, SX_XLOCKED); if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { error = EINVAL; break; } error = pmclog_flush(po); } break; /* * Retrieve hardware configuration. */ case PMC_OP_GETCPUINFO: /* CPU information */ { struct pmc_op_getcpuinfo gci; gci.pm_cputype = md->pmd_cputype; gci.pm_ncpu = mp_ncpus; gci.pm_npmc = md->pmd_npmc; gci.pm_nclass = md->pmd_nclass; bcopy(md->pmd_classes, &gci.pm_classes, sizeof(gci.pm_classes)); error = copyout(&gci, arg, sizeof(gci)); } break; /* * Get module statistics */ case PMC_OP_GETDRIVERSTATS: { struct pmc_op_getdriverstats gms; bcopy(&pmc_stats, &gms, sizeof(gms)); error = copyout(&gms, arg, sizeof(gms)); } break; /* * Retrieve module version number */ case PMC_OP_GETMODULEVERSION: { uint32_t cv, modv; /* retrieve the client's idea of the ABI version */ if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0) break; /* don't service clients newer than our driver */ modv = PMC_VERSION; if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) { error = EPROGMISMATCH; break; } error = copyout(&modv, arg, sizeof(int)); } break; /* * Retrieve the state of all the PMCs on a given * CPU. */ case PMC_OP_GETPMCINFO: { uint32_t cpu, n, npmc; size_t pmcinfo_size; struct pmc *pm; struct pmc_info *p, *pmcinfo; struct pmc_op_getpmcinfo *gpi; struct pmc_owner *po; struct pmc_binding pb; PMC_DOWNGRADE_SX(); gpi = (struct pmc_op_getpmcinfo *) arg; if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0) break; if (cpu >= (unsigned int) mp_ncpus) { error = EINVAL; break; } if (pmc_cpu_is_disabled(cpu)) { error = ENXIO; break; } /* switch to CPU 'cpu' */ pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); npmc = md->pmd_npmc; pmcinfo_size = npmc * sizeof(struct pmc_info); MALLOC(pmcinfo, struct pmc_info *, pmcinfo_size, M_PMC, M_WAITOK); p = pmcinfo; for (n = 0; n < md->pmd_npmc; n++, p++) { if ((error = md->pmd_describe(cpu, n, p, &pm)) != 0) break; if (PMC_ROW_DISP_IS_STANDALONE(n)) p->pm_rowdisp = PMC_DISP_STANDALONE; else if (PMC_ROW_DISP_IS_THREAD(n)) p->pm_rowdisp = PMC_DISP_THREAD; else p->pm_rowdisp = PMC_DISP_FREE; p->pm_ownerpid = -1; if (pm == NULL) /* no PMC associated */ continue; po = pm->pm_owner; KASSERT(po->po_owner != NULL, ("[pmc,%d] pmc_owner had a null proc pointer", __LINE__)); p->pm_ownerpid = po->po_owner->p_pid; p->pm_mode = PMC_TO_MODE(pm); p->pm_event = pm->pm_event; p->pm_flags = pm->pm_flags; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) p->pm_reloadcount = pm->pm_sc.pm_reloadcount; } pmc_restore_cpu_binding(&pb); /* now copy out the PMC info collected */ if (error == 0) error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size); FREE(pmcinfo, M_PMC); } break; /* * Set the administrative state of a PMC. I.e. whether * the PMC is to be used or not. */ case PMC_OP_PMCADMIN: { int cpu, ri; enum pmc_state request; struct pmc_cpu *pc; struct pmc_hw *phw; struct pmc_op_pmcadmin pma; struct pmc_binding pb; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(td == curthread, ("[pmc,%d] td != curthread", __LINE__)); if (suser(td) || jailed(td->td_ucred)) { error = EPERM; break; } if ((error = copyin(arg, &pma, sizeof(pma))) != 0) break; cpu = pma.pm_cpu; if (cpu < 0 || cpu >= mp_ncpus) { error = EINVAL; break; } if (pmc_cpu_is_disabled(cpu)) { error = ENXIO; break; } request = pma.pm_state; if (request != PMC_STATE_DISABLED && request != PMC_STATE_FREE) { error = EINVAL; break; } ri = pma.pm_pmc; /* pmc id == row index */ if (ri < 0 || ri >= (int) md->pmd_npmc) { error = EINVAL; break; } /* * We can't disable a PMC with a row-index allocated * for process virtual PMCs. */ if (PMC_ROW_DISP_IS_THREAD(ri) && request == PMC_STATE_DISABLED) { error = EBUSY; break; } /* * otherwise, this PMC on this CPU is either free or * in system-wide mode. */ pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); pc = pmc_pcpu[cpu]; phw = pc->pc_hwpmcs[ri]; /* * XXX do we need some kind of 'forced' disable? */ if (phw->phw_pmc == NULL) { if (request == PMC_STATE_DISABLED && (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) { phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED; PMC_MARK_ROW_STANDALONE(ri); } else if (request == PMC_STATE_FREE && (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) { phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED; PMC_UNMARK_ROW_STANDALONE(ri); } /* other cases are a no-op */ } else error = EBUSY; pmc_restore_cpu_binding(&pb); } break; /* * Allocate a PMC. */ case PMC_OP_PMCALLOCATE: { uint32_t caps; u_int cpu; int n; enum pmc_mode mode; struct pmc *pmc; struct pmc_hw *phw; struct pmc_op_pmcallocate pa; struct pmc_binding pb; if ((error = copyin(arg, &pa, sizeof(pa))) != 0) break; caps = pa.pm_caps; mode = pa.pm_mode; cpu = pa.pm_cpu; if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC && mode != PMC_MODE_TS && mode != PMC_MODE_TC) || (cpu != (u_int) PMC_CPU_ANY && cpu >= (u_int) mp_ncpus)) { error = EINVAL; break; } /* * Virtual PMCs should only ask for a default CPU. * System mode PMCs need to specify a non-default CPU. */ if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) || (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) { error = EINVAL; break; } /* * Check that a disabled CPU is not being asked for. */ if (PMC_IS_SYSTEM_MODE(mode) && pmc_cpu_is_disabled(cpu)) { error = ENXIO; break; } /* * Refuse an allocation for a system-wide PMC if this * process has been jailed, or if this process lacks * super-user credentials and the sysctl tunable * 'security.bsd.unprivileged_syspmcs' is zero. */ if (PMC_IS_SYSTEM_MODE(mode)) { if (jailed(curthread->td_ucred)) error = EPERM; else if (suser(curthread) && (pmc_unprivileged_syspmcs == 0)) error = EPERM; } if (error) break; /* * Look for valid values for 'pm_flags' */ if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT)) != 0) { error = EINVAL; break; } /* process logging options are not allowed for system PMCs */ if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags & (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) { error = EINVAL; break; } /* * All sampling mode PMCs need to be able to interrupt the * CPU. */ if (PMC_IS_SAMPLING_MODE(mode)) caps |= PMC_CAP_INTERRUPT; /* A valid class specifier should have been passed in. */ for (n = 0; n < md->pmd_nclass; n++) if (md->pmd_classes[n].pm_class == pa.pm_class) break; if (n == md->pmd_nclass) { error = EINVAL; break; } /* The requested PMC capabilities should be feasible. */ if ((md->pmd_classes[n].pm_caps & caps) != caps) { error = EOPNOTSUPP; break; } PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", pa.pm_ev, caps, mode, cpu); pmc = pmc_allocate_pmc_descriptor(); pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class, PMC_ID_INVALID); pmc->pm_event = pa.pm_ev; pmc->pm_state = PMC_STATE_FREE; pmc->pm_caps = caps; pmc->pm_flags = pa.pm_flags; /* switch thread to CPU 'cpu' */ pmc_save_cpu_binding(&pb); #define PMC_IS_SHAREABLE_PMC(cpu, n) \ (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \ PMC_PHW_FLAG_IS_SHAREABLE) #define PMC_IS_UNALLOCATED(cpu, n) \ (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL) if (PMC_IS_SYSTEM_MODE(mode)) { pmc_select_cpu(cpu); for (n = 0; n < (int) md->pmd_npmc; n++) if (pmc_can_allocate_row(n, mode) == 0 && pmc_can_allocate_rowindex( curthread->td_proc, n, cpu) == 0 && (PMC_IS_UNALLOCATED(cpu, n) || PMC_IS_SHAREABLE_PMC(cpu, n)) && md->pmd_allocate_pmc(cpu, n, pmc, &pa) == 0) break; } else { /* Process virtual mode */ for (n = 0; n < (int) md->pmd_npmc; n++) { if (pmc_can_allocate_row(n, mode) == 0 && pmc_can_allocate_rowindex( curthread->td_proc, n, PMC_CPU_ANY) == 0 && md->pmd_allocate_pmc(curthread->td_oncpu, n, pmc, &pa) == 0) break; } } #undef PMC_IS_UNALLOCATED #undef PMC_IS_SHAREABLE_PMC pmc_restore_cpu_binding(&pb); if (n == (int) md->pmd_npmc) { pmc_destroy_pmc_descriptor(pmc); FREE(pmc, M_PMC); pmc = NULL; error = EINVAL; break; } /* Fill in the correct value in the ID field */ pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n); PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x", pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id); /* Process mode PMCs with logging enabled need log files */ if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW)) pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; /* All system mode sampling PMCs require a log file */ if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode)) pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; /* * Configure global pmc's immediately */ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) { pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); phw = pmc_pcpu[cpu]->pc_hwpmcs[n]; if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 || (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) { (void) md->pmd_release_pmc(cpu, n, pmc); pmc_destroy_pmc_descriptor(pmc); FREE(pmc, M_PMC); pmc = NULL; pmc_restore_cpu_binding(&pb); error = EPERM; break; } pmc_restore_cpu_binding(&pb); } pmc->pm_state = PMC_STATE_ALLOCATED; /* * mark row disposition */ if (PMC_IS_SYSTEM_MODE(mode)) PMC_MARK_ROW_STANDALONE(n); else PMC_MARK_ROW_THREAD(n); /* * Register this PMC with the current thread as its owner. */ if ((error = pmc_register_owner(curthread->td_proc, pmc)) != 0) { pmc_release_pmc_descriptor(pmc); FREE(pmc, M_PMC); pmc = NULL; break; } /* * Return the allocated index. */ pa.pm_pmcid = pmc->pm_id; error = copyout(&pa, arg, sizeof(pa)); } break; /* * Attach a PMC to a process. */ case PMC_OP_PMCATTACH: { struct pmc *pm; struct proc *p; struct pmc_op_pmcattach a; sx_assert(&pmc_sx, SX_XLOCKED); if ((error = copyin(arg, &a, sizeof(a))) != 0) break; if (a.pm_pid < 0) { error = EINVAL; break; } else if (a.pm_pid == 0) a.pm_pid = td->td_proc->p_pid; if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) break; if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { error = EINVAL; break; } /* PMCs may be (re)attached only when allocated or stopped */ if (pm->pm_state == PMC_STATE_RUNNING) { error = EBUSY; break; } else if (pm->pm_state != PMC_STATE_ALLOCATED && pm->pm_state != PMC_STATE_STOPPED) { error = EINVAL; break; } /* lookup pid */ if ((p = pfind(a.pm_pid)) == NULL) { error = ESRCH; break; } /* * Ignore processes that are working on exiting. */ if (p->p_flag & P_WEXIT) { error = ESRCH; PROC_UNLOCK(p); /* pfind() returns a locked process */ break; } /* * we are allowed to attach a PMC to a process if * we can debug it. */ error = p_candebug(curthread, p); PROC_UNLOCK(p); if (error == 0) error = pmc_attach_process(p, pm); } break; /* * Detach an attached PMC from a process. */ case PMC_OP_PMCDETACH: { struct pmc *pm; struct proc *p; struct pmc_op_pmcattach a; if ((error = copyin(arg, &a, sizeof(a))) != 0) break; if (a.pm_pid < 0) { error = EINVAL; break; } else if (a.pm_pid == 0) a.pm_pid = td->td_proc->p_pid; if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) break; if ((p = pfind(a.pm_pid)) == NULL) { error = ESRCH; break; } /* * Treat processes that are in the process of exiting * as if they were not present. */ if (p->p_flag & P_WEXIT) error = ESRCH; PROC_UNLOCK(p); /* pfind() returns a locked process */ if (error == 0) error = pmc_detach_process(p, pm); } break; /* * Retrieve the MSR number associated with the counter * 'pmc_id'. This allows processes to directly use RDPMC * instructions to read their PMCs, without the overhead of a * system call. */ case PMC_OP_PMCGETMSR: { int ri; struct pmc *pm; struct pmc_target *pt; struct pmc_op_getmsr gm; PMC_DOWNGRADE_SX(); /* CPU has no 'GETMSR' support */ if (md->pmd_get_msr == NULL) { error = ENOSYS; break; } if ((error = copyin(arg, &gm, sizeof(gm))) != 0) break; if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) break; /* * The allocated PMC has to be a process virtual PMC, * i.e., of type MODE_T[CS]. Global PMCs can only be * read using the PMCREAD operation since they may be * allocated on a different CPU than the one we could * be running on at the time of the RDPMC instruction. * * The GETMSR operation is not allowed for PMCs that * are inherited across processes. */ if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) || (pm->pm_flags & PMC_F_DESCENDANTS)) { error = EINVAL; break; } /* * It only makes sense to use a RDPMC (or its * equivalent instruction on non-x86 architectures) on * a process that has allocated and attached a PMC to * itself. Conversely the PMC is only allowed to have * one process attached to it -- its owner. */ if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || LIST_NEXT(pt, pt_next) != NULL || pt->pt_process->pp_proc != pm->pm_owner->po_owner) { error = EINVAL; break; } ri = PMC_TO_ROWINDEX(pm); if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0) break; if ((error = copyout(&gm, arg, sizeof(gm))) < 0) break; /* * Mark our process as using MSRs. Update machine * state using a forced context switch. */ pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS; pmc_force_context_switch(); } break; /* * Release an allocated PMC */ case PMC_OP_PMCRELEASE: { pmc_id_t pmcid; struct pmc *pm; struct pmc_owner *po; struct pmc_op_simple sp; /* * Find PMC pointer for the named PMC. * * Use pmc_release_pmc_descriptor() to switch off the * PMC, remove all its target threads, and remove the * PMC from its owner's list. * * Remove the owner record if this is the last PMC * owned. * * Free up space. */ if ((error = copyin(arg, &sp, sizeof(sp))) != 0) break; pmcid = sp.pm_pmcid; if ((error = pmc_find_pmc(pmcid, &pm)) != 0) break; po = pm->pm_owner; pmc_release_pmc_descriptor(pm); pmc_maybe_remove_owner(po); FREE(pm, M_PMC); } break; /* * Read and/or write a PMC. */ case PMC_OP_PMCRW: { uint32_t cpu, ri; struct pmc *pm; struct pmc_op_pmcrw *pprw; struct pmc_op_pmcrw prw; struct pmc_binding pb; pmc_value_t oldvalue; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &prw, sizeof(prw))) != 0) break; ri = 0; PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid, prw.pm_flags); /* must have at least one flag set */ if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) { error = EINVAL; break; } /* locate pmc descriptor */ if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0) break; /* Can't read a PMC that hasn't been started. */ if (pm->pm_state != PMC_STATE_ALLOCATED && pm->pm_state != PMC_STATE_STOPPED && pm->pm_state != PMC_STATE_RUNNING) { error = EINVAL; break; } /* writing a new value is allowed only for 'STOPPED' pmcs */ if (pm->pm_state == PMC_STATE_RUNNING && (prw.pm_flags & PMC_F_NEWVALUE)) { error = EBUSY; break; } if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) { /* * If this PMC is attached to its owner (i.e., * the process requesting this operation) and * is running, then attempt to get an * upto-date reading from hardware for a READ. * Writes are only allowed when the PMC is * stopped, so only update the saved value * field. * * If the PMC is not running, or is not * attached to its owner, read/write to the * savedvalue field. */ ri = PMC_TO_ROWINDEX(pm); mtx_pool_lock_spin(pmc_mtxpool, pm); cpu = curthread->td_oncpu; if (prw.pm_flags & PMC_F_OLDVALUE) { if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && (pm->pm_state == PMC_STATE_RUNNING)) error = (*md->pmd_read_pmc)(cpu, ri, &oldvalue); else oldvalue = pm->pm_gv.pm_savedvalue; } if (prw.pm_flags & PMC_F_NEWVALUE) pm->pm_gv.pm_savedvalue = prw.pm_value; mtx_pool_unlock_spin(pmc_mtxpool, pm); } else { /* System mode PMCs */ cpu = PMC_TO_CPU(pm); ri = PMC_TO_ROWINDEX(pm); if (pmc_cpu_is_disabled(cpu)) { error = ENXIO; break; } /* move this thread to CPU 'cpu' */ pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); critical_enter(); /* save old value */ if (prw.pm_flags & PMC_F_OLDVALUE) if ((error = (*md->pmd_read_pmc)(cpu, ri, &oldvalue))) goto error; /* write out new value */ if (prw.pm_flags & PMC_F_NEWVALUE) error = (*md->pmd_write_pmc)(cpu, ri, prw.pm_value); error: critical_exit(); pmc_restore_cpu_binding(&pb); if (error) break; } pprw = (struct pmc_op_pmcrw *) arg; #ifdef DEBUG if (prw.pm_flags & PMC_F_NEWVALUE) PMCDBG(PMC,OPS,2, "rw id=%d new %jx -> old %jx", ri, prw.pm_value, oldvalue); else if (prw.pm_flags & PMC_F_OLDVALUE) PMCDBG(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue); #endif /* return old value if requested */ if (prw.pm_flags & PMC_F_OLDVALUE) if ((error = copyout(&oldvalue, &pprw->pm_value, sizeof(prw.pm_value)))) break; } break; /* * Set the sampling rate for a sampling mode PMC and the * initial count for a counting mode PMC. */ case PMC_OP_PMCSETCOUNT: { struct pmc *pm; struct pmc_op_pmcsetcount sc; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &sc, sizeof(sc))) != 0) break; if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0) break; if (pm->pm_state == PMC_STATE_RUNNING) { error = EBUSY; break; } if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pm->pm_sc.pm_reloadcount = sc.pm_count; else pm->pm_sc.pm_initial = sc.pm_count; } break; /* * Start a PMC. */ case PMC_OP_PMCSTART: { pmc_id_t pmcid; struct pmc *pm; struct pmc_op_simple sp; sx_assert(&pmc_sx, SX_XLOCKED); if ((error = copyin(arg, &sp, sizeof(sp))) != 0) break; pmcid = sp.pm_pmcid; if ((error = pmc_find_pmc(pmcid, &pm)) != 0) break; KASSERT(pmcid == pm->pm_id, ("[pmc,%d] pmcid %x != id %x", __LINE__, pm->pm_id, pmcid)); if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ break; else if (pm->pm_state != PMC_STATE_STOPPED && pm->pm_state != PMC_STATE_ALLOCATED) { error = EINVAL; break; } error = pmc_start(pm); } break; /* * Stop a PMC. */ case PMC_OP_PMCSTOP: { pmc_id_t pmcid; struct pmc *pm; struct pmc_op_simple sp; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &sp, sizeof(sp))) != 0) break; pmcid = sp.pm_pmcid; /* * Mark the PMC as inactive and invoke the MD stop * routines if needed. */ if ((error = pmc_find_pmc(pmcid, &pm)) != 0) break; KASSERT(pmcid == pm->pm_id, ("[pmc,%d] pmc id %x != pmcid %x", __LINE__, pm->pm_id, pmcid)); if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ break; else if (pm->pm_state != PMC_STATE_RUNNING) { error = EINVAL; break; } error = pmc_stop(pm); } break; /* * Write a user supplied value to the log file. */ case PMC_OP_WRITELOG: { struct pmc_op_writelog wl; struct pmc_owner *po; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &wl, sizeof(wl))) != 0) break; if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { error = EINVAL; break; } if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) { error = EINVAL; break; } error = pmclog_process_userlog(po, &wl); } break; default: error = EINVAL; break; } if (is_sx_downgraded) sx_sunlock(&pmc_sx); else sx_xunlock(&pmc_sx); if (error) atomic_add_int(&pmc_stats.pm_syscall_errors, 1); PICKUP_GIANT(); return error; } /* * Helper functions */ /* * Interrupt processing. * * Find a free slot in the per-cpu array of PC samples and write the * current (PMC,PID,PC) triple to it. If an event was successfully * added, a bit is set in mask 'pmc_cpumask' denoting that the * DO_SAMPLES hook needs to be invoked from the clock handler. * * This function is meant to be called from an NMI handler. It cannot * use any of the locking primitives supplied by the OS. */ int pmc_process_interrupt(int cpu, struct pmc *pm, uintfptr_t pc, int usermode) { int error, ri; struct thread *td; struct pmc_sample *ps; struct pmc_samplebuffer *psb; error = 0; ri = PMC_TO_ROWINDEX(pm); psb = pmc_pcpu[cpu]->pc_sb; ps = psb->ps_write; if (ps->ps_pc) { /* in use, reader hasn't caught up */ pm->pm_stalled = 1; atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1); PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, pm, (uint64_t) pc, usermode, (int) (psb->ps_write - psb->ps_samples), (int) (psb->ps_read - psb->ps_samples)); error = ENOMEM; goto done; } /* fill in entry */ PMCDBG(SAM,INT,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, pm, (uint64_t) pc, usermode, (int) (psb->ps_write - psb->ps_samples), (int) (psb->ps_read - psb->ps_samples)); atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */ ps->ps_pmc = pm; if ((td = curthread) && td->td_proc) ps->ps_pid = td->td_proc->p_pid; else ps->ps_pid = -1; ps->ps_usermode = usermode; ps->ps_pc = pc; /* mark entry as in use */ /* increment write pointer, modulo ring buffer size */ ps++; if (ps == psb->ps_fence) psb->ps_write = psb->ps_samples; else psb->ps_write = ps; done: /* mark CPU as needing processing */ atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); return error; } /* * Process saved PC samples. */ static void pmc_process_samples(int cpu) { int n, ri; struct pmc *pm; struct thread *td; struct pmc_owner *po; struct pmc_sample *ps; struct pmc_samplebuffer *psb; KASSERT(PCPU_GET(cpuid) == cpu, ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, PCPU_GET(cpuid), cpu)); psb = pmc_pcpu[cpu]->pc_sb; for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */ ps = psb->ps_read; if (ps->ps_pc == (uintfptr_t) 0) /* no data */ break; pm = ps->ps_pmc; po = pm->pm_owner; KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__, pm, PMC_TO_MODE(pm))); /* Ignore PMCs that have been switched off */ if (pm->pm_state != PMC_STATE_RUNNING) goto entrydone; PMCDBG(SAM,OPS,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, pm, (uint64_t) ps->ps_pc, ps->ps_usermode, (int) (psb->ps_write - psb->ps_samples), (int) (psb->ps_read - psb->ps_samples)); /* * If this is a process-mode PMC that is attached to * its owner, and if the PC is in user mode, update * profiling statistics like timer-based profiling * would have done. */ if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { if (ps->ps_usermode) { td = FIRST_THREAD_IN_PROC(po->po_owner); addupc_intr(td, ps->ps_pc, 1); } goto entrydone; } /* * Otherwise, this is either a sampling mode PMC that * is attached to a different process than its owner, * or a system-wide sampling PMC. Dispatch a log * entry to the PMC's owner process. */ pmclog_process_pcsample(pm, ps); entrydone: ps->ps_pc = (uintfptr_t) 0; /* mark entry as free */ atomic_subtract_rel_32(&pm->pm_runcount, 1); /* increment read pointer, modulo sample size */ if (++ps == psb->ps_fence) psb->ps_read = psb->ps_samples; else psb->ps_read = ps; } atomic_add_int(&pmc_stats.pm_log_sweeps, 1); /* Do not re-enable stalled PMCs if we failed to process any samples */ if (n == 0) return; /* * Restart any stalled sampling PMCs on this CPU. * * If the NMI handler sets the pm_stalled field of a PMC after * the check below, we'll end up processing the stalled PMC at * the next hardclock tick. */ for (n = 0; n < md->pmd_npmc; n++) { (void) (*md->pmd_get_config)(cpu,n,&pm); if (pm == NULL || /* !cfg'ed */ pm->pm_state != PMC_STATE_RUNNING || /* !active */ !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ pm->pm_stalled == 0) /* !stalled */ continue; pm->pm_stalled = 0; ri = PMC_TO_ROWINDEX(pm); (*md->pmd_start_pmc)(cpu, ri); } } /* * Event handlers. */ /* * Handle a process exit. * * Remove this process from all hash tables. If this process * owned any PMCs, turn off those PMCs and deallocate them, * removing any associations with target processes. * * This function will be called by the last 'thread' of a * process. * * XXX This eventhandler gets called early in the exit process. * Consider using a 'hook' invocation from thread_exit() or equivalent * spot. Another negative is that kse_exit doesn't seem to call * exit1() [??]. * */ static void pmc_process_exit(void *arg __unused, struct proc *p) { int is_using_hwpmcs; int cpu; unsigned int ri; struct pmc *pm; struct pmc_process *pp; struct pmc_owner *po; pmc_value_t newvalue, tmp; PROC_LOCK(p); is_using_hwpmcs = p->p_flag & P_HWPMC; PROC_UNLOCK(p); /* * Log a sysexit event to all SS PMC owners. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_sysexit(po, p->p_pid); if (!is_using_hwpmcs) return; PMC_GET_SX_XLOCK(); PMCDBG(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid, p->p_comm); /* * Since this code is invoked by the last thread in an exiting * process, we would have context switched IN at some prior * point. However, with PREEMPTION, kernel mode context * switches may happen any time, so we want to disable a * context switch OUT till we get any PMCs targetting this * process off the hardware. * * We also need to atomically remove this process' * entry from our target process hash table, using * PMC_FLAG_REMOVE. */ PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid, p->p_comm); critical_enter(); /* no preemption */ cpu = curthread->td_oncpu; if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_REMOVE)) != NULL) { PMCDBG(PRC,EXT,2, "process-exit proc=%p pmc-process=%p", p, pp); /* * The exiting process could the target of * some PMCs which will be running on * currently executing CPU. * * We need to turn these PMCs off like we * would do at context switch OUT time. */ for (ri = 0; ri < md->pmd_npmc; ri++) { /* * Pick up the pmc pointer from hardware * state similar to the CSW_OUT code. */ pm = NULL; (void) (*md->pmd_get_config)(cpu, ri, &pm); PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); if (pm == NULL || !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) continue; PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " "state=%d", ri, pp->pp_pmcs[ri].pp_pmc, pm, pm->pm_state); KASSERT(PMC_TO_ROWINDEX(pm) == ri, ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", __LINE__, PMC_TO_ROWINDEX(pm), ri)); KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); (void) md->pmd_stop_pmc(cpu, ri); KASSERT(pm->pm_runcount > 0, ("[pmc,%d] bad runcount ri %d rc %d", __LINE__, ri, pm->pm_runcount)); /* Stop hardware only if it is actually running */ if (pm->pm_state == PMC_STATE_RUNNING && pm->pm_stalled == 0) { md->pmd_read_pmc(cpu, ri, &newvalue); tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); mtx_pool_lock_spin(pmc_mtxpool, pm); pm->pm_gv.pm_savedvalue += tmp; pp->pp_pmcs[ri].pp_pmcval += tmp; mtx_pool_unlock_spin(pmc_mtxpool, pm); } atomic_subtract_rel_32(&pm->pm_runcount,1); KASSERT((int) pm->pm_runcount >= 0, ("[pmc,%d] runcount is %d", __LINE__, ri)); (void) md->pmd_config_pmc(cpu, ri, NULL); } /* * Inform the MD layer of this pseudo "context switch * out" */ (void) md->pmd_switch_out(pmc_pcpu[cpu], pp); critical_exit(); /* ok to be pre-empted now */ /* * Unlink this process from the PMCs that are * targetting it. This will send a signal to * all PMC owner's whose PMCs are orphaned. * * Log PMC value at exit time if requested. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) pmclog_process_procexit(pm, pp); pmc_unlink_target_process(pm, pp); } FREE(pp, M_PMC); } else critical_exit(); /* pp == NULL */ /* * If the process owned PMCs, free them up and free up * memory. */ if ((po = pmc_find_owner_descriptor(p)) != NULL) { pmc_remove_owner(po); pmc_destroy_owner_descriptor(po); } sx_xunlock(&pmc_sx); } /* * Handle a process fork. * * If the parent process 'p1' is under HWPMC monitoring, then copy * over any attached PMCs that have 'do_descendants' semantics. */ static void pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc, int flags) { int is_using_hwpmcs; unsigned int ri; uint32_t do_descendants; struct pmc *pm; struct pmc_owner *po; struct pmc_process *ppnew, *ppold; (void) flags; /* unused parameter */ PROC_LOCK(p1); is_using_hwpmcs = p1->p_flag & P_HWPMC; PROC_UNLOCK(p1); /* * If there are system-wide sampling PMCs active, we need to * log all fork events to their owner's logs. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procfork(po, p1->p_pid, newproc->p_pid); if (!is_using_hwpmcs) return; PMC_GET_SX_XLOCK(); PMCDBG(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1, p1->p_pid, p1->p_comm, newproc); /* * If the parent process (curthread->td_proc) is a * target of any PMCs, look for PMCs that are to be * inherited, and link these into the new process * descriptor. */ if ((ppold = pmc_find_process_descriptor(curthread->td_proc, PMC_FLAG_NONE)) == NULL) goto done; /* nothing to do */ do_descendants = 0; for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL) do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS; if (do_descendants == 0) /* nothing to do */ goto done; /* allocate a descriptor for the new process */ if ((ppnew = pmc_find_process_descriptor(newproc, PMC_FLAG_ALLOCATE)) == NULL) goto done; /* * Run through all PMCs that were targeting the old process * and which specified F_DESCENDANTS and attach them to the * new process. * * Log the fork event to all owners of PMCs attached to this * process, if not already logged. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL && (pm->pm_flags & PMC_F_DESCENDANTS)) { pmc_link_target_process(pm, ppnew); po = pm->pm_owner; if (po->po_sscount == 0 && po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procfork(po, p1->p_pid, newproc->p_pid); } /* * Now mark the new process as being tracked by this driver. */ PROC_LOCK(newproc); newproc->p_flag |= P_HWPMC; PROC_UNLOCK(newproc); done: sx_xunlock(&pmc_sx); } /* * initialization */ static const char *pmc_name_of_pmcclass[] = { #undef __PMC_CLASS #define __PMC_CLASS(N) #N , __PMC_CLASSES() }; static int pmc_initialize(void) { int cpu, error, n; struct pmc_binding pb; struct pmc_samplebuffer *sb; md = NULL; error = 0; #ifdef DEBUG /* parse debug flags first */ if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, sizeof(pmc_debugstr))) pmc_debugflags_parse(pmc_debugstr, pmc_debugstr+strlen(pmc_debugstr)); #endif PMCDBG(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION); /* check kernel version */ if (pmc_kernel_version != PMC_VERSION) { if (pmc_kernel_version == 0) printf("hwpmc: this kernel has not been compiled with " "'options HWPMC_HOOKS'.\n"); else printf("hwpmc: kernel version (0x%x) does not match " "module version (0x%x).\n", pmc_kernel_version, PMC_VERSION); return EPROGMISMATCH; } /* * check sysctl parameters */ if (pmc_hashsize <= 0) { (void) printf("hwpmc: tunable hashsize=%d must be greater " "than zero.\n", pmc_hashsize); pmc_hashsize = PMC_HASH_SIZE; } if (pmc_nsamples <= 0 || pmc_nsamples > 65535) { (void) printf("hwpmc: tunable nsamples=%d out of range.\n", pmc_nsamples); pmc_nsamples = PMC_NSAMPLES; } md = pmc_md_initialize(); if (md == NULL || md->pmd_init == NULL) return ENOSYS; /* allocate space for the per-cpu array */ MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *), M_PMC, M_WAITOK|M_ZERO); /* per-cpu 'saved values' for managing process-mode PMCs */ MALLOC(pmc_pcpu_saved, pmc_value_t *, sizeof(pmc_value_t) * mp_ncpus * md->pmd_npmc, M_PMC, M_WAITOK); /* perform cpu dependent initialization */ pmc_save_cpu_binding(&pb); for (cpu = 0; cpu < mp_ncpus; cpu++) { if (pmc_cpu_is_disabled(cpu)) continue; pmc_select_cpu(cpu); if ((error = md->pmd_init(cpu)) != 0) break; } pmc_restore_cpu_binding(&pb); if (error != 0) return error; /* allocate space for the sample array */ for (cpu = 0; cpu < mp_ncpus; cpu++) { if (pmc_cpu_is_disabled(cpu)) continue; MALLOC(sb, struct pmc_samplebuffer *, sizeof(struct pmc_samplebuffer) + pmc_nsamples * sizeof(struct pmc_sample), M_PMC, M_WAITOK|M_ZERO); sb->ps_read = sb->ps_write = sb->ps_samples; sb->ps_fence = sb->ps_samples + pmc_nsamples; KASSERT(pmc_pcpu[cpu] != NULL, ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); pmc_pcpu[cpu]->pc_sb = sb; } /* allocate space for the row disposition array */ pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO); KASSERT(pmc_pmcdisp != NULL, ("[pmc,%d] pmcdisp allocation returned NULL", __LINE__)); /* mark all PMCs as available */ for (n = 0; n < (int) md->pmd_npmc; n++) PMC_MARK_ROW_FREE(n); /* allocate thread hash tables */ pmc_ownerhash = hashinit(pmc_hashsize, M_PMC, &pmc_ownerhashmask); pmc_processhash = hashinit(pmc_hashsize, M_PMC, &pmc_processhashmask); mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc", MTX_SPIN); LIST_INIT(&pmc_ss_owners); pmc_ss_count = 0; /* allocate a pool of spin mutexes */ pmc_mtxpool = mtx_pool_create("pmc", pmc_mtxpool_size, MTX_SPIN); PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx " "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask, pmc_processhash, pmc_processhashmask); /* register process {exit,fork,exec} handlers */ pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit, pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY); pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork, pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY); /* initialize logging */ pmclog_initialize(); /* set hook functions */ pmc_intr = md->pmd_intr; pmc_hook = pmc_hook_handler; if (error == 0) { printf(PMC_MODULE_NAME ":"); for (n = 0; n < (int) md->pmd_nclass; n++) { printf(" %s/%d/0x%b", pmc_name_of_pmcclass[md->pmd_classes[n].pm_class], md->pmd_nclasspmcs[n], md->pmd_classes[n].pm_caps, "\20" "\1INT\2USR\3SYS\4EDG\5THR" "\6REA\7WRI\10INV\11QUA\12PRC" "\13TAG\14CSC"); } printf("\n"); } return error; } /* prepare to be unloaded */ static void pmc_cleanup(void) { int cpu; struct pmc_ownerhash *ph; struct pmc_owner *po, *tmp; struct pmc_binding pb; #ifdef DEBUG struct pmc_processhash *prh; #endif PMCDBG(MOD,INI,0, "%s", "cleanup"); /* switch off sampling */ atomic_store_rel_int(&pmc_cpumask, 0); pmc_intr = NULL; sx_xlock(&pmc_sx); if (pmc_hook == NULL) { /* being unloaded already */ sx_xunlock(&pmc_sx); return; } pmc_hook = NULL; /* prevent new threads from entering module */ /* deregister event handlers */ EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag); EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag); /* send SIGBUS to all owner threads, free up allocations */ if (pmc_ownerhash) for (ph = pmc_ownerhash; ph <= &pmc_ownerhash[pmc_ownerhashmask]; ph++) { LIST_FOREACH_SAFE(po, ph, po_next, tmp) { pmc_remove_owner(po); /* send SIGBUS to owner processes */ PMCDBG(MOD,INI,2, "cleanup signal proc=%p " "(%d, %s)", po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm); PROC_LOCK(po->po_owner); psignal(po->po_owner, SIGBUS); PROC_UNLOCK(po->po_owner); pmc_destroy_owner_descriptor(po); } } /* reclaim allocated data structures */ if (pmc_mtxpool) mtx_pool_destroy(&pmc_mtxpool); mtx_destroy(&pmc_processhash_mtx); if (pmc_processhash) { #ifdef DEBUG struct pmc_process *pp; PMCDBG(MOD,INI,3, "%s", "destroy process hash"); for (prh = pmc_processhash; prh <= &pmc_processhash[pmc_processhashmask]; prh++) LIST_FOREACH(pp, prh, pp_next) PMCDBG(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid); #endif hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask); pmc_processhash = NULL; } if (pmc_ownerhash) { PMCDBG(MOD,INI,3, "%s", "destroy owner hash"); hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask); pmc_ownerhash = NULL; } KASSERT(LIST_EMPTY(&pmc_ss_owners), ("[pmc,%d] Global SS owner list not empty", __LINE__)); KASSERT(pmc_ss_count == 0, ("[pmc,%d] Global SS count not empty", __LINE__)); /* free the per-cpu sample buffers */ for (cpu = 0; cpu < mp_ncpus; cpu++) { if (pmc_cpu_is_disabled(cpu)) continue; KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, cpu)); FREE(pmc_pcpu[cpu]->pc_sb, M_PMC); pmc_pcpu[cpu]->pc_sb = NULL; } /* do processor dependent cleanup */ PMCDBG(MOD,INI,3, "%s", "md cleanup"); if (md) { pmc_save_cpu_binding(&pb); for (cpu = 0; cpu < mp_ncpus; cpu++) { PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p", cpu, pmc_pcpu[cpu]); if (pmc_cpu_is_disabled(cpu)) continue; pmc_select_cpu(cpu); if (pmc_pcpu[cpu]) (void) md->pmd_cleanup(cpu); } FREE(md, M_PMC); md = NULL; pmc_restore_cpu_binding(&pb); } /* deallocate per-cpu structures */ FREE(pmc_pcpu, M_PMC); pmc_pcpu = NULL; FREE(pmc_pcpu_saved, M_PMC); pmc_pcpu_saved = NULL; if (pmc_pmcdisp) { FREE(pmc_pmcdisp, M_PMC); pmc_pmcdisp = NULL; } pmclog_shutdown(); sx_xunlock(&pmc_sx); /* we are done */ } /* * The function called at load/unload. */ static int load (struct module *module __unused, int cmd, void *arg __unused) { int error; error = 0; switch (cmd) { case MOD_LOAD : /* initialize the subsystem */ error = pmc_initialize(); if (error != 0) break; PMCDBG(MOD,INI,1, "syscall=%d ncpus=%d", pmc_syscall_num, mp_ncpus); break; case MOD_UNLOAD : case MOD_SHUTDOWN: pmc_cleanup(); PMCDBG(MOD,INI,1, "%s", "unloaded"); break; default : error = EINVAL; /* XXX should panic(9) */ break; } return error; } /* memory pool */ MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module"); Index: head/sys/kern/kern_linker.c =================================================================== --- head/sys/kern/kern_linker.c (revision 157143) +++ head/sys/kern/kern_linker.c (revision 157144) @@ -1,1858 +1,1936 @@ /*- * Copyright (c) 1997-2000 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" +#include "opt_hwpmc_hooks.h" #include "opt_mac.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "linker_if.h" +#ifdef HWPMC_HOOKS +#include +#endif + #ifdef KLD_DEBUG int kld_debug = 0; #endif /* * static char *linker_search_path(const char *name, struct mod_depend * *verinfo); */ static const char *linker_basename(const char *path); /* Metadata from the static kernel */ SET_DECLARE(modmetadata_set, struct mod_metadata); MALLOC_DEFINE(M_LINKER, "linker", "kernel linker"); linker_file_t linker_kernel_file; static struct mtx kld_mtx; /* kernel linker mutex */ static linker_class_list_t classes; static linker_file_list_t linker_files; static int next_file_id = 1; static int linker_no_more_classes = 0; #define LINKER_GET_NEXT_FILE_ID(a) do { \ linker_file_t lftmp; \ \ retry: \ mtx_lock(&kld_mtx); \ TAILQ_FOREACH(lftmp, &linker_files, link) { \ if (next_file_id == lftmp->id) { \ next_file_id++; \ mtx_unlock(&kld_mtx); \ goto retry; \ } \ } \ (a) = next_file_id; \ mtx_unlock(&kld_mtx); /* Hold for safe read of id variable */ \ } while(0) /* XXX wrong name; we're looking at version provision tags here, not modules */ typedef TAILQ_HEAD(, modlist) modlisthead_t; struct modlist { TAILQ_ENTRY(modlist) link; /* chain together all modules */ linker_file_t container; const char *name; int version; }; typedef struct modlist *modlist_t; static modlisthead_t found_modules; static modlist_t modlist_lookup2(const char *name, struct mod_depend *verinfo); static char * linker_strdup(const char *str) { char *result; if ((result = malloc((strlen(str) + 1), M_LINKER, M_WAITOK)) != NULL) strcpy(result, str); return (result); } static void linker_init(void *arg) { mtx_init(&kld_mtx, "kernel linker", NULL, MTX_DEF); TAILQ_INIT(&classes); TAILQ_INIT(&linker_files); } SYSINIT(linker, SI_SUB_KLD, SI_ORDER_FIRST, linker_init, 0) static void linker_stop_class_add(void *arg) { linker_no_more_classes = 1; } SYSINIT(linker_class, SI_SUB_KLD, SI_ORDER_ANY, linker_stop_class_add, NULL) int linker_add_class(linker_class_t lc) { /* * We disallow any class registration past SI_ORDER_ANY * of SI_SUB_KLD. We bump the reference count to keep the * ops from being freed. */ if (linker_no_more_classes == 1) return (EPERM); kobj_class_compile((kobj_class_t) lc); ((kobj_class_t)lc)->refs++; /* XXX: kobj_mtx */ TAILQ_INSERT_TAIL(&classes, lc, link); return (0); } static void linker_file_sysinit(linker_file_t lf) { struct sysinit **start, **stop, **sipp, **xipp, *save; KLD_DPF(FILE, ("linker_file_sysinit: calling SYSINITs for %s\n", lf->filename)); if (linker_file_lookup_set(lf, "sysinit_set", &start, &stop, NULL) != 0) return; /* * Perform a bubble sort of the system initialization objects by * their subsystem (primary key) and order (secondary key). * * Since some things care about execution order, this is the operation * which ensures continued function. */ for (sipp = start; sipp < stop; sipp++) { for (xipp = sipp + 1; xipp < stop; xipp++) { if ((*sipp)->subsystem < (*xipp)->subsystem || ((*sipp)->subsystem == (*xipp)->subsystem && (*sipp)->order <= (*xipp)->order)) continue; /* skip */ save = *sipp; *sipp = *xipp; *xipp = save; } } /* * Traverse the (now) ordered list of system initialization tasks. * Perform each task, and continue on to the next task. */ for (sipp = start; sipp < stop; sipp++) { if ((*sipp)->subsystem == SI_SUB_DUMMY) continue; /* skip dummy task(s) */ /* Call function */ (*((*sipp)->func)) ((*sipp)->udata); } } static void linker_file_sysuninit(linker_file_t lf) { struct sysinit **start, **stop, **sipp, **xipp, *save; KLD_DPF(FILE, ("linker_file_sysuninit: calling SYSUNINITs for %s\n", lf->filename)); if (linker_file_lookup_set(lf, "sysuninit_set", &start, &stop, NULL) != 0) return; /* * Perform a reverse bubble sort of the system initialization objects * by their subsystem (primary key) and order (secondary key). * * Since some things care about execution order, this is the operation * which ensures continued function. */ for (sipp = start; sipp < stop; sipp++) { for (xipp = sipp + 1; xipp < stop; xipp++) { if ((*sipp)->subsystem > (*xipp)->subsystem || ((*sipp)->subsystem == (*xipp)->subsystem && (*sipp)->order >= (*xipp)->order)) continue; /* skip */ save = *sipp; *sipp = *xipp; *xipp = save; } } /* * Traverse the (now) ordered list of system initialization tasks. * Perform each task, and continue on to the next task. */ for (sipp = start; sipp < stop; sipp++) { if ((*sipp)->subsystem == SI_SUB_DUMMY) continue; /* skip dummy task(s) */ /* Call function */ (*((*sipp)->func)) ((*sipp)->udata); } } static void linker_file_register_sysctls(linker_file_t lf) { struct sysctl_oid **start, **stop, **oidp; KLD_DPF(FILE, ("linker_file_register_sysctls: registering SYSCTLs for %s\n", lf->filename)); if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0) return; for (oidp = start; oidp < stop; oidp++) sysctl_register_oid(*oidp); } static void linker_file_unregister_sysctls(linker_file_t lf) { struct sysctl_oid **start, **stop, **oidp; KLD_DPF(FILE, ("linker_file_unregister_sysctls: registering SYSCTLs" " for %s\n", lf->filename)); if (linker_file_lookup_set(lf, "sysctl_set", &start, &stop, NULL) != 0) return; for (oidp = start; oidp < stop; oidp++) sysctl_unregister_oid(*oidp); } static int linker_file_register_modules(linker_file_t lf) { struct mod_metadata **start, **stop, **mdp; const moduledata_t *moddata; int first_error, error; KLD_DPF(FILE, ("linker_file_register_modules: registering modules" " in %s\n", lf->filename)); if (linker_file_lookup_set(lf, "modmetadata_set", &start, &stop, 0) != 0) { /* * This fallback should be unnecessary, but if we get booted * from boot2 instead of loader and we are missing our * metadata then we have to try the best we can. */ if (lf == linker_kernel_file) { start = SET_BEGIN(modmetadata_set); stop = SET_LIMIT(modmetadata_set); } else return (0); } first_error = 0; for (mdp = start; mdp < stop; mdp++) { if ((*mdp)->md_type != MDT_MODULE) continue; moddata = (*mdp)->md_data; KLD_DPF(FILE, ("Registering module %s in %s\n", moddata->name, lf->filename)); error = module_register(moddata, lf); if (error) { printf("Module %s failed to register: %d\n", moddata->name, error); if (first_error == 0) first_error = error; } } return (first_error); } static void linker_init_kernel_modules(void) { linker_file_register_modules(linker_kernel_file); } SYSINIT(linker_kernel, SI_SUB_KLD, SI_ORDER_ANY, linker_init_kernel_modules, 0) static int linker_load_file(const char *filename, linker_file_t *result) { linker_class_t lc; linker_file_t lf; int foundfile, error = 0; /* Refuse to load modules if securelevel raised */ if (securelevel > 0) return (EPERM); lf = linker_find_file_by_name(filename); if (lf) { KLD_DPF(FILE, ("linker_load_file: file %s is already loaded," " incrementing refs\n", filename)); *result = lf; lf->refs++; goto out; } lf = NULL; foundfile = 0; /* * We do not need to protect (lock) classes here because there is * no class registration past startup (SI_SUB_KLD, SI_ORDER_ANY) * and there is no class deregistration mechanism at this time. */ TAILQ_FOREACH(lc, &classes, link) { KLD_DPF(FILE, ("linker_load_file: trying to load %s\n", filename)); error = LINKER_LOAD_FILE(lc, filename, &lf); /* * If we got something other than ENOENT, then it exists but * we cannot load it for some other reason. */ if (error != ENOENT) foundfile = 1; if (lf) { error = linker_file_register_modules(lf); if (error == EEXIST) { linker_file_unload(lf, LINKER_UNLOAD_FORCE); goto out; } linker_file_register_sysctls(lf); linker_file_sysinit(lf); lf->flags |= LINKER_FILE_LINKED; *result = lf; error = 0; goto out; } } /* * Less than ideal, but tells the user whether it failed to load or * the module was not found. */ if (foundfile) { /* * Format not recognized or otherwise unloadable. * When loading a module that is statically built into * the kernel EEXIST percolates back up as the return * value. Preserve this so that apps like sysinstall * can recognize this special case and not post bogus * dialog boxes. */ if (error != EEXIST) error = ENOEXEC; } else error = ENOENT; /* Nothing found */ out: return (error); } int linker_reference_module(const char *modname, struct mod_depend *verinfo, linker_file_t *result) { modlist_t mod; if ((mod = modlist_lookup2(modname, verinfo)) != NULL) { *result = mod->container; (*result)->refs++; return (0); } return (linker_load_module(NULL, modname, NULL, verinfo, result)); } linker_file_t linker_find_file_by_name(const char *filename) { linker_file_t lf = 0; char *koname; koname = malloc(strlen(filename) + 4, M_LINKER, M_WAITOK); if (koname == NULL) goto out; sprintf(koname, "%s.ko", filename); mtx_lock(&kld_mtx); TAILQ_FOREACH(lf, &linker_files, link) { if (strcmp(lf->filename, koname) == 0) break; if (strcmp(lf->filename, filename) == 0) break; } mtx_unlock(&kld_mtx); out: if (koname) free(koname, M_LINKER); return (lf); } linker_file_t linker_find_file_by_id(int fileid) { linker_file_t lf = 0; mtx_lock(&kld_mtx); TAILQ_FOREACH(lf, &linker_files, link) if (lf->id == fileid) break; mtx_unlock(&kld_mtx); return (lf); } linker_file_t linker_make_file(const char *pathname, linker_class_t lc) { linker_file_t lf; const char *filename; lf = NULL; filename = linker_basename(pathname); KLD_DPF(FILE, ("linker_make_file: new file, filename=%s\n", filename)); lf = (linker_file_t)kobj_create((kobj_class_t)lc, M_LINKER, M_WAITOK); if (lf == NULL) goto out; lf->refs = 1; lf->userrefs = 0; lf->flags = 0; lf->filename = linker_strdup(filename); LINKER_GET_NEXT_FILE_ID(lf->id); lf->ndeps = 0; lf->deps = NULL; STAILQ_INIT(&lf->common); TAILQ_INIT(&lf->modules); mtx_lock(&kld_mtx); TAILQ_INSERT_TAIL(&linker_files, lf, link); mtx_unlock(&kld_mtx); out: return (lf); } int linker_file_unload(linker_file_t file, int flags) { module_t mod, next; modlist_t ml, nextml; struct common_symbol *cp; int error, i; error = 0; /* Refuse to unload modules if securelevel raised. */ if (securelevel > 0) return (EPERM); #ifdef MAC error = mac_check_kld_unload(curthread->td_ucred); if (error) return (error); #endif KLD_DPF(FILE, ("linker_file_unload: lf->refs=%d\n", file->refs)); if (file->refs == 1) { KLD_DPF(FILE, ("linker_file_unload: file is unloading," " informing modules\n")); /* * Inform any modules associated with this file. */ MOD_XLOCK; for (mod = TAILQ_FIRST(&file->modules); mod; mod = next) { next = module_getfnext(mod); MOD_XUNLOCK; /* * Give the module a chance to veto the unload. */ if ((error = module_unload(mod, flags)) != 0) { KLD_DPF(FILE, ("linker_file_unload: module %p" " vetoes unload\n", mod)); goto out; } else MOD_XLOCK; module_release(mod); } MOD_XUNLOCK; } file->refs--; if (file->refs > 0) { goto out; } for (ml = TAILQ_FIRST(&found_modules); ml; ml = nextml) { nextml = TAILQ_NEXT(ml, link); if (ml->container == file) { TAILQ_REMOVE(&found_modules, ml, link); free(ml, M_LINKER); } } /* * Don't try to run SYSUNINITs if we are unloaded due to a * link error. */ if (file->flags & LINKER_FILE_LINKED) { linker_file_sysuninit(file); linker_file_unregister_sysctls(file); } mtx_lock(&kld_mtx); TAILQ_REMOVE(&linker_files, file, link); mtx_unlock(&kld_mtx); if (file->deps) { for (i = 0; i < file->ndeps; i++) linker_file_unload(file->deps[i], flags); free(file->deps, M_LINKER); file->deps = NULL; } for (cp = STAILQ_FIRST(&file->common); cp; cp = STAILQ_FIRST(&file->common)) { STAILQ_REMOVE(&file->common, cp, common_symbol, link); free(cp, M_LINKER); } LINKER_UNLOAD(file); if (file->filename) { free(file->filename, M_LINKER); file->filename = NULL; } kobj_delete((kobj_t) file, M_LINKER); out: return (error); } int linker_file_add_dependency(linker_file_t file, linker_file_t dep) { linker_file_t *newdeps; newdeps = malloc((file->ndeps + 1) * sizeof(linker_file_t *), M_LINKER, M_WAITOK | M_ZERO); if (newdeps == NULL) return (ENOMEM); if (file->deps) { bcopy(file->deps, newdeps, file->ndeps * sizeof(linker_file_t *)); free(file->deps, M_LINKER); } file->deps = newdeps; file->deps[file->ndeps] = dep; file->ndeps++; return (0); } /* * Locate a linker set and its contents. This is a helper function to avoid * linker_if.h exposure elsewhere. Note: firstp and lastp are really void *** */ int linker_file_lookup_set(linker_file_t file, const char *name, void *firstp, void *lastp, int *countp) { return (LINKER_LOOKUP_SET(file, name, firstp, lastp, countp)); } caddr_t linker_file_lookup_symbol(linker_file_t file, const char *name, int deps) { c_linker_sym_t sym; linker_symval_t symval; caddr_t address; size_t common_size = 0; int i; KLD_DPF(SYM, ("linker_file_lookup_symbol: file=%p, name=%s, deps=%d\n", file, name, deps)); if (LINKER_LOOKUP_SYMBOL(file, name, &sym) == 0) { LINKER_SYMBOL_VALUES(file, sym, &symval); if (symval.value == 0) /* * For commons, first look them up in the * dependencies and only allocate space if not found * there. */ common_size = symval.size; else { KLD_DPF(SYM, ("linker_file_lookup_symbol: symbol" ".value=%p\n", symval.value)); return (symval.value); } } if (deps) { for (i = 0; i < file->ndeps; i++) { address = linker_file_lookup_symbol(file->deps[i], name, 0); if (address) { KLD_DPF(SYM, ("linker_file_lookup_symbol:" " deps value=%p\n", address)); return (address); } } } if (common_size > 0) { /* * This is a common symbol which was not found in the * dependencies. We maintain a simple common symbol table in * the file object. */ struct common_symbol *cp; STAILQ_FOREACH(cp, &file->common, link) { if (strcmp(cp->name, name) == 0) { KLD_DPF(SYM, ("linker_file_lookup_symbol:" " old common value=%p\n", cp->address)); return (cp->address); } } /* * Round the symbol size up to align. */ common_size = (common_size + sizeof(int) - 1) & -sizeof(int); cp = malloc(sizeof(struct common_symbol) + common_size + strlen(name) + 1, M_LINKER, M_WAITOK | M_ZERO); if (cp == NULL) { KLD_DPF(SYM, ("linker_file_lookup_symbol: nomem\n")); return (0); } cp->address = (caddr_t)(cp + 1); cp->name = cp->address + common_size; strcpy(cp->name, name); bzero(cp->address, common_size); STAILQ_INSERT_TAIL(&file->common, cp, link); KLD_DPF(SYM, ("linker_file_lookup_symbol: new common" " value=%p\n", cp->address)); return (cp->address); } KLD_DPF(SYM, ("linker_file_lookup_symbol: fail\n")); return (0); } #ifdef DDB /* * DDB Helpers. DDB has to look across multiple files with their own symbol * tables and string tables. * * Note that we do not obey list locking protocols here. We really don't need * DDB to hang because somebody's got the lock held. We'll take the chance * that the files list is inconsistant instead. */ int linker_ddb_lookup(const char *symstr, c_linker_sym_t *sym) { linker_file_t lf; TAILQ_FOREACH(lf, &linker_files, link) { if (LINKER_LOOKUP_SYMBOL(lf, symstr, sym) == 0) return (0); } return (ENOENT); } int linker_ddb_search_symbol(caddr_t value, c_linker_sym_t *sym, long *diffp) { linker_file_t lf; c_linker_sym_t best, es; u_long diff, bestdiff, off; best = 0; off = (uintptr_t)value; bestdiff = off; TAILQ_FOREACH(lf, &linker_files, link) { if (LINKER_SEARCH_SYMBOL(lf, value, &es, &diff) != 0) continue; if (es != 0 && diff < bestdiff) { best = es; bestdiff = diff; } if (bestdiff == 0) break; } if (best) { *sym = best; *diffp = bestdiff; return (0); } else { *sym = 0; *diffp = off; return (ENOENT); } } int linker_ddb_symbol_values(c_linker_sym_t sym, linker_symval_t *symval) { linker_file_t lf; TAILQ_FOREACH(lf, &linker_files, link) { if (LINKER_SYMBOL_VALUES(lf, sym, symval) == 0) return (0); } return (ENOENT); } #endif /* * Syscalls. */ /* * MPSAFE */ int kldload(struct thread *td, struct kldload_args *uap) { +#ifdef HWPMC_HOOKS + struct pmckern_map_in pkm; +#endif char *kldname, *modname; char *pathname = NULL; linker_file_t lf; int error = 0; td->td_retval[0] = -1; mtx_lock(&Giant); if ((error = securelevel_gt(td->td_ucred, 0)) != 0) goto out; if ((error = suser(td)) != 0) goto out; pathname = malloc(MAXPATHLEN, M_TEMP, M_WAITOK); if ((error = copyinstr(uap->file, pathname, MAXPATHLEN, NULL)) != 0) goto out; /* * If path do not contain qualified name or any dot in it * (kldname.ko, or kldname.ver.ko) treat it as interface * name. */ if (index(pathname, '/') || index(pathname, '.')) { kldname = pathname; modname = NULL; } else { kldname = NULL; modname = pathname; } error = linker_load_module(kldname, modname, NULL, NULL, &lf); if (error) goto out; +#ifdef HWPMC_HOOKS + pkm.pm_file = lf->filename; + pkm.pm_address = (uintptr_t) lf->address; + PMC_CALL_HOOK(td, PMC_FN_KLD_LOAD, (void *) &pkm); +#endif lf->userrefs++; td->td_retval[0] = lf->id; out: if (pathname) free(pathname, M_TEMP); mtx_unlock(&Giant); return (error); } /* * MPSAFE */ static int kern_kldunload(struct thread *td, int fileid, int flags) { +#ifdef HWPMC_HOOKS + struct pmckern_map_out pkm; +#endif linker_file_t lf; int error = 0; mtx_lock(&Giant); if ((error = securelevel_gt(td->td_ucred, 0)) != 0) goto out; if ((error = suser(td)) != 0) goto out; lf = linker_find_file_by_id(fileid); if (lf) { KLD_DPF(FILE, ("kldunload: lf->userrefs=%d\n", lf->userrefs)); if (lf->userrefs == 0) { /* * XXX: maybe LINKER_UNLOAD_FORCE should override ? */ printf("kldunload: attempt to unload file that was" " loaded by the kernel\n"); error = EBUSY; goto out; } lf->userrefs--; +#ifdef HWPMC_HOOKS + /* Save data needed by hwpmc(4) before unloading the kld. */ + pkm.pm_address = (uintptr_t) lf->address; + pkm.pm_size = lf->size; +#endif error = linker_file_unload(lf, flags); if (error) lf->userrefs++; } else error = ENOENT; + +#ifdef HWPMC_HOOKS + if (error == 0) + PMC_CALL_HOOK(td, PMC_FN_KLD_UNLOAD, (void *) &pkm); +#endif out: mtx_unlock(&Giant); return (error); } /* * MPSAFE */ int kldunload(struct thread *td, struct kldunload_args *uap) { return (kern_kldunload(td, uap->fileid, LINKER_UNLOAD_NORMAL)); } /* * MPSAFE */ int kldunloadf(struct thread *td, struct kldunloadf_args *uap) { if (uap->flags != LINKER_UNLOAD_NORMAL && uap->flags != LINKER_UNLOAD_FORCE) return (EINVAL); return (kern_kldunload(td, uap->fileid, uap->flags)); } /* * MPSAFE */ int kldfind(struct thread *td, struct kldfind_args *uap) { char *pathname; const char *filename; linker_file_t lf; int error = 0; #ifdef MAC error = mac_check_kld_stat(td->td_ucred); if (error) return (error); #endif mtx_lock(&Giant); td->td_retval[0] = -1; pathname = malloc(MAXPATHLEN, M_TEMP, M_WAITOK); if ((error = copyinstr(uap->file, pathname, MAXPATHLEN, NULL)) != 0) goto out; filename = linker_basename(pathname); lf = linker_find_file_by_name(filename); if (lf) td->td_retval[0] = lf->id; else error = ENOENT; out: if (pathname) free(pathname, M_TEMP); mtx_unlock(&Giant); return (error); } /* * MPSAFE */ int kldnext(struct thread *td, struct kldnext_args *uap) { linker_file_t lf; int error = 0; #ifdef MAC error = mac_check_kld_stat(td->td_ucred); if (error) return (error); #endif mtx_lock(&Giant); if (uap->fileid == 0) { mtx_lock(&kld_mtx); if (TAILQ_FIRST(&linker_files)) td->td_retval[0] = TAILQ_FIRST(&linker_files)->id; else td->td_retval[0] = 0; mtx_unlock(&kld_mtx); goto out; } lf = linker_find_file_by_id(uap->fileid); if (lf) { if (TAILQ_NEXT(lf, link)) td->td_retval[0] = TAILQ_NEXT(lf, link)->id; else td->td_retval[0] = 0; } else error = ENOENT; out: mtx_unlock(&Giant); return (error); } /* * MPSAFE */ int kldstat(struct thread *td, struct kldstat_args *uap) { linker_file_t lf; int error = 0; int namelen, version; struct kld_file_stat *stat; #ifdef MAC error = mac_check_kld_stat(td->td_ucred); if (error) return (error); #endif mtx_lock(&Giant); lf = linker_find_file_by_id(uap->fileid); if (lf == NULL) { error = ENOENT; goto out; } stat = uap->stat; /* * Check the version of the user's structure. */ if ((error = copyin(&stat->version, &version, sizeof(version))) != 0) goto out; if (version != sizeof(struct kld_file_stat)) { error = EINVAL; goto out; } namelen = strlen(lf->filename) + 1; if (namelen > MAXPATHLEN) namelen = MAXPATHLEN; if ((error = copyout(lf->filename, &stat->name[0], namelen)) != 0) goto out; if ((error = copyout(&lf->refs, &stat->refs, sizeof(int))) != 0) goto out; if ((error = copyout(&lf->id, &stat->id, sizeof(int))) != 0) goto out; if ((error = copyout(&lf->address, &stat->address, sizeof(caddr_t))) != 0) goto out; if ((error = copyout(&lf->size, &stat->size, sizeof(size_t))) != 0) goto out; td->td_retval[0] = 0; out: mtx_unlock(&Giant); return (error); } /* * MPSAFE */ int kldfirstmod(struct thread *td, struct kldfirstmod_args *uap) { linker_file_t lf; module_t mp; int error = 0; #ifdef MAC error = mac_check_kld_stat(td->td_ucred); if (error) return (error); #endif mtx_lock(&Giant); lf = linker_find_file_by_id(uap->fileid); if (lf) { MOD_SLOCK; mp = TAILQ_FIRST(&lf->modules); if (mp != NULL) td->td_retval[0] = module_getid(mp); else td->td_retval[0] = 0; MOD_SUNLOCK; } else error = ENOENT; mtx_unlock(&Giant); return (error); } /* * MPSAFE */ int kldsym(struct thread *td, struct kldsym_args *uap) { char *symstr = NULL; c_linker_sym_t sym; linker_symval_t symval; linker_file_t lf; struct kld_sym_lookup lookup; int error = 0; #ifdef MAC error = mac_check_kld_stat(td->td_ucred); if (error) return (error); #endif mtx_lock(&Giant); if ((error = copyin(uap->data, &lookup, sizeof(lookup))) != 0) goto out; if (lookup.version != sizeof(lookup) || uap->cmd != KLDSYM_LOOKUP) { error = EINVAL; goto out; } symstr = malloc(MAXPATHLEN, M_TEMP, M_WAITOK); if ((error = copyinstr(lookup.symname, symstr, MAXPATHLEN, NULL)) != 0) goto out; if (uap->fileid != 0) { lf = linker_find_file_by_id(uap->fileid); if (lf == NULL) { error = ENOENT; goto out; } if (LINKER_LOOKUP_SYMBOL(lf, symstr, &sym) == 0 && LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) { lookup.symvalue = (uintptr_t) symval.value; lookup.symsize = symval.size; error = copyout(&lookup, uap->data, sizeof(lookup)); } else error = ENOENT; } else { mtx_lock(&kld_mtx); TAILQ_FOREACH(lf, &linker_files, link) { if (LINKER_LOOKUP_SYMBOL(lf, symstr, &sym) == 0 && LINKER_SYMBOL_VALUES(lf, sym, &symval) == 0) { lookup.symvalue = (uintptr_t)symval.value; lookup.symsize = symval.size; error = copyout(&lookup, uap->data, sizeof(lookup)); break; } } mtx_unlock(&kld_mtx); if (lf == NULL) error = ENOENT; } out: if (symstr) free(symstr, M_TEMP); mtx_unlock(&Giant); return (error); } /* * Preloaded module support */ static modlist_t modlist_lookup(const char *name, int ver) { modlist_t mod; TAILQ_FOREACH(mod, &found_modules, link) { if (strcmp(mod->name, name) == 0 && (ver == 0 || mod->version == ver)) return (mod); } return (NULL); } static modlist_t modlist_lookup2(const char *name, struct mod_depend *verinfo) { modlist_t mod, bestmod; int ver; if (verinfo == NULL) return (modlist_lookup(name, 0)); bestmod = NULL; for (mod = TAILQ_FIRST(&found_modules); mod; mod = TAILQ_NEXT(mod, link)) { if (strcmp(mod->name, name) != 0) continue; ver = mod->version; if (ver == verinfo->md_ver_preferred) return (mod); if (ver >= verinfo->md_ver_minimum && ver <= verinfo->md_ver_maximum && (bestmod == NULL || ver > bestmod->version)) bestmod = mod; } return (bestmod); } static modlist_t modlist_newmodule(const char *modname, int version, linker_file_t container) { modlist_t mod; mod = malloc(sizeof(struct modlist), M_LINKER, M_NOWAIT | M_ZERO); if (mod == NULL) panic("no memory for module list"); mod->container = container; mod->name = modname; mod->version = version; TAILQ_INSERT_TAIL(&found_modules, mod, link); return (mod); } static void linker_addmodules(linker_file_t lf, struct mod_metadata **start, struct mod_metadata **stop, int preload) { struct mod_metadata *mp, **mdp; const char *modname; int ver; for (mdp = start; mdp < stop; mdp++) { mp = *mdp; if (mp->md_type != MDT_VERSION) continue; modname = mp->md_cval; ver = ((struct mod_version *)mp->md_data)->mv_version; if (modlist_lookup(modname, ver) != NULL) { printf("module %s already present!\n", modname); /* XXX what can we do? this is a build error. :-( */ continue; } modlist_newmodule(modname, ver, lf); } } static void linker_preload(void *arg) { caddr_t modptr; const char *modname, *nmodname; char *modtype; linker_file_t lf; linker_class_t lc; int error; linker_file_list_t loaded_files; linker_file_list_t depended_files; struct mod_metadata *mp, *nmp; struct mod_metadata **start, **stop, **mdp, **nmdp; struct mod_depend *verinfo; int nver; int resolves; modlist_t mod; struct sysinit **si_start, **si_stop; TAILQ_INIT(&loaded_files); TAILQ_INIT(&depended_files); TAILQ_INIT(&found_modules); error = 0; modptr = NULL; while ((modptr = preload_search_next_name(modptr)) != NULL) { modname = (char *)preload_search_info(modptr, MODINFO_NAME); modtype = (char *)preload_search_info(modptr, MODINFO_TYPE); if (modname == NULL) { printf("Preloaded module at %p does not have a" " name!\n", modptr); continue; } if (modtype == NULL) { printf("Preloaded module at %p does not have a type!\n", modptr); continue; } if (bootverbose) printf("Preloaded %s \"%s\" at %p.\n", modtype, modname, modptr); lf = NULL; TAILQ_FOREACH(lc, &classes, link) { error = LINKER_LINK_PRELOAD(lc, modname, &lf); if (!error) break; lf = NULL; } if (lf) TAILQ_INSERT_TAIL(&loaded_files, lf, loaded); } /* * First get a list of stuff in the kernel. */ if (linker_file_lookup_set(linker_kernel_file, MDT_SETNAME, &start, &stop, NULL) == 0) linker_addmodules(linker_kernel_file, start, stop, 1); /* * this is a once-off kinky bubble sort resolve relocation dependency * requirements */ restart: TAILQ_FOREACH(lf, &loaded_files, loaded) { error = linker_file_lookup_set(lf, MDT_SETNAME, &start, &stop, NULL); /* * First, look to see if we would successfully link with this * stuff. */ resolves = 1; /* unless we know otherwise */ if (!error) { for (mdp = start; mdp < stop; mdp++) { mp = *mdp; if (mp->md_type != MDT_DEPEND) continue; modname = mp->md_cval; verinfo = mp->md_data; for (nmdp = start; nmdp < stop; nmdp++) { nmp = *nmdp; if (nmp->md_type != MDT_VERSION) continue; nmodname = nmp->md_cval; if (strcmp(modname, nmodname) == 0) break; } if (nmdp < stop) /* it's a self reference */ continue; /* * ok, the module isn't here yet, we * are not finished */ if (modlist_lookup2(modname, verinfo) == NULL) resolves = 0; } } /* * OK, if we found our modules, we can link. So, "provide" * the modules inside and add it to the end of the link order * list. */ if (resolves) { if (!error) { for (mdp = start; mdp < stop; mdp++) { mp = *mdp; if (mp->md_type != MDT_VERSION) continue; modname = mp->md_cval; nver = ((struct mod_version *) mp->md_data)->mv_version; if (modlist_lookup(modname, nver) != NULL) { printf("module %s already" " present!\n", modname); linker_file_unload(lf, LINKER_UNLOAD_FORCE); TAILQ_REMOVE(&loaded_files, lf, loaded); /* we changed tailq next ptr */ goto restart; } modlist_newmodule(modname, nver, lf); } } TAILQ_REMOVE(&loaded_files, lf, loaded); TAILQ_INSERT_TAIL(&depended_files, lf, loaded); /* * Since we provided modules, we need to restart the * sort so that the previous files that depend on us * have a chance. Also, we've busted the tailq next * pointer with the REMOVE. */ goto restart; } } /* * At this point, we check to see what could not be resolved.. */ TAILQ_FOREACH(lf, &loaded_files, loaded) { printf("KLD file %s is missing dependencies\n", lf->filename); linker_file_unload(lf, LINKER_UNLOAD_FORCE); TAILQ_REMOVE(&loaded_files, lf, loaded); } /* * We made it. Finish off the linking in the order we determined. */ TAILQ_FOREACH(lf, &depended_files, loaded) { if (linker_kernel_file) { linker_kernel_file->refs++; error = linker_file_add_dependency(lf, linker_kernel_file); if (error) panic("cannot add dependency"); } lf->userrefs++; /* so we can (try to) kldunload it */ error = linker_file_lookup_set(lf, MDT_SETNAME, &start, &stop, NULL); if (!error) { for (mdp = start; mdp < stop; mdp++) { mp = *mdp; if (mp->md_type != MDT_DEPEND) continue; modname = mp->md_cval; verinfo = mp->md_data; mod = modlist_lookup2(modname, verinfo); /* Don't count self-dependencies */ if (lf == mod->container) continue; mod->container->refs++; error = linker_file_add_dependency(lf, mod->container); if (error) panic("cannot add dependency"); } } /* * Now do relocation etc using the symbol search paths * established by the dependencies */ error = LINKER_LINK_PRELOAD_FINISH(lf); if (error) { printf("KLD file %s - could not finalize loading\n", lf->filename); linker_file_unload(lf, LINKER_UNLOAD_FORCE); continue; } linker_file_register_modules(lf); if (linker_file_lookup_set(lf, "sysinit_set", &si_start, &si_stop, NULL) == 0) sysinit_add(si_start, si_stop); linker_file_register_sysctls(lf); lf->flags |= LINKER_FILE_LINKED; } /* woohoo! we made it! */ } SYSINIT(preload, SI_SUB_KLD, SI_ORDER_MIDDLE, linker_preload, 0) /* * Search for a not-loaded module by name. * * Modules may be found in the following locations: * * - preloaded (result is just the module name) - on disk (result is full path * to module) * * If the module name is qualified in any way (contains path, etc.) the we * simply return a copy of it. * * The search path can be manipulated via sysctl. Note that we use the ';' * character as a separator to be consistent with the bootloader. */ static char linker_hintfile[] = "linker.hints"; static char linker_path[MAXPATHLEN] = "/boot/kernel;/boot/modules"; SYSCTL_STRING(_kern, OID_AUTO, module_path, CTLFLAG_RW, linker_path, sizeof(linker_path), "module load search path"); TUNABLE_STR("module_path", linker_path, sizeof(linker_path)); static char *linker_ext_list[] = { "", ".ko", NULL }; /* * Check if file actually exists either with or without extension listed in * the linker_ext_list. (probably should be generic for the rest of the * kernel) */ static char * linker_lookup_file(const char *path, int pathlen, const char *name, int namelen, struct vattr *vap) { struct nameidata nd; struct thread *td = curthread; /* XXX */ char *result, **cpp, *sep; int error, len, extlen, reclen, flags; enum vtype type; extlen = 0; for (cpp = linker_ext_list; *cpp; cpp++) { len = strlen(*cpp); if (len > extlen) extlen = len; } extlen++; /* trailing '\0' */ sep = (path[pathlen - 1] != '/') ? "/" : ""; reclen = pathlen + strlen(sep) + namelen + extlen + 1; result = malloc(reclen, M_LINKER, M_WAITOK); for (cpp = linker_ext_list; *cpp; cpp++) { snprintf(result, reclen, "%.*s%s%.*s%s", pathlen, path, sep, namelen, name, *cpp); /* * Attempt to open the file, and return the path if * we succeed and it's a regular file. */ NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, result, td); flags = FREAD; error = vn_open(&nd, &flags, 0, -1); if (error == 0) { NDFREE(&nd, NDF_ONLY_PNBUF); type = nd.ni_vp->v_type; if (vap) VOP_GETATTR(nd.ni_vp, vap, td->td_ucred, td); VOP_UNLOCK(nd.ni_vp, 0, td); vn_close(nd.ni_vp, FREAD, td->td_ucred, td); if (type == VREG) return (result); } } free(result, M_LINKER); return (NULL); } #define INT_ALIGN(base, ptr) ptr = \ (base) + (((ptr) - (base) + sizeof(int) - 1) & ~(sizeof(int) - 1)) /* * Lookup KLD which contains requested module in the "linker.hints" file. If * version specification is available, then try to find the best KLD. * Otherwise just find the latest one. */ static char * linker_hints_lookup(const char *path, int pathlen, const char *modname, int modnamelen, struct mod_depend *verinfo) { struct thread *td = curthread; /* XXX */ struct ucred *cred = td ? td->td_ucred : NULL; struct nameidata nd; struct vattr vattr, mattr; u_char *hints = NULL; u_char *cp, *recptr, *bufend, *result, *best, *pathbuf, *sep; int error, ival, bestver, *intp, reclen, found, flags, clen, blen; result = NULL; bestver = found = 0; sep = (path[pathlen - 1] != '/') ? "/" : ""; reclen = imax(modnamelen, strlen(linker_hintfile)) + pathlen + strlen(sep) + 1; pathbuf = malloc(reclen, M_LINKER, M_WAITOK); snprintf(pathbuf, reclen, "%.*s%s%s", pathlen, path, sep, linker_hintfile); NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_SYSSPACE, pathbuf, td); flags = FREAD; error = vn_open(&nd, &flags, 0, -1); if (error) goto bad; NDFREE(&nd, NDF_ONLY_PNBUF); if (nd.ni_vp->v_type != VREG) goto bad; best = cp = NULL; error = VOP_GETATTR(nd.ni_vp, &vattr, cred, td); if (error) goto bad; /* * XXX: we need to limit this number to some reasonable value */ if (vattr.va_size > 100 * 1024) { printf("hints file too large %ld\n", (long)vattr.va_size); goto bad; } hints = malloc(vattr.va_size, M_TEMP, M_WAITOK); if (hints == NULL) goto bad; error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)hints, vattr.va_size, 0, UIO_SYSSPACE, IO_NODELOCKED, cred, NOCRED, &reclen, td); if (error) goto bad; VOP_UNLOCK(nd.ni_vp, 0, td); vn_close(nd.ni_vp, FREAD, cred, td); nd.ni_vp = NULL; if (reclen != 0) { printf("can't read %d\n", reclen); goto bad; } intp = (int *)hints; ival = *intp++; if (ival != LINKER_HINTS_VERSION) { printf("hints file version mismatch %d\n", ival); goto bad; } bufend = hints + vattr.va_size; recptr = (u_char *)intp; clen = blen = 0; while (recptr < bufend && !found) { intp = (int *)recptr; reclen = *intp++; ival = *intp++; cp = (char *)intp; switch (ival) { case MDT_VERSION: clen = *cp++; if (clen != modnamelen || bcmp(cp, modname, clen) != 0) break; cp += clen; INT_ALIGN(hints, cp); ival = *(int *)cp; cp += sizeof(int); clen = *cp++; if (verinfo == NULL || ival == verinfo->md_ver_preferred) { found = 1; break; } if (ival >= verinfo->md_ver_minimum && ival <= verinfo->md_ver_maximum && ival > bestver) { bestver = ival; best = cp; blen = clen; } break; default: break; } recptr += reclen + sizeof(int); } /* * Finally check if KLD is in the place */ if (found) result = linker_lookup_file(path, pathlen, cp, clen, &mattr); else if (best) result = linker_lookup_file(path, pathlen, best, blen, &mattr); /* * KLD is newer than hints file. What we should do now? */ if (result && timespeccmp(&mattr.va_mtime, &vattr.va_mtime, >)) printf("warning: KLD '%s' is newer than the linker.hints" " file\n", result); bad: free(pathbuf, M_LINKER); if (hints) free(hints, M_TEMP); if (nd.ni_vp != NULL) { VOP_UNLOCK(nd.ni_vp, 0, td); vn_close(nd.ni_vp, FREAD, cred, td); } /* * If nothing found or hints is absent - fallback to the old * way by using "kldname[.ko]" as module name. */ if (!found && !bestver && result == NULL) result = linker_lookup_file(path, pathlen, modname, modnamelen, NULL); return (result); } /* * Lookup KLD which contains requested module in the all directories. */ static char * linker_search_module(const char *modname, int modnamelen, struct mod_depend *verinfo) { char *cp, *ep, *result; /* * traverse the linker path */ for (cp = linker_path; *cp; cp = ep + 1) { /* find the end of this component */ for (ep = cp; (*ep != 0) && (*ep != ';'); ep++); result = linker_hints_lookup(cp, ep - cp, modname, modnamelen, verinfo); if (result != NULL) return (result); if (*ep == 0) break; } return (NULL); } /* * Search for module in all directories listed in the linker_path. */ static char * linker_search_kld(const char *name) { char *cp, *ep, *result, **cpp; int extlen, len; /* qualified at all? */ if (index(name, '/')) return (linker_strdup(name)); extlen = 0; for (cpp = linker_ext_list; *cpp; cpp++) { len = strlen(*cpp); if (len > extlen) extlen = len; } extlen++; /* trailing '\0' */ /* traverse the linker path */ len = strlen(name); for (ep = linker_path; *ep; ep++) { cp = ep; /* find the end of this component */ for (; *ep != 0 && *ep != ';'; ep++); result = linker_lookup_file(cp, ep - cp, name, len, NULL); if (result != NULL) return (result); } return (NULL); } static const char * linker_basename(const char *path) { const char *filename; filename = rindex(path, '/'); if (filename == NULL) return path; if (filename[1]) filename++; return (filename); } + +#ifdef HWPMC_HOOKS + +/* + * Inform hwpmc about the set of kernel modules currently loaded. + */ +void * +linker_hwpmc_list_objects(void) +{ + int nobjects, nmappings; + linker_file_t lf; + struct pmckern_map_in *ko, *kobase; + + nmappings = 15; /* a reasonable default */ + + retry: + /* allocate nmappings+1 entries */ + MALLOC(kobase, struct pmckern_map_in *, + (nmappings + 1) * sizeof(struct pmckern_map_in), M_LINKER, + M_WAITOK | M_ZERO); + + nobjects = 0; + mtx_lock(&kld_mtx); + TAILQ_FOREACH(lf, &linker_files, link) + nobjects++; + + KASSERT(nobjects > 0, ("linker_hpwmc_list_objects: no kernel " + "objects?")); + + if (nobjects > nmappings) { + nmappings = nobjects; + FREE(kobase, M_LINKER); + mtx_unlock(&kld_mtx); + goto retry; + } + + ko = kobase; + TAILQ_FOREACH(lf, &linker_files, link) { + ko->pm_file = lf->filename; + ko->pm_address = (uintptr_t) lf->address; + ko++; + } + + /* The last entry of the malloced area comprises of all zeros. */ + KASSERT(ko->pm_file == NULL, + ("linker_hwpmc_list_objects: last object not NULL")); + + mtx_unlock(&kld_mtx); + + return ((void *) kobase); +} +#endif /* * Find a file which contains given module and load it, if "parent" is not * NULL, register a reference to it. */ int linker_load_module(const char *kldname, const char *modname, struct linker_file *parent, struct mod_depend *verinfo, struct linker_file **lfpp) { linker_file_t lfdep; const char *filename; char *pathname; int error; if (modname == NULL) { /* * We have to load KLD */ KASSERT(verinfo == NULL, ("linker_load_module: verinfo" " is not NULL")); pathname = linker_search_kld(kldname); } else { if (modlist_lookup2(modname, verinfo) != NULL) return (EEXIST); if (kldname != NULL) pathname = linker_strdup(kldname); else if (rootvnode == NULL) pathname = NULL; else /* * Need to find a KLD with required module */ pathname = linker_search_module(modname, strlen(modname), verinfo); } if (pathname == NULL) return (ENOENT); /* * Can't load more than one file with the same basename XXX: * Actually it should be possible to have multiple KLDs with * the same basename but different path because they can * provide different versions of the same modules. */ filename = linker_basename(pathname); if (linker_find_file_by_name(filename)) { error = EEXIST; goto out; } do { error = linker_load_file(pathname, &lfdep); if (error) break; if (modname && verinfo && modlist_lookup2(modname, verinfo) == NULL) { linker_file_unload(lfdep, LINKER_UNLOAD_FORCE); error = ENOENT; break; } if (parent) { error = linker_file_add_dependency(parent, lfdep); if (error) break; } if (lfpp) *lfpp = lfdep; } while (0); out: if (pathname) free(pathname, M_LINKER); return (error); } /* * This routine is responsible for finding dependencies of userland initiated * kldload(2)'s of files. */ int linker_load_dependencies(linker_file_t lf) { linker_file_t lfdep; struct mod_metadata **start, **stop, **mdp, **nmdp; struct mod_metadata *mp, *nmp; struct mod_depend *verinfo; modlist_t mod; const char *modname, *nmodname; int ver, error = 0, count; /* * All files are dependant on /kernel. */ if (linker_kernel_file) { linker_kernel_file->refs++; error = linker_file_add_dependency(lf, linker_kernel_file); if (error) return (error); } if (linker_file_lookup_set(lf, MDT_SETNAME, &start, &stop, &count) != 0) return (0); for (mdp = start; mdp < stop; mdp++) { mp = *mdp; if (mp->md_type != MDT_VERSION) continue; modname = mp->md_cval; ver = ((struct mod_version *)mp->md_data)->mv_version; mod = modlist_lookup(modname, ver); if (mod != NULL) { printf("interface %s.%d already present in the KLD" " '%s'!\n", modname, ver, mod->container->filename); return (EEXIST); } } for (mdp = start; mdp < stop; mdp++) { mp = *mdp; if (mp->md_type != MDT_DEPEND) continue; modname = mp->md_cval; verinfo = mp->md_data; nmodname = NULL; for (nmdp = start; nmdp < stop; nmdp++) { nmp = *nmdp; if (nmp->md_type != MDT_VERSION) continue; nmodname = nmp->md_cval; if (strcmp(modname, nmodname) == 0) break; } if (nmdp < stop)/* early exit, it's a self reference */ continue; mod = modlist_lookup2(modname, verinfo); if (mod) { /* woohoo, it's loaded already */ lfdep = mod->container; lfdep->refs++; error = linker_file_add_dependency(lf, lfdep); if (error) break; continue; } error = linker_load_module(NULL, modname, lf, verinfo, NULL); if (error) { printf("KLD %s: depends on %s - not available\n", lf->filename, modname); break; } } if (error) return (error); linker_addmodules(lf, start, stop, 0); return (error); } static int sysctl_kern_function_list_iterate(const char *name, void *opaque) { struct sysctl_req *req; req = opaque; return (SYSCTL_OUT(req, name, strlen(name) + 1)); } /* * Export a nul-separated, double-nul-terminated list of all function names * in the kernel. */ static int sysctl_kern_function_list(SYSCTL_HANDLER_ARGS) { linker_file_t lf; int error; #ifdef MAC error = mac_check_kld_stat(req->td->td_ucred); if (error) return (error); #endif error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); mtx_lock(&kld_mtx); TAILQ_FOREACH(lf, &linker_files, link) { error = LINKER_EACH_FUNCTION_NAME(lf, sysctl_kern_function_list_iterate, req); if (error) { mtx_unlock(&kld_mtx); return (error); } } mtx_unlock(&kld_mtx); return (SYSCTL_OUT(req, "", 1)); } SYSCTL_PROC(_kern, OID_AUTO, function_list, CTLFLAG_RD, NULL, 0, sysctl_kern_function_list, "", "kernel function list"); Index: head/sys/sys/linker.h =================================================================== --- head/sys/sys/linker.h (revision 157143) +++ head/sys/sys/linker.h (revision 157144) @@ -1,302 +1,305 @@ /*- * Copyright (c) 1997-2000 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_LINKER_H_ #define _SYS_LINKER_H_ #ifdef _KERNEL #include #include #ifdef MALLOC_DECLARE MALLOC_DECLARE(M_LINKER); #endif struct mod_depend; /* * Object representing a file which has been loaded by the linker. */ typedef struct linker_file* linker_file_t; typedef TAILQ_HEAD(, linker_file) linker_file_list_t; typedef caddr_t linker_sym_t; /* opaque symbol */ typedef c_caddr_t c_linker_sym_t; /* const opaque symbol */ typedef int (*linker_function_name_callback_t)(const char *, void *); /* * expanded out linker_sym_t */ typedef struct linker_symval { const char* name; caddr_t value; size_t size; } linker_symval_t; struct common_symbol { STAILQ_ENTRY(common_symbol) link; char* name; caddr_t address; }; struct linker_file { KOBJ_FIELDS; int refs; /* reference count */ int userrefs; /* kldload(2) count */ int flags; #define LINKER_FILE_LINKED 0x1 /* file has been fully linked */ TAILQ_ENTRY(linker_file) link; /* list of all loaded files */ char* filename; /* file which was loaded */ int id; /* unique id */ caddr_t address; /* load address */ size_t size; /* size of file */ int ndeps; /* number of dependencies */ linker_file_t* deps; /* list of dependencies */ STAILQ_HEAD(, common_symbol) common; /* list of common symbols */ TAILQ_HEAD(, module) modules; /* modules in this file */ TAILQ_ENTRY(linker_file) loaded; /* preload dependency support */ }; /* * Object implementing a class of file (a.out, elf, etc.) */ typedef struct linker_class *linker_class_t; typedef TAILQ_HEAD(, linker_class) linker_class_list_t; struct linker_class { KOBJ_CLASS_FIELDS; TAILQ_ENTRY(linker_class) link; /* list of all file classes */ }; /* * The "file" for the kernel. */ extern linker_file_t linker_kernel_file; /* * Add a new file class to the linker. */ int linker_add_class(linker_class_t _cls); /* * Load a kernel module. */ int linker_load_module(const char *_kldname, const char *_modname, struct linker_file *_parent, struct mod_depend *_verinfo, struct linker_file **_lfpp); /* * Obtain a reference to a module, loading it if required. */ int linker_reference_module(const char* _modname, struct mod_depend *_verinfo, linker_file_t* _result); /* * Find a currently loaded file given its filename. */ linker_file_t linker_find_file_by_name(const char* _filename); /* * Find a currently loaded file given its file id. */ linker_file_t linker_find_file_by_id(int _fileid); /* * Called from a class handler when a file is laoded. */ linker_file_t linker_make_file(const char* _filename, linker_class_t _cls); /* * Unload a file, freeing up memory. */ int linker_file_unload(linker_file_t _file, int flags); /* * Add a dependency to a file. */ int linker_file_add_dependency(linker_file_t _file, linker_file_t _dep); /* * Lookup a symbol in a file. If deps is TRUE, look in dependencies * if not found in file. */ caddr_t linker_file_lookup_symbol(linker_file_t _file, const char* _name, int _deps); /* * Lookup a linker set in a file. Return pointers to the first entry, * last + 1, and count of entries. Use: for (p = start; p < stop; p++) {} * void *start is really: "struct yoursetmember ***start;" */ int linker_file_lookup_set(linker_file_t _file, const char *_name, void *_start, void *_stop, int *_count); /* * This routine is responsible for finding dependencies of userland * initiated kldload(2)'s of files. */ int linker_load_dependencies(linker_file_t _lf); /* * DDB Helpers, tuned specifically for ddb/db_kld.c */ int linker_ddb_lookup(const char *_symstr, c_linker_sym_t *_sym); int linker_ddb_search_symbol(caddr_t _value, c_linker_sym_t *_sym, long *_diffp); int linker_ddb_symbol_values(c_linker_sym_t _sym, linker_symval_t *_symval); +/* HWPMC helper */ +void *linker_hwpmc_list_objects(void); + #endif /* _KERNEL */ /* * Module information subtypes */ #define MODINFO_END 0x0000 /* End of list */ #define MODINFO_NAME 0x0001 /* Name of module (string) */ #define MODINFO_TYPE 0x0002 /* Type of module (string) */ #define MODINFO_ADDR 0x0003 /* Loaded address */ #define MODINFO_SIZE 0x0004 /* Size of module */ #define MODINFO_EMPTY 0x0005 /* Has been deleted */ #define MODINFO_ARGS 0x0006 /* Parameters string */ #define MODINFO_METADATA 0x8000 /* Module-specfic */ #define MODINFOMD_AOUTEXEC 0x0001 /* a.out exec header */ #define MODINFOMD_ELFHDR 0x0002 /* ELF header */ #define MODINFOMD_SSYM 0x0003 /* start of symbols */ #define MODINFOMD_ESYM 0x0004 /* end of symbols */ #define MODINFOMD_DYNAMIC 0x0005 /* _DYNAMIC pointer */ /* These values are MD on these two platforms */ #if !defined(__sparc64__) && !defined(__powerpc__) #define MODINFOMD_ENVP 0x0006 /* envp[] */ #define MODINFOMD_HOWTO 0x0007 /* boothowto */ #define MODINFOMD_KERNEND 0x0008 /* kernend */ #endif #define MODINFOMD_SHDR 0x0009 /* section header table */ #define MODINFOMD_NOCOPY 0x8000 /* don't copy this metadata to the kernel */ #define MODINFOMD_DEPLIST (0x4001 | MODINFOMD_NOCOPY) /* depends on */ #ifdef _KERNEL #define MD_FETCH(mdp, info, type) ({ \ type *__p; \ __p = (type *)preload_search_info((mdp), MODINFO_METADATA | (info)); \ __p ? *__p : 0; \ }) #endif #define LINKER_HINTS_VERSION 1 /* linker.hints file version */ #ifdef _KERNEL /* * Module lookup */ extern caddr_t preload_metadata; extern caddr_t preload_search_by_name(const char *_name); extern caddr_t preload_search_by_type(const char *_type); extern caddr_t preload_search_next_name(caddr_t _base); extern caddr_t preload_search_info(caddr_t _mod, int _inf); extern void preload_delete_name(const char *_name); extern void preload_bootstrap_relocate(vm_offset_t _offset); #ifdef KLD_DEBUG extern int kld_debug; #define KLD_DEBUG_FILE 1 /* file load/unload */ #define KLD_DEBUG_SYM 2 /* symbol lookup */ #define KLD_DPF(cat, args) \ do { \ if (kld_debug & KLD_DEBUG_##cat) printf args; \ } while (0) #else #define KLD_DPF(cat, args) #endif typedef Elf_Addr elf_lookup_fn(linker_file_t, Elf_Size, int); /* Support functions */ int elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu); int elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu); const Elf_Sym *elf_get_sym(linker_file_t _lf, Elf_Size _symidx); const char *elf_get_symname(linker_file_t _lf, Elf_Size _symidx); int elf_cpu_load_file(linker_file_t); int elf_cpu_unload_file(linker_file_t); /* values for type */ #define ELF_RELOC_REL 1 #define ELF_RELOC_RELA 2 #endif /* _KERNEL */ struct kld_file_stat { int version; /* set to sizeof(linker_file_stat) */ char name[MAXPATHLEN]; int refs; int id; caddr_t address; /* load address */ size_t size; /* size in bytes */ }; struct kld_sym_lookup { int version; /* set to sizeof(struct kld_sym_lookup) */ char *symname; /* Symbol name we are looking up */ u_long symvalue; size_t symsize; }; #define KLDSYM_LOOKUP 1 /* * Flags for kldunloadf() and linker_file_unload() */ #define LINKER_UNLOAD_NORMAL 0 #define LINKER_UNLOAD_FORCE 1 #ifndef _KERNEL #include __BEGIN_DECLS int kldload(const char* _file); int kldunload(int _fileid); int kldunloadf(int _fileid, int flags); int kldfind(const char* _file); int kldnext(int _fileid); int kldstat(int _fileid, struct kld_file_stat* _stat); int kldfirstmod(int _fileid); int kldsym(int _fileid, int _cmd, void *_data); __END_DECLS #endif #endif /* !_SYS_LINKER_H_ */ Index: head/sys/sys/param.h =================================================================== --- head/sys/sys/param.h (revision 157143) +++ head/sys/sys/param.h (revision 157144) @@ -1,334 +1,334 @@ /*- * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)param.h 8.3 (Berkeley) 4/4/95 * $FreeBSD$ */ #ifndef _SYS_PARAM_H_ #define _SYS_PARAM_H_ #include #define BSD 199506 /* System version (year & month). */ #define BSD4_3 1 #define BSD4_4 1 /* * __FreeBSD_version numbers are documented in the Porter's Handbook. * If you bump the version for any reason, you should update the documentation * there. * Currently this lives here: * * doc/en_US.ISO8859-1/books/porters-handbook/book.sgml * * scheme is: Rxx * 'R' is 0 if release branch or x.0-CURRENT before RELENG_*_0 * is created, otherwise 1. */ #undef __FreeBSD_version -#define __FreeBSD_version 700014 /* Master, propagated to newvers */ +#define __FreeBSD_version 700015 /* Master, propagated to newvers */ #ifndef LOCORE #include #endif /* * Machine-independent constants (some used in following include files). * Redefined constants are from POSIX 1003.1 limits file. * * MAXCOMLEN should be >= sizeof(ac_comm) (see ) * MAXLOGNAME should be == UT_NAMESIZE+1 (see ) */ #include #define MAXCOMLEN 19 /* max command name remembered */ #define MAXINTERP 32 /* max interpreter file name length */ #define MAXLOGNAME 17 /* max login name length (incl. NUL) */ #define MAXUPRC CHILD_MAX /* max simultaneous processes */ #define NCARGS ARG_MAX /* max bytes for an exec function */ #define NGROUPS NGROUPS_MAX /* max number groups */ #define NOFILE OPEN_MAX /* max open files per process */ #define NOGROUP 65535 /* marker for empty group set member */ #define MAXHOSTNAMELEN 256 /* max hostname size */ #define SPECNAMELEN 63 /* max length of devicename */ /* More types and definitions used throughout the kernel. */ #ifdef _KERNEL #if 0 #if (defined(BURN_BRIDGES) || __FreeBSD_version >= 600000) \ && defined(OBSOLETE_IN_6) #error "This file contains obsolete code to be removed in 6.0-current" #endif #endif #include #include #ifndef LOCORE #include #include #endif #define FALSE 0 #define TRUE 1 #endif #ifndef _KERNEL /* Signals. */ #include #endif /* Machine type dependent parameters. */ #include #ifndef _KERNEL #include #endif #ifndef _NO_NAMESPACE_POLLUTION #ifndef DEV_BSHIFT #define DEV_BSHIFT 9 /* log2(DEV_BSIZE) */ #endif #define DEV_BSIZE (1<>PAGE_SHIFT) #endif /* * btodb() is messy and perhaps slow because `bytes' may be an off_t. We * want to shift an unsigned type to avoid sign extension and we don't * want to widen `bytes' unnecessarily. Assume that the result fits in * a daddr_t. */ #ifndef btodb #define btodb(bytes) /* calculates (bytes / DEV_BSIZE) */ \ (sizeof (bytes) > sizeof(long) \ ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \ : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT)) #endif #ifndef dbtob #define dbtob(db) /* calculates (db * DEV_BSIZE) */ \ ((off_t)(db) << DEV_BSHIFT) #endif #endif /* _NO_NAMESPACE_POLLUTION */ #define PRIMASK 0x0ff #define PCATCH 0x100 /* OR'd with pri for tsleep to check signals */ #define PDROP 0x200 /* OR'd with pri to stop re-entry of interlock mutex */ #define NZERO 0 /* default "nice" */ #define NBBY 8 /* number of bits in a byte */ #define NBPW sizeof(int) /* number of bytes per word (integer) */ #define CMASK 022 /* default file mask: S_IWGRP|S_IWOTH */ #define NODEV (dev_t)(-1) /* non-existent device */ #define CBLOCK 128 /* Clist block size, must be a power of 2. */ #define CBQSIZE (CBLOCK/NBBY) /* Quote bytes/cblock - can do better. */ /* Data chars/clist. */ #define CBSIZE (CBLOCK - sizeof(struct cblock *) - CBQSIZE) #define CROUND (CBLOCK - 1) /* Clist rounding. */ /* * File system parameters and macros. * * MAXBSIZE - Filesystems are made out of blocks of at most MAXBSIZE bytes * per block. MAXBSIZE may be made larger without effecting * any existing filesystems as long as it does not exceed MAXPHYS, * and may be made smaller at the risk of not being able to use * filesystems which require a block size exceeding MAXBSIZE. * * BKVASIZE - Nominal buffer space per buffer, in bytes. BKVASIZE is the * minimum KVM memory reservation the kernel is willing to make. * Filesystems can of course request smaller chunks. Actual * backing memory uses a chunk size of a page (PAGE_SIZE). * * If you make BKVASIZE too small you risk seriously fragmenting * the buffer KVM map which may slow things down a bit. If you * make it too big the kernel will not be able to optimally use * the KVM memory reserved for the buffer cache and will wind * up with too-few buffers. * * The default is 16384, roughly 2x the block size used by a * normal UFS filesystem. */ #define MAXBSIZE 65536 /* must be power of 2 */ #define BKVASIZE 16384 /* must be power of 2 */ #define BKVAMASK (BKVASIZE-1) /* * MAXPATHLEN defines the longest permissible path length after expanding * symbolic links. It is used to allocate a temporary buffer from the buffer * pool in which to do the name expansion, hence should be a power of two, * and must be less than or equal to MAXBSIZE. MAXSYMLINKS defines the * maximum number of symbolic links that may be expanded in a path name. * It should be set high enough to allow all legitimate uses, but halt * infinite loops reasonably quickly. */ #define MAXPATHLEN PATH_MAX #define MAXSYMLINKS 32 /* Bit map related macros. */ #define setbit(a,i) ((a)[(i)/NBBY] |= 1<<((i)%NBBY)) #define clrbit(a,i) ((a)[(i)/NBBY] &= ~(1<<((i)%NBBY))) #define isset(a,i) ((a)[(i)/NBBY] & (1<<((i)%NBBY))) #define isclr(a,i) (((a)[(i)/NBBY] & (1<<((i)%NBBY))) == 0) /* Macros for counting and rounding. */ #ifndef howmany #define howmany(x, y) (((x)+((y)-1))/(y)) #endif #define rounddown(x, y) (((x)/(y))*(y)) #define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) /* to any y */ #define roundup2(x, y) (((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */ #define powerof2(x) ((((x)-1)&(x))==0) /* Macros for min/max. */ #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) #ifdef _KERNEL /* * Basic byte order function prototypes for non-inline functions. */ #ifndef LOCORE #ifndef _BYTEORDER_PROTOTYPED #define _BYTEORDER_PROTOTYPED __BEGIN_DECLS __uint32_t htonl(__uint32_t); __uint16_t htons(__uint16_t); __uint32_t ntohl(__uint32_t); __uint16_t ntohs(__uint16_t); __END_DECLS #endif #endif #ifndef lint #ifndef _BYTEORDER_FUNC_DEFINED #define _BYTEORDER_FUNC_DEFINED #define htonl(x) __htonl(x) #define htons(x) __htons(x) #define ntohl(x) __ntohl(x) #define ntohs(x) __ntohs(x) #endif /* !_BYTEORDER_FUNC_DEFINED */ #endif /* lint */ #endif /* _KERNEL */ /* * Constants for setting the parameters of the kernel memory allocator. * * 2 ** MINBUCKET is the smallest unit of memory that will be * allocated. It must be at least large enough to hold a pointer. * * Units of memory less or equal to MAXALLOCSAVE will permanently * allocate physical memory; requests for these size pieces of * memory are quite fast. Allocations greater than MAXALLOCSAVE must * always allocate and free physical memory; requests for these * size allocations should be done infrequently as they will be slow. * * Constraints: PAGE_SIZE <= MAXALLOCSAVE <= 2 ** (MINBUCKET + 14), and * MAXALLOCSIZE must be a power of two. */ #if defined(__alpha__) || defined(__ia64__) || defined(__sparc64__) #define MINBUCKET 5 /* 5 => min allocation of 32 bytes */ #else #define MINBUCKET 4 /* 4 => min allocation of 16 bytes */ #endif #define MAXALLOCSAVE (2 * PAGE_SIZE) /* * Scale factor for scaled integers used to count %cpu time and load avgs. * * The number of CPU `tick's that map to a unique `%age' can be expressed * by the formula (1 / (2 ^ (FSHIFT - 11))). The maximum load average that * can be calculated (assuming 32 bits) can be closely approximated using * the formula (2 ^ (2 * (16 - FSHIFT))) for (FSHIFT < 15). * * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age', * FSHIFT must be at least 11; this gives us a maximum load avg of ~1024. */ #define FSHIFT 11 /* bits to right of fixed binary point */ #define FSCALE (1<> (PAGE_SHIFT - DEV_BSHIFT)) #define ctodb(db) /* calculates pages to devblks */ \ ((db) << (PAGE_SHIFT - DEV_BSHIFT)) #endif /* _SYS_PARAM_H_ */ Index: head/sys/sys/pmc.h =================================================================== --- head/sys/sys/pmc.h (revision 157143) +++ head/sys/sys/pmc.h (revision 157144) @@ -1,1002 +1,1002 @@ /*- - * Copyright (c) 2003-2005, Joseph Koshy + * Copyright (c) 2003-2006, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_PMC_H_ #define _SYS_PMC_H_ #include #include #include #define PMC_MODULE_NAME "hwpmc" #define PMC_NAME_MAX 16 /* HW counter name size */ #define PMC_CLASS_MAX 4 /* #classes of PMCs in a system */ /* * Kernel<->userland API version number [MMmmpppp] * * Major numbers are to be incremented when an incompatible change to * the ABI occurs that older clients will not be able to handle. * * Minor numbers are incremented when a backwards compatible change * occurs that allows older correct programs to run unchanged. For * example, when support for a new PMC type is added. * * The patch version is incremented for every bug fix. */ -#define PMC_VERSION_MAJOR 0x01 -#define PMC_VERSION_MINOR 0x02 -#define PMC_VERSION_PATCH 0x0003 +#define PMC_VERSION_MAJOR 0x02 +#define PMC_VERSION_MINOR 0x00 +#define PMC_VERSION_PATCH 0x0000 #define PMC_VERSION (PMC_VERSION_MAJOR << 24 | \ PMC_VERSION_MINOR << 16 | PMC_VERSION_PATCH) /* * Kinds of CPUs known */ #define __PMC_CPUS() \ __PMC_CPU(AMD_K7, "AMD K7") \ __PMC_CPU(AMD_K8, "AMD K8") \ __PMC_CPU(INTEL_P5, "Intel Pentium") \ __PMC_CPU(INTEL_P6, "Intel Pentium Pro") \ __PMC_CPU(INTEL_CL, "Intel Celeron") \ __PMC_CPU(INTEL_PII, "Intel Pentium II") \ __PMC_CPU(INTEL_PIII, "Intel Pentium III") \ __PMC_CPU(INTEL_PM, "Intel Pentium M") \ __PMC_CPU(INTEL_PIV, "Intel Pentium IV") enum pmc_cputype { #undef __PMC_CPU #define __PMC_CPU(S,D) PMC_CPU_##S , __PMC_CPUS() }; #define PMC_CPU_FIRST PMC_CPU_AMD_K7 #define PMC_CPU_LAST PMC_CPU_INTEL_PIV /* * Classes of PMCs */ #define __PMC_CLASSES() \ __PMC_CLASS(TSC) /* CPU Timestamp counter */ \ __PMC_CLASS(K7) /* AMD K7 performance counters */ \ __PMC_CLASS(K8) /* AMD K8 performance counters */ \ __PMC_CLASS(P5) /* Intel Pentium counters */ \ __PMC_CLASS(P6) /* Intel Pentium Pro counters */ \ __PMC_CLASS(P4) /* Intel Pentium-IV counters */ enum pmc_class { #undef __PMC_CLASS #define __PMC_CLASS(N) PMC_CLASS_##N , __PMC_CLASSES() }; #define PMC_CLASS_FIRST PMC_CLASS_TSC #define PMC_CLASS_LAST PMC_CLASS_P4 /* * A PMC can be in the following states: * * Hardware states: * DISABLED -- administratively prohibited from being used. * FREE -- HW available for use * Software states: * ALLOCATED -- allocated * STOPPED -- allocated, but not counting events * RUNNING -- allocated, and in operation; 'pm_runcount' * holds the number of CPUs using this PMC at * a given instant * DELETED -- being destroyed */ #define __PMC_HWSTATES() \ __PMC_STATE(DISABLED) \ __PMC_STATE(FREE) #define __PMC_SWSTATES() \ __PMC_STATE(ALLOCATED) \ __PMC_STATE(STOPPED) \ __PMC_STATE(RUNNING) \ __PMC_STATE(DELETED) #define __PMC_STATES() \ __PMC_HWSTATES() \ __PMC_SWSTATES() enum pmc_state { #undef __PMC_STATE #define __PMC_STATE(S) PMC_STATE_##S, __PMC_STATES() __PMC_STATE(MAX) }; #define PMC_STATE_FIRST PMC_STATE_DISABLED #define PMC_STATE_LAST PMC_STATE_DELETED /* * An allocated PMC may used as a 'global' counter or as a * 'thread-private' one. Each such mode of use can be in either * statistical sampling mode or in counting mode. Thus a PMC in use * * SS i.e., SYSTEM STATISTICAL -- system-wide statistical profiling * SC i.e., SYSTEM COUNTER -- system-wide counting mode * TS i.e., THREAD STATISTICAL -- thread virtual, statistical profiling * TC i.e., THREAD COUNTER -- thread virtual, counting mode * * Statistical profiling modes rely on the PMC periodically delivering * a interrupt to the CPU (when the configured number of events have * been measured), so the PMC must have the ability to generate * interrupts. * * In counting modes, the PMC counts its configured events, with the * value of the PMC being read whenever needed by its owner process. * * The thread specific modes "virtualize" the PMCs -- the PMCs appear * to be thread private and count events only when the profiled thread * actually executes on the CPU. * * The system-wide "global" modes keep the PMCs running all the time * and are used to measure the behaviour of the whole system. */ #define __PMC_MODES() \ __PMC_MODE(SS, 0) \ __PMC_MODE(SC, 1) \ __PMC_MODE(TS, 2) \ __PMC_MODE(TC, 3) enum pmc_mode { #undef __PMC_MODE #define __PMC_MODE(M,N) PMC_MODE_##M = N, __PMC_MODES() }; #define PMC_MODE_FIRST PMC_MODE_SS #define PMC_MODE_LAST PMC_MODE_TC #define PMC_IS_COUNTING_MODE(mode) \ ((mode) == PMC_MODE_SC || (mode) == PMC_MODE_TC) #define PMC_IS_SYSTEM_MODE(mode) \ ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_SC) #define PMC_IS_SAMPLING_MODE(mode) \ ((mode) == PMC_MODE_SS || (mode) == PMC_MODE_TS) #define PMC_IS_VIRTUAL_MODE(mode) \ ((mode) == PMC_MODE_TS || (mode) == PMC_MODE_TC) /* * PMC row disposition */ #define __PMC_DISPOSITIONS(N) \ __PMC_DISP(STANDALONE) /* global/disabled counters */ \ __PMC_DISP(FREE) /* free/available */ \ __PMC_DISP(THREAD) /* thread-virtual PMCs */ \ __PMC_DISP(UNKNOWN) /* sentinel */ enum pmc_disp { #undef __PMC_DISP #define __PMC_DISP(D) PMC_DISP_##D , __PMC_DISPOSITIONS() }; #define PMC_DISP_FIRST PMC_DISP_STANDALONE #define PMC_DISP_LAST PMC_DISP_THREAD /* * Counter capabilities * * __PMC_CAPS(NAME, VALUE, DESCRIPTION) */ #define __PMC_CAPS() \ __PMC_CAP(INTERRUPT, 0, "generate interrupts") \ __PMC_CAP(USER, 1, "count user-mode events") \ __PMC_CAP(SYSTEM, 2, "count system-mode events") \ __PMC_CAP(EDGE, 3, "do edge detection of events") \ __PMC_CAP(THRESHOLD, 4, "ignore events below a threshold") \ __PMC_CAP(READ, 5, "read PMC counter") \ __PMC_CAP(WRITE, 6, "reprogram PMC counter") \ __PMC_CAP(INVERT, 7, "invert comparision sense") \ __PMC_CAP(QUALIFIER, 8, "further qualify monitored events") \ __PMC_CAP(PRECISE, 9, "perform precise sampling") \ __PMC_CAP(TAGGING, 10, "tag upstream events") \ __PMC_CAP(CASCADE, 11, "cascade counters") enum pmc_caps { #undef __PMC_CAP #define __PMC_CAP(NAME, VALUE, DESCR) PMC_CAP_##NAME = (1 << VALUE) , __PMC_CAPS() }; #define PMC_CAP_FIRST PMC_CAP_INTERRUPT #define PMC_CAP_LAST PMC_CAP_CASCADE /* * PMC Event Numbers * * These are generated from the definitions in "dev/hwpmc/pmc_events.h". */ enum pmc_event { #undef __PMC_EV #define __PMC_EV(C,N,D) PMC_EV_ ## C ## _ ## N , __PMC_EVENTS() }; #define PMC_EVENT_FIRST PMC_EV_TSC_TSC #define PMC_EVENT_LAST PMC_EV_P5_LAST /* * PMC SYSCALL INTERFACE */ /* * "PMC_OPS" -- these are the commands recognized by the kernel * module, and are used when performing a system call from userland. */ #define __PMC_OPS() \ __PMC_OP(CONFIGURELOG, "Set log file") \ __PMC_OP(FLUSHLOG, "Flush log file") \ __PMC_OP(GETCPUINFO, "Get system CPU information") \ __PMC_OP(GETDRIVERSTATS, "Get driver statistics") \ __PMC_OP(GETMODULEVERSION, "Get module version") \ __PMC_OP(GETPMCINFO, "Get per-cpu PMC information") \ __PMC_OP(PMCADMIN, "Set PMC state") \ __PMC_OP(PMCALLOCATE, "Allocate and configure a PMC") \ __PMC_OP(PMCATTACH, "Attach a PMC to a process") \ __PMC_OP(PMCDETACH, "Detach a PMC from a process") \ __PMC_OP(PMCGETMSR, "Get a PMC's hardware address") \ __PMC_OP(PMCRELEASE, "Release a PMC") \ __PMC_OP(PMCRW, "Read/Set a PMC") \ __PMC_OP(PMCSETCOUNT, "Set initial count/sampling rate") \ __PMC_OP(PMCSTART, "Start a PMC") \ __PMC_OP(PMCSTOP, "Start a PMC") \ __PMC_OP(WRITELOG, "Write a cookie to the log file") enum pmc_ops { #undef __PMC_OP #define __PMC_OP(N, D) PMC_OP_##N, __PMC_OPS() }; /* * Flags used in operations on PMCs. */ #define PMC_F_FORCE 0x00000001 /*OP ADMIN force operation */ #define PMC_F_DESCENDANTS 0x00000002 /*OP ALLOCATE track descendants */ #define PMC_F_LOG_PROCCSW 0x00000004 /*OP ALLOCATE track ctx switches */ #define PMC_F_LOG_PROCEXIT 0x00000008 /*OP ALLOCATE log proc exits */ #define PMC_F_NEWVALUE 0x00000010 /*OP RW write new value */ #define PMC_F_OLDVALUE 0x00000020 /*OP RW get old value */ #define PMC_F_KGMON 0x00000040 /*OP ALLOCATE kgmon(8) profiling */ /* internal flags */ #define PMC_F_ATTACHED_TO_OWNER 0x00010000 /*attached to owner*/ #define PMC_F_NEEDS_LOGFILE 0x00020000 /*needs log file */ #define PMC_F_ATTACH_DONE 0x00040000 /*attached at least once */ /* * Cookies used to denote allocated PMCs, and the values of PMCs. */ typedef uint32_t pmc_id_t; typedef uint64_t pmc_value_t; #define PMC_ID_INVALID (~ (pmc_id_t) 0) /* * PMC IDs have the following format: * * +--------+----------+-----------+-----------+ * | CPU | PMC MODE | PMC CLASS | ROW INDEX | * +--------+----------+-----------+-----------+ * * where each field is 8 bits wide. Field 'CPU' is set to the * requested CPU for system-wide PMCs or PMC_CPU_ANY for process-mode * PMCs. Field 'PMC MODE' is the allocated PMC mode. Field 'PMC * CLASS' is the class of the PMC. Field 'ROW INDEX' is the row index * for the PMC. * * The 'ROW INDEX' ranges over 0..NWPMCS where NHWPMCS is the total * number of hardware PMCs on this cpu. */ #define PMC_ID_TO_ROWINDEX(ID) ((ID) & 0xFF) #define PMC_ID_TO_CLASS(ID) (((ID) & 0xFF00) >> 8) #define PMC_ID_TO_MODE(ID) (((ID) & 0xFF0000) >> 16) #define PMC_ID_TO_CPU(ID) (((ID) & 0xFF000000) >> 24) #define PMC_ID_MAKE_ID(CPU,MODE,CLASS,ROWINDEX) \ ((((CPU) & 0xFF) << 24) | (((MODE) & 0xFF) << 16) | \ (((CLASS) & 0xFF) << 8) | ((ROWINDEX) & 0xFF)) /* * Data structures for system calls supported by the pmc driver. */ /* * OP PMCALLOCATE * * Allocate a PMC on the named CPU. */ #define PMC_CPU_ANY ~0 struct pmc_op_pmcallocate { uint32_t pm_caps; /* PMC_CAP_* */ uint32_t pm_cpu; /* CPU number or PMC_CPU_ANY */ enum pmc_class pm_class; /* class of PMC desired */ enum pmc_event pm_ev; /* [enum pmc_event] desired */ uint32_t pm_flags; /* additional modifiers PMC_F_* */ enum pmc_mode pm_mode; /* desired mode */ pmc_id_t pm_pmcid; /* [return] process pmc id */ union pmc_md_op_pmcallocate pm_md; /* MD layer extensions */ }; /* * OP PMCADMIN * * Set the administrative state (i.e., whether enabled or disabled) of * a PMC 'pm_pmc' on CPU 'pm_cpu'. Note that 'pm_pmc' specifies an * absolute PMC number and need not have been first allocated by the * calling process. */ struct pmc_op_pmcadmin { int pm_cpu; /* CPU# */ uint32_t pm_flags; /* flags */ int pm_pmc; /* PMC# */ enum pmc_state pm_state; /* desired state */ }; /* * OP PMCATTACH / OP PMCDETACH * * Attach/detach a PMC and a process. */ struct pmc_op_pmcattach { pmc_id_t pm_pmc; /* PMC to attach to */ pid_t pm_pid; /* target process */ }; /* * OP PMCSETCOUNT * * Set the sampling rate (i.e., the reload count) for statistical counters. * 'pm_pmcid' need to have been previously allocated using PMCALLOCATE. */ struct pmc_op_pmcsetcount { pmc_value_t pm_count; /* initial/sample count */ pmc_id_t pm_pmcid; /* PMC id to set */ }; /* * OP PMCRW * * Read the value of a PMC named by 'pm_pmcid'. 'pm_pmcid' needs * to have been previously allocated using PMCALLOCATE. */ struct pmc_op_pmcrw { uint32_t pm_flags; /* PMC_F_{OLD,NEW}VALUE*/ pmc_id_t pm_pmcid; /* pmc id */ pmc_value_t pm_value; /* new&returned value */ }; /* * OP GETPMCINFO * * retrieve PMC state for a named CPU. The caller is expected to * allocate 'npmc' * 'struct pmc_info' bytes of space for the return * values. */ struct pmc_info { char pm_name[PMC_NAME_MAX]; /* pmc name */ enum pmc_class pm_class; /* enum pmc_class */ int pm_enabled; /* whether enabled */ enum pmc_disp pm_rowdisp; /* FREE, THREAD or STANDLONE */ pid_t pm_ownerpid; /* owner, or -1 */ enum pmc_mode pm_mode; /* current mode [enum pmc_mode] */ enum pmc_event pm_event; /* current event */ uint32_t pm_flags; /* current flags */ pmc_value_t pm_reloadcount; /* sampling counters only */ }; struct pmc_op_getpmcinfo { int32_t pm_cpu; /* 0 <= cpu < mp_maxid */ struct pmc_info pm_pmcs[]; /* space for 'npmc' structures */ }; /* * OP GETCPUINFO * * Retrieve system CPU information. */ struct pmc_classinfo { enum pmc_class pm_class; /* class id */ uint32_t pm_caps; /* counter capabilities */ uint32_t pm_width; /* width of the PMC */ }; struct pmc_op_getcpuinfo { enum pmc_cputype pm_cputype; /* what kind of CPU */ uint32_t pm_ncpu; /* number of CPUs */ uint32_t pm_npmc; /* #PMCs per CPU */ uint32_t pm_nclass; /* #classes of PMCs */ struct pmc_classinfo pm_classes[PMC_CLASS_MAX]; }; /* * OP CONFIGURELOG * * Configure a log file for writing system-wide statistics to. */ struct pmc_op_configurelog { int pm_flags; int pm_logfd; /* logfile fd (or -1) */ }; /* * OP GETDRIVERSTATS * * Retrieve pmc(4) driver-wide statistics. */ struct pmc_op_getdriverstats { int pm_intr_ignored; /* #interrupts ignored */ int pm_intr_processed; /* #interrupts processed */ int pm_intr_bufferfull; /* #interrupts with ENOSPC */ int pm_syscalls; /* #syscalls */ int pm_syscall_errors; /* #syscalls with errors */ int pm_buffer_requests; /* #buffer requests */ int pm_buffer_requests_failed; /* #failed buffer requests */ int pm_log_sweeps; /* #sample buffer processing passes */ }; /* * OP RELEASE / OP START / OP STOP * * Simple operations on a PMC id. */ struct pmc_op_simple { pmc_id_t pm_pmcid; }; /* * OP WRITELOG * * Flush the current log buffer and write 4 bytes of user data to it. */ struct pmc_op_writelog { uint32_t pm_userdata; }; /* * OP GETMSR * * Retrieve the machine specific address assoicated with the allocated * PMC. This number can be used subsequently with a read-performance-counter * instruction. */ struct pmc_op_getmsr { uint32_t pm_msr; /* machine specific address */ pmc_id_t pm_pmcid; /* allocated pmc id */ }; #ifdef _KERNEL #include #include #define PMC_REQUEST_POOL_SIZE 32 #define PMC_HASH_SIZE 16 #define PMC_MTXPOOL_SIZE 32 #define PMC_LOG_BUFFER_SIZE 4 #define PMC_NLOGBUFFERS 16 -#define PMC_NSAMPLES 16 +#define PMC_NSAMPLES 32 #define PMC_SYSCTL_NAME_PREFIX "kern." PMC_MODULE_NAME "." /* * Locking keys * * (b) - pmc_bufferlist_mtx (spin lock) * (k) - pmc_kthread_mtx (sleep lock) * (o) - po->po_mtx (spin lock) */ /* * PMC commands */ struct pmc_syscall_args { uint32_t pmop_code; /* one of PMC_OP_* */ void *pmop_data; /* syscall parameter */ }; /* * Interface to processor specific s1tuff */ /* * struct pmc_descr * * Machine independent (i.e., the common parts) of a human readable * PMC description. */ struct pmc_descr { char pd_name[PMC_NAME_MAX]; /* name */ uint32_t pd_caps; /* capabilities */ enum pmc_class pd_class; /* class of the PMC */ uint32_t pd_width; /* width in bits */ }; /* * struct pmc_target * * This structure records all the target processes associated with a * PMC. */ struct pmc_target { LIST_ENTRY(pmc_target) pt_next; struct pmc_process *pt_process; /* target descriptor */ }; /* * struct pmc * * Describes each allocated PMC. * * Each PMC has precisely one owner, namely the process that allocated * the PMC. * * A PMC may be attached to multiple target processes. The * 'pm_targets' field links all the target processes being monitored * by this PMC. * * The 'pm_savedvalue' field is protected by a mutex. * * On a multi-cpu machine, multiple target threads associated with a * process-virtual PMC could be concurrently executing on different * CPUs. The 'pm_runcount' field is atomically incremented every time * the PMC gets scheduled on a CPU and atomically decremented when it * get descheduled. Deletion of a PMC is only permitted when this * field is '0'. * */ struct pmc { LIST_HEAD(,pmc_target) pm_targets; /* list of target processes */ LIST_ENTRY(pmc) pm_next; /* owner's list */ /* * System-wide PMCs are allocated on a CPU and are not moved * around. For system-wide PMCs we record the CPU the PMC was * allocated on in the 'CPU' field of the pmc ID. * * Virtual PMCs run on whichever CPU is currently executing * their targets' threads. For these PMCs we need to save * their current PMC counter values when they are taken off * CPU. */ union { pmc_value_t pm_savedvalue; /* Virtual PMCS */ } pm_gv; /* * For sampling mode PMCs, we keep track of the PMC's "reload * count", which is the counter value to be loaded in when * arming the PMC for the next counting session. For counting * modes on PMCs that are read-only (e.g., the x86 TSC), we * keep track of the initial value at the start of * counting-mode operation. */ union { pmc_value_t pm_reloadcount; /* sampling PMC modes */ pmc_value_t pm_initial; /* counting PMC modes */ } pm_sc; uint32_t pm_stalled; /* true for stalled sampling PMCs */ uint32_t pm_caps; /* PMC capabilities */ enum pmc_event pm_event; /* event being measured */ uint32_t pm_flags; /* additional flags PMC_F_... */ struct pmc_owner *pm_owner; /* owner thread state */ uint32_t pm_runcount; /* #cpus currently on */ enum pmc_state pm_state; /* current PMC state */ /* * The PMC ID field encodes the row-index for the PMC, its * mode, class and the CPU# associated with the PMC. */ pmc_id_t pm_id; /* allocated PMC id */ /* md extensions */ union pmc_md_pmc pm_md; }; /* * Accessor macros for 'struct pmc' */ #define PMC_TO_MODE(P) PMC_ID_TO_MODE((P)->pm_id) #define PMC_TO_CLASS(P) PMC_ID_TO_CLASS((P)->pm_id) #define PMC_TO_ROWINDEX(P) PMC_ID_TO_ROWINDEX((P)->pm_id) #define PMC_TO_CPU(P) PMC_ID_TO_CPU((P)->pm_id) /* * struct pmc_process * * Record a 'target' process being profiled. * * The target process being profiled could be different from the owner * process which allocated the PMCs. Each target process descriptor * is associated with NHWPMC 'struct pmc *' pointers. Each PMC at a * given hardware row-index 'n' will use slot 'n' of the 'pp_pmcs[]' * array. The size of this structure is thus PMC architecture * dependent. * * TODO: Only process-private counting mode PMCs may be attached to a * process different from the allocator process (since we do not have * the infrastructure to make sense of an interrupted PC value from a * 'target' process (yet)). * */ struct pmc_targetstate { struct pmc *pp_pmc; /* target PMC */ pmc_value_t pp_pmcval; /* per-process value */ }; struct pmc_process { LIST_ENTRY(pmc_process) pp_next; /* hash chain */ int pp_refcnt; /* reference count */ uint32_t pp_flags; /* flags PMC_PP_* */ struct proc *pp_proc; /* target thread */ struct pmc_targetstate pp_pmcs[]; /* NHWPMCs */ }; #define PMC_PP_ENABLE_MSR_ACCESS 0x00000001 /* * struct pmc_owner * * We associate a PMC with an 'owner' process. * * A process can be associated with 0..NCPUS*NHWPMC PMCs during its * lifetime, where NCPUS is the numbers of CPUS in the system and * NHWPMC is the number of hardware PMCs per CPU. These are * maintained in the list headed by the 'po_pmcs' to save on space. * */ struct pmc_owner { LIST_ENTRY(pmc_owner) po_next; /* hash chain */ LIST_ENTRY(pmc_owner) po_ssnext; /* list of SS PMC owners */ LIST_HEAD(, pmc) po_pmcs; /* owned PMC list */ TAILQ_HEAD(, pmclog_buffer) po_logbuffers; /* (o) logbuffer list */ struct mtx po_mtx; /* spin lock for (o) */ struct proc *po_owner; /* owner proc */ uint32_t po_flags; /* (k) flags PMC_PO_* */ struct proc *po_kthread; /* (k) helper kthread */ struct pmclog_buffer *po_curbuf; /* current log buffer */ struct file *po_file; /* file reference */ int po_error; /* recorded error */ int po_sscount; /* # SS PMCs owned */ }; #define PMC_PO_OWNS_LOGFILE 0x00000001 /* has a log file */ #define PMC_PO_IN_FLUSH 0x00000010 /* in the middle of a flush */ /* * struct pmc_hw -- describe the state of the PMC hardware * * When in use, a HW PMC is associated with one allocated 'struct pmc' * pointed to by field 'phw_pmc'. When inactive, this field is NULL. * * On an SMP box, one or more HW PMC's in process virtual mode with * the same 'phw_pmc' could be executing on different CPUs. In order * to handle this case correctly, we need to ensure that only * incremental counts get added to the saved value in the associated * 'struct pmc'. The 'phw_save' field is used to keep the saved PMC * value at the time the hardware is started during this context * switch (i.e., the difference between the new (hardware) count and * the saved count is atomically added to the count field in 'struct * pmc' at context switch time). * */ struct pmc_hw { uint32_t phw_state; /* see PHW_* macros below */ struct pmc *phw_pmc; /* current thread PMC */ }; #define PMC_PHW_RI_MASK 0x000000FF #define PMC_PHW_CPU_SHIFT 8 #define PMC_PHW_CPU_MASK 0x0000FF00 #define PMC_PHW_FLAGS_SHIFT 16 #define PMC_PHW_FLAGS_MASK 0xFFFF0000 #define PMC_PHW_INDEX_TO_STATE(ri) ((ri) & PMC_PHW_RI_MASK) #define PMC_PHW_STATE_TO_INDEX(state) ((state) & PMC_PHW_RI_MASK) #define PMC_PHW_CPU_TO_STATE(cpu) (((cpu) << PMC_PHW_CPU_SHIFT) & \ PMC_PHW_CPU_MASK) #define PMC_PHW_STATE_TO_CPU(state) (((state) & PMC_PHW_CPU_MASK) >> \ PMC_PHW_CPU_SHIFT) #define PMC_PHW_FLAGS_TO_STATE(flags) (((flags) << PMC_PHW_FLAGS_SHIFT) & \ PMC_PHW_FLAGS_MASK) #define PMC_PHW_STATE_TO_FLAGS(state) (((state) & PMC_PHW_FLAGS_MASK) >> \ PMC_PHW_FLAGS_SHIFT) #define PMC_PHW_FLAG_IS_ENABLED (PMC_PHW_FLAGS_TO_STATE(0x01)) #define PMC_PHW_FLAG_IS_SHAREABLE (PMC_PHW_FLAGS_TO_STATE(0x02)) /* * struct pmc_sample * * Space for N (tunable) PC samples and associated control data. */ struct pmc_sample { uintfptr_t ps_pc; /* PC value at interrupt */ struct pmc *ps_pmc; /* interrupting PMC */ int ps_usermode; /* true for user mode PCs */ pid_t ps_pid; /* process PID or -1 */ }; struct pmc_samplebuffer { struct pmc_sample * volatile ps_read; /* read pointer */ struct pmc_sample * volatile ps_write; /* write pointer */ struct pmc_sample *ps_fence; /* one beyond ps_samples[] */ struct pmc_sample ps_samples[]; /* array of sample entries */ }; /* * struct pmc_cpustate * * A CPU is modelled as a collection of HW PMCs with space for additional * flags. */ struct pmc_cpu { uint32_t pc_state; /* physical cpu number + flags */ struct pmc_samplebuffer *pc_sb; /* space for samples */ struct pmc_hw *pc_hwpmcs[]; /* 'npmc' pointers */ /* other machine dependent fields come here */ }; #define PMC_PCPU_CPU_MASK 0x000000FF #define PMC_PCPU_FLAGS_MASK 0xFFFFFF00 #define PMC_PCPU_FLAGS_SHIFT 8 #define PMC_PCPU_STATE_TO_CPU(S) ((S) & PMC_PCPU_CPU_MASK) #define PMC_PCPU_STATE_TO_FLAGS(S) (((S) & PMC_PCPU_FLAGS_MASK) >> PMC_PCPU_FLAGS_SHIFT) #define PMC_PCPU_FLAGS_TO_STATE(F) (((F) << PMC_PCPU_FLAGS_SHIFT) & PMC_PCPU_FLAGS_MASK) #define PMC_PCPU_CPU_TO_STATE(C) ((C) & PMC_PCPU_CPU_MASK) #define PMC_PCPU_FLAG_HTT (PMC_PCPU_FLAGS_TO_STATE(0x1)) /* * struct pmc_binding * * CPU binding information. */ struct pmc_binding { int pb_bound; /* is bound? */ int pb_cpu; /* if so, to which CPU */ }; /* * struct pmc_mdep * * Machine dependent bits needed per CPU type. */ struct pmc_mdep { uint32_t pmd_cputype; /* from enum pmc_cputype */ uint32_t pmd_npmc; /* max PMCs per CPU */ uint32_t pmd_nclass; /* # PMC classes supported */ struct pmc_classinfo pmd_classes[PMC_CLASS_MAX]; int pmd_nclasspmcs[PMC_CLASS_MAX]; /* * Methods */ int (*pmd_init)(int _cpu); /* machine dependent initialization */ int (*pmd_cleanup)(int _cpu); /* machine dependent cleanup */ /* thread context switch in/out */ int (*pmd_switch_in)(struct pmc_cpu *_p, struct pmc_process *_pp); int (*pmd_switch_out)(struct pmc_cpu *_p, struct pmc_process *_pp); /* configuring/reading/writing the hardware PMCs */ int (*pmd_config_pmc)(int _cpu, int _ri, struct pmc *_pm); int (*pmd_get_config)(int _cpu, int _ri, struct pmc **_ppm); int (*pmd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value); int (*pmd_write_pmc)(int _cpu, int _ri, pmc_value_t _value); /* pmc allocation/release */ int (*pmd_allocate_pmc)(int _cpu, int _ri, struct pmc *_t, const struct pmc_op_pmcallocate *_a); int (*pmd_release_pmc)(int _cpu, int _ri, struct pmc *_pm); /* starting and stopping PMCs */ int (*pmd_start_pmc)(int _cpu, int _ri); int (*pmd_stop_pmc)(int _cpu, int _ri); /* handle a PMC interrupt */ int (*pmd_intr)(int _cpu, uintptr_t _pc, int _usermode); int (*pmd_describe)(int _cpu, int _ri, struct pmc_info *_pi, struct pmc **_ppmc); int (*pmd_get_msr)(int _ri, uint32_t *_msr); }; /* * Per-CPU state. This is an array of 'mp_ncpu' pointers * to struct pmc_cpu descriptors. */ extern struct pmc_cpu **pmc_pcpu; extern struct pmc_mdep *md; /* driver statistics */ extern struct pmc_op_getdriverstats pmc_stats; #if defined(DEBUG) && DEBUG /* debug flags, major flag groups */ struct pmc_debugflags { int pdb_CPU; int pdb_CSW; int pdb_LOG; int pdb_MDP; int pdb_MOD; int pdb_OWN; int pdb_PMC; int pdb_PRC; int pdb_SAM; }; extern struct pmc_debugflags pmc_debugflags; #define PMC_DEBUG_STRSIZE 128 #define PMC_DEBUG_DEFAULT_FLAGS { 0, 0, 0, 0, 0, 0, 0, 0 } #define PMCDBG(M,N,L,F,...) do { \ if (pmc_debugflags.pdb_ ## M & (1 << PMC_DEBUG_MIN_ ## N)) \ printf(#M ":" #N ":" #L ": " F "\n", __VA_ARGS__); \ } while (0) /* Major numbers */ #define PMC_DEBUG_MAJ_CPU 0 /* cpu switches */ #define PMC_DEBUG_MAJ_CSW 1 /* context switches */ #define PMC_DEBUG_MAJ_LOG 2 /* logging */ #define PMC_DEBUG_MAJ_MDP 3 /* machine dependent */ #define PMC_DEBUG_MAJ_MOD 4 /* misc module infrastructure */ #define PMC_DEBUG_MAJ_OWN 5 /* owner */ #define PMC_DEBUG_MAJ_PMC 6 /* pmc management */ #define PMC_DEBUG_MAJ_PRC 7 /* processes */ #define PMC_DEBUG_MAJ_SAM 8 /* sampling */ /* Minor numbers */ /* Common (8 bits) */ #define PMC_DEBUG_MIN_ALL 0 /* allocation */ #define PMC_DEBUG_MIN_REL 1 /* release */ #define PMC_DEBUG_MIN_OPS 2 /* ops: start, stop, ... */ #define PMC_DEBUG_MIN_INI 3 /* init */ #define PMC_DEBUG_MIN_FND 4 /* find */ /* MODULE */ #define PMC_DEBUG_MIN_PMH 14 /* pmc_hook */ #define PMC_DEBUG_MIN_PMS 15 /* pmc_syscall */ /* OWN */ #define PMC_DEBUG_MIN_ORM 8 /* owner remove */ #define PMC_DEBUG_MIN_OMR 9 /* owner maybe remove */ /* PROCESSES */ #define PMC_DEBUG_MIN_TLK 8 /* link target */ #define PMC_DEBUG_MIN_TUL 9 /* unlink target */ #define PMC_DEBUG_MIN_EXT 10 /* process exit */ #define PMC_DEBUG_MIN_EXC 11 /* process exec */ #define PMC_DEBUG_MIN_FRK 12 /* process fork */ #define PMC_DEBUG_MIN_ATT 13 /* attach/detach */ #define PMC_DEBUG_MIN_SIG 14 /* signalling */ /* CONTEXT SWITCHES */ #define PMC_DEBUG_MIN_SWI 8 /* switch in */ #define PMC_DEBUG_MIN_SWO 9 /* switch out */ /* PMC */ #define PMC_DEBUG_MIN_REG 8 /* pmc register */ #define PMC_DEBUG_MIN_ALR 9 /* allocate row */ /* MACHINE DEPENDENT LAYER */ #define PMC_DEBUG_MIN_REA 8 /* read */ #define PMC_DEBUG_MIN_WRI 9 /* write */ #define PMC_DEBUG_MIN_CFG 10 /* config */ #define PMC_DEBUG_MIN_STA 11 /* start */ #define PMC_DEBUG_MIN_STO 12 /* stop */ #define PMC_DEBUG_MIN_INT 13 /* interrupts */ /* CPU */ #define PMC_DEBUG_MIN_BND 8 /* bind */ #define PMC_DEBUG_MIN_SEL 9 /* select */ /* LOG */ #define PMC_DEBUG_MIN_GTB 8 /* get buf */ #define PMC_DEBUG_MIN_SIO 9 /* schedule i/o */ #define PMC_DEBUG_MIN_FLS 10 /* flush */ #define PMC_DEBUG_MIN_SAM 11 /* sample */ #else #define PMCDBG(M,N,L,F,...) /* nothing */ #endif /* declare a dedicated memory pool */ MALLOC_DECLARE(M_PMC); /* * Functions */ struct pmc_mdep *pmc_md_initialize(void); /* MD init function */ int pmc_getrowdisp(int _ri); int pmc_process_interrupt(int _cpu, struct pmc *_pm, uintfptr_t _pc, int _usermode); #endif /* _KERNEL */ #endif /* _SYS_PMC_H_ */ Index: head/sys/sys/pmckern.h =================================================================== --- head/sys/sys/pmckern.h (revision 157143) +++ head/sys/sys/pmckern.h (revision 157144) @@ -1,112 +1,126 @@ /*- - * Copyright (c) 2003-2005, Joseph Koshy + * Copyright (c) 2003-2006, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * PMC interface used by the base kernel. */ #ifndef _SYS_PMCKERN_H_ #define _SYS_PMCKERN_H_ #include #include #include #include #include #define PMC_FN_PROCESS_EXEC 1 #define PMC_FN_CSW_IN 2 #define PMC_FN_CSW_OUT 3 #define PMC_FN_DO_SAMPLES 4 +#define PMC_FN_KLD_LOAD 5 +#define PMC_FN_KLD_UNLOAD 6 +#define PMC_FN_MMAP 7 +#define PMC_FN_MUNMAP 8 struct pmckern_procexec { int pm_credentialschanged; - uintptr_t pm_entryaddr; + uintfptr_t pm_entryaddr; +}; + +struct pmckern_map_in { + void *pm_file; /* filename or vnode pointer */ + uintfptr_t pm_address; /* address object is loaded at */ +}; + +struct pmckern_map_out { + uintfptr_t pm_address; /* start address of region */ + size_t pm_size; /* size of unmapped region */ }; /* hook */ extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg); extern int (*pmc_intr)(int _cpu, uintptr_t _pc, int _usermode); /* SX lock protecting the hook */ extern struct sx pmc_sx; /* Per-cpu flags indicating availability of sampling data */ extern volatile cpumask_t pmc_cpumask; /* Count of system-wide sampling PMCs in existence */ extern volatile int pmc_ss_count; /* kernel version number */ extern const int pmc_kernel_version; /* Hook invocation; for use within the kernel */ #define PMC_CALL_HOOK(t, cmd, arg) \ do { \ sx_slock(&pmc_sx); \ if (pmc_hook != NULL) \ (pmc_hook)((t), (cmd), (arg)); \ sx_sunlock(&pmc_sx); \ } while (0) /* Hook invocation that needs an exclusive lock */ #define PMC_CALL_HOOK_X(t, cmd, arg) \ do { \ sx_xlock(&pmc_sx); \ if (pmc_hook != NULL) \ (pmc_hook)((t), (cmd), (arg)); \ sx_xunlock(&pmc_sx); \ } while (0) /* * Some hook invocations (e.g., from context switch and clock handling * code) need to be lock-free. */ #define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg) \ do { \ if (pmc_hook != NULL) \ (pmc_hook)((t), (cmd), (arg)); \ } while (0) #define PMC_SWITCH_CONTEXT(t,cmd) PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL) /* Check if a process is using HWPMCs.*/ #define PMC_PROC_IS_USING_PMCS(p) \ (__predict_false(atomic_load_acq_int(&(p)->p_flag) & \ P_HWPMC)) #define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) /* Check if a CPU has recorded samples. */ #define PMC_CPU_HAS_SAMPLES(C) (__predict_false(pmc_cpumask & (1 << (C)))) /* helper functions */ int pmc_cpu_is_disabled(int _cpu); int pmc_cpu_is_logical(int _cpu); #endif /* _SYS_PMCKERN_H_ */ Index: head/sys/sys/pmclog.h =================================================================== --- head/sys/sys/pmclog.h (revision 157143) +++ head/sys/sys/pmclog.h (revision 157144) @@ -1,233 +1,250 @@ /*- - * Copyright (c) 2005 Joseph Koshy + * Copyright (c) 2005-2006, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_PMCLOG_H_ #define _SYS_PMCLOG_H_ #include enum pmclog_type { + /* V1 ABI */ PMCLOG_TYPE_CLOSELOG, PMCLOG_TYPE_DROPNOTIFY, PMCLOG_TYPE_INITIALIZE, - PMCLOG_TYPE_MAPPINGCHANGE, + PMCLOG_TYPE_MAPPINGCHANGE, /* unused in v1 */ PMCLOG_TYPE_PCSAMPLE, PMCLOG_TYPE_PMCALLOCATE, PMCLOG_TYPE_PMCATTACH, PMCLOG_TYPE_PMCDETACH, PMCLOG_TYPE_PROCCSW, PMCLOG_TYPE_PROCEXEC, PMCLOG_TYPE_PROCEXIT, PMCLOG_TYPE_PROCFORK, PMCLOG_TYPE_SYSEXIT, - PMCLOG_TYPE_USERDATA + PMCLOG_TYPE_USERDATA, + /* + * V2 ABI + * + * The MAP_{IN,OUT} event types obsolete the MAPPING_CHANGE + * event type of the older (V1) ABI. + */ + PMCLOG_TYPE_MAP_IN, + PMCLOG_TYPE_MAP_OUT }; -#define PMCLOG_MAPPING_INSERT 0x01 -#define PMCLOG_MAPPING_DELETE 0x02 +#define PMCLOG_MAPPING_INSERT 0x01 /* obsolete */ +#define PMCLOG_MAPPING_DELETE 0x02 /* obsolete */ /* * A log entry descriptor comprises of a 32 bit header and a 64 bit * time stamp followed by as many 32 bit words are required to record * the event. * * Header field format: * * 31 24 16 0 * +------------+------------+-----------------------------------+ * | MAGIC | TYPE | LENGTH | * +------------+------------+-----------------------------------+ * * MAGIC is the constant PMCLOG_HEADER_MAGIC. * TYPE contains a value of type enum pmclog_type. * LENGTH contains the length of the event record, in bytes. */ #define PMCLOG_ENTRY_HEADER \ uint32_t pl_header; \ uint32_t pl_ts_sec; \ uint32_t pl_ts_nsec; /* * The following structures are used to describe the size of each kind * of log entry to sizeof(). To keep the compiler from adding * padding, the fields of each structure are aligned to their natural * boundaries, and the structures are marked as 'packed'. * * The actual reading and writing of the log file is always in terms * of 4 byte quantities. */ struct pmclog_closelog { PMCLOG_ENTRY_HEADER }; struct pmclog_dropnotify { PMCLOG_ENTRY_HEADER }; struct pmclog_initialize { PMCLOG_ENTRY_HEADER uint32_t pl_version; /* driver version */ uint32_t pl_cpu; /* enum pmc_cputype */ } __packed; -struct pmclog_mappingchange { +struct pmclog_map_in { PMCLOG_ENTRY_HEADER - uint32_t pl_type; - uintfptr_t pl_start; /* 8 byte aligned */ - uintfptr_t pl_end; uint32_t pl_pid; + uintfptr_t pl_start; /* 8 byte aligned */ char pl_pathname[PATH_MAX]; } __packed; +struct pmclog_map_out { + PMCLOG_ENTRY_HEADER + uint32_t pl_pid; + uintfptr_t pl_start; /* 8 byte aligned */ + uintfptr_t pl_end; +} __packed; struct pmclog_pcsample { PMCLOG_ENTRY_HEADER uint32_t pl_pid; uintfptr_t pl_pc; /* 8 byte aligned */ uint32_t pl_pmcid; uint32_t pl_usermode; } __packed; struct pmclog_pmcallocate { PMCLOG_ENTRY_HEADER uint32_t pl_pmcid; uint32_t pl_event; uint32_t pl_flags; } __packed; struct pmclog_pmcattach { PMCLOG_ENTRY_HEADER uint32_t pl_pmcid; uint32_t pl_pid; char pl_pathname[PATH_MAX]; } __packed; struct pmclog_pmcdetach { PMCLOG_ENTRY_HEADER uint32_t pl_pmcid; uint32_t pl_pid; } __packed; struct pmclog_proccsw { PMCLOG_ENTRY_HEADER uint32_t pl_pmcid; uint64_t pl_value; /* keep 8 byte aligned */ uint32_t pl_pid; } __packed; struct pmclog_procexec { PMCLOG_ENTRY_HEADER uint32_t pl_pid; uintfptr_t pl_start; /* keep 8 byte aligned */ uint32_t pl_pmcid; char pl_pathname[PATH_MAX]; } __packed; struct pmclog_procexit { PMCLOG_ENTRY_HEADER uint32_t pl_pmcid; uint64_t pl_value; /* keep 8 byte aligned */ uint32_t pl_pid; } __packed; struct pmclog_procfork { PMCLOG_ENTRY_HEADER uint32_t pl_oldpid; uint32_t pl_newpid; } __packed; struct pmclog_sysexit { PMCLOG_ENTRY_HEADER uint32_t pl_pid; } __packed; struct pmclog_userdata { PMCLOG_ENTRY_HEADER uint32_t pl_userdata; } __packed; union pmclog_entry { /* only used to size scratch areas */ struct pmclog_closelog pl_cl; struct pmclog_dropnotify pl_dn; struct pmclog_initialize pl_i; + struct pmclog_map_in pl_mi; + struct pmclog_map_out pl_mo; struct pmclog_pcsample pl_s; struct pmclog_pmcallocate pl_a; struct pmclog_pmcattach pl_t; struct pmclog_pmcdetach pl_d; struct pmclog_proccsw pl_c; struct pmclog_procexec pl_x; struct pmclog_procexit pl_e; struct pmclog_procfork pl_f; struct pmclog_sysexit pl_se; struct pmclog_userdata pl_u; }; #define PMCLOG_HEADER_MAGIC 0xEEU #define PMCLOG_HEADER_TO_LENGTH(H) \ ((H) & 0x0000FFFF) #define PMCLOG_HEADER_TO_TYPE(H) \ (((H) & 0x00FF0000) >> 16) #define PMCLOG_HEADER_TO_MAGIC(H) \ (((H) & 0xFF000000) >> 24) #define PMCLOG_HEADER_CHECK_MAGIC(H) \ (PMCLOG_HEADER_TO_MAGIC(H) == PMCLOG_HEADER_MAGIC) #ifdef _KERNEL /* * Prototypes */ int pmclog_configure_log(struct pmc_owner *_po, int _logfd); int pmclog_deconfigure_log(struct pmc_owner *_po); int pmclog_flush(struct pmc_owner *_po); void pmclog_initialize(void); void pmclog_process_closelog(struct pmc_owner *po); void pmclog_process_dropnotify(struct pmc_owner *po); -void pmclog_process_mappingchange(struct pmc_owner *po, pid_t pid, int type, - uintfptr_t start, uintfptr_t end, char *path); +void pmclog_process_map_in(struct pmc_owner *po, pid_t pid, + uintfptr_t start, const char *path); +void pmclog_process_map_out(struct pmc_owner *po, pid_t pid, + uintfptr_t start, uintfptr_t end); void pmclog_process_pcsample(struct pmc *_pm, struct pmc_sample *_ps); void pmclog_process_pmcallocate(struct pmc *_pm); void pmclog_process_pmcattach(struct pmc *_pm, pid_t _pid, char *_path); void pmclog_process_pmcdetach(struct pmc *_pm, pid_t _pid); void pmclog_process_proccsw(struct pmc *_pm, struct pmc_process *_pp, pmc_value_t _v); void pmclog_process_procexec(struct pmc_owner *_po, pmc_id_t _pmid, pid_t _pid, uintfptr_t _startaddr, char *_path); void pmclog_process_procexit(struct pmc *_pm, struct pmc_process *_pp); void pmclog_process_procfork(struct pmc_owner *_po, pid_t _oldpid, pid_t _newpid); void pmclog_process_sysexit(struct pmc_owner *_po, pid_t _pid); int pmclog_process_userlog(struct pmc_owner *_po, struct pmc_op_writelog *_wl); void pmclog_shutdown(void); #endif /* _KERNEL */ #endif /* _SYS_PMCLOG_H_ */ Index: head/sys/vm/vm_mmap.c =================================================================== --- head/sys/vm/vm_mmap.c (revision 157143) +++ head/sys/vm/vm_mmap.c (revision 157144) @@ -1,1372 +1,1413 @@ /*- * Copyright (c) 1988 University of Utah. * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ * * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 */ /* * Mapped file (mmap) interface to VM */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" +#include "opt_hwpmc_hooks.h" #include "opt_mac.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#ifdef HWPMC_HOOKS +#include +#endif + #ifndef _SYS_SYSPROTO_H_ struct sbrk_args { int incr; }; #endif static int max_proc_mmap; SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0, ""); /* * Set the maximum number of vm_map_entry structures per process. Roughly * speaking vm_map_entry structures are tiny, so allowing them to eat 1/100 * of our KVM malloc space still results in generous limits. We want a * default that is good enough to prevent the kernel running out of resources * if attacked from compromised user account but generous enough such that * multi-threaded processes are not unduly inconvenienced. */ static void vmmapentry_rsrc_init(void *); SYSINIT(vmmersrc, SI_SUB_KVM_RSRC, SI_ORDER_FIRST, vmmapentry_rsrc_init, NULL) static void vmmapentry_rsrc_init(dummy) void *dummy; { max_proc_mmap = vm_kmem_size / sizeof(struct vm_map_entry); max_proc_mmap /= 100; } static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, int *, struct vnode *, vm_ooffset_t, vm_object_t *); static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *, int *, struct cdev *, vm_ooffset_t, vm_object_t *); /* * MPSAFE */ /* ARGSUSED */ int sbrk(td, uap) struct thread *td; struct sbrk_args *uap; { /* Not yet implemented */ return (EOPNOTSUPP); } #ifndef _SYS_SYSPROTO_H_ struct sstk_args { int incr; }; #endif /* * MPSAFE */ /* ARGSUSED */ int sstk(td, uap) struct thread *td; struct sstk_args *uap; { /* Not yet implemented */ return (EOPNOTSUPP); } #if defined(COMPAT_43) #ifndef _SYS_SYSPROTO_H_ struct getpagesize_args { int dummy; }; #endif /* ARGSUSED */ int ogetpagesize(td, uap) struct thread *td; struct getpagesize_args *uap; { /* MP SAFE */ td->td_retval[0] = PAGE_SIZE; return (0); } #endif /* COMPAT_43 */ /* * Memory Map (mmap) system call. Note that the file offset * and address are allowed to be NOT page aligned, though if * the MAP_FIXED flag it set, both must have the same remainder * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not * page-aligned, the actual mapping starts at trunc_page(addr) * and the return value is adjusted up by the page offset. * * Generally speaking, only character devices which are themselves * memory-based, such as a video framebuffer, can be mmap'd. Otherwise * there would be no cache coherency between a descriptor and a VM mapping * both to the same character device. * * Block devices can be mmap'd no matter what they represent. Cache coherency * is maintained as long as you do not write directly to the underlying * character device. */ #ifndef _SYS_SYSPROTO_H_ struct mmap_args { void *addr; size_t len; int prot; int flags; int fd; long pad; off_t pos; }; #endif /* * MPSAFE */ int mmap(td, uap) struct thread *td; struct mmap_args *uap; { +#ifdef HWPMC_HOOKS + struct pmckern_map_in pkm; +#endif struct file *fp; struct vnode *vp; vm_offset_t addr; vm_size_t size, pageoff; vm_prot_t prot, maxprot; void *handle; objtype_t handle_type; int flags, error; off_t pos; struct vmspace *vms = td->td_proc->p_vmspace; addr = (vm_offset_t) uap->addr; size = uap->len; prot = uap->prot & VM_PROT_ALL; flags = uap->flags; pos = uap->pos; fp = NULL; /* make sure mapping fits into numeric range etc */ if ((ssize_t) uap->len < 0 || ((flags & MAP_ANON) && uap->fd != -1)) return (EINVAL); if (flags & MAP_STACK) { if ((uap->fd != -1) || ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE))) return (EINVAL); flags |= MAP_ANON; pos = 0; } /* * Align the file position to a page boundary, * and save its page offset component. */ pageoff = (pos & PAGE_MASK); pos -= pageoff; /* Adjust size for rounding (on both ends). */ size += pageoff; /* low end... */ size = (vm_size_t) round_page(size); /* hi end */ /* * Check for illegal addresses. Watch out for address wrap... Note * that VM_*_ADDRESS are not constants due to casts (argh). */ if (flags & MAP_FIXED) { /* * The specified address must have the same remainder * as the file offset taken modulo PAGE_SIZE, so it * should be aligned after adjustment by pageoff. */ addr -= pageoff; if (addr & PAGE_MASK) return (EINVAL); /* Address range must be all in user VM space. */ if (addr < vm_map_min(&vms->vm_map) || addr + size > vm_map_max(&vms->vm_map)) return (EINVAL); if (addr + size < addr) return (EINVAL); } else { /* * XXX for non-fixed mappings where no hint is provided or * the hint would fall in the potential heap space, * place it after the end of the largest possible heap. * * There should really be a pmap call to determine a reasonable * location. */ PROC_LOCK(td->td_proc); if (addr == 0 || (addr >= round_page((vm_offset_t)vms->vm_taddr) && addr < round_page((vm_offset_t)vms->vm_daddr + lim_max(td->td_proc, RLIMIT_DATA)))) addr = round_page((vm_offset_t)vms->vm_daddr + lim_max(td->td_proc, RLIMIT_DATA)); PROC_UNLOCK(td->td_proc); } if (flags & MAP_ANON) { /* * Mapping blank space is trivial. */ handle = NULL; handle_type = OBJT_DEFAULT; maxprot = VM_PROT_ALL; pos = 0; } else { /* * Mapping file, get fp for validation. Obtain vnode and make * sure it is of appropriate type. * don't let the descriptor disappear on us if we block */ if ((error = fget(td, uap->fd, &fp)) != 0) goto done; if (fp->f_type != DTYPE_VNODE) { error = EINVAL; goto done; } /* * POSIX shared-memory objects are defined to have * kernel persistence, and are not defined to support * read(2)/write(2) -- or even open(2). Thus, we can * use MAP_ASYNC to trade on-disk coherence for speed. * The shm_open(3) library routine turns on the FPOSIXSHM * flag to request this behavior. */ if (fp->f_flag & FPOSIXSHM) flags |= MAP_NOSYNC; vp = fp->f_vnode; /* * Ensure that file and memory protections are * compatible. Note that we only worry about * writability if mapping is shared; in this case, * current and max prot are dictated by the open file. * XXX use the vnode instead? Problem is: what * credentials do we use for determination? What if * proc does a setuid? */ if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC) maxprot = VM_PROT_NONE; else maxprot = VM_PROT_EXECUTE; if (fp->f_flag & FREAD) { maxprot |= VM_PROT_READ; } else if (prot & PROT_READ) { error = EACCES; goto done; } /* * If we are sharing potential changes (either via * MAP_SHARED or via the implicit sharing of character * device mappings), and we are trying to get write * permission although we opened it without asking * for it, bail out. */ if ((flags & MAP_SHARED) != 0) { if ((fp->f_flag & FWRITE) != 0) { maxprot |= VM_PROT_WRITE; } else if ((prot & PROT_WRITE) != 0) { error = EACCES; goto done; } } else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) { maxprot |= VM_PROT_WRITE; } handle = (void *)vp; handle_type = OBJT_VNODE; } /* * Do not allow more then a certain number of vm_map_entry structures * per process. Scale with the number of rforks sharing the map * to make the limit reasonable for threads. */ if (max_proc_mmap && vms->vm_map.nentries >= max_proc_mmap * vms->vm_refcnt) { error = ENOMEM; goto done; } error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot, flags, handle_type, handle, pos); +#ifdef HWPMC_HOOKS + /* inform hwpmc(4) if an executable is being mapped */ + if (error == 0 && handle_type == OBJT_VNODE && + (prot & PROT_EXEC)) { + pkm.pm_file = handle; + pkm.pm_address = (uintptr_t) addr; + PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm); + } +#endif if (error == 0) td->td_retval[0] = (register_t) (addr + pageoff); done: if (fp) fdrop(fp, td); return (error); } #ifdef COMPAT_43 #ifndef _SYS_SYSPROTO_H_ struct ommap_args { caddr_t addr; int len; int prot; int flags; int fd; long pos; }; #endif int ommap(td, uap) struct thread *td; struct ommap_args *uap; { struct mmap_args nargs; static const char cvtbsdprot[8] = { 0, PROT_EXEC, PROT_WRITE, PROT_EXEC | PROT_WRITE, PROT_READ, PROT_EXEC | PROT_READ, PROT_WRITE | PROT_READ, PROT_EXEC | PROT_WRITE | PROT_READ, }; #define OMAP_ANON 0x0002 #define OMAP_COPY 0x0020 #define OMAP_SHARED 0x0010 #define OMAP_FIXED 0x0100 nargs.addr = uap->addr; nargs.len = uap->len; nargs.prot = cvtbsdprot[uap->prot & 0x7]; nargs.flags = 0; if (uap->flags & OMAP_ANON) nargs.flags |= MAP_ANON; if (uap->flags & OMAP_COPY) nargs.flags |= MAP_COPY; if (uap->flags & OMAP_SHARED) nargs.flags |= MAP_SHARED; else nargs.flags |= MAP_PRIVATE; if (uap->flags & OMAP_FIXED) nargs.flags |= MAP_FIXED; nargs.fd = uap->fd; nargs.pos = uap->pos; return (mmap(td, &nargs)); } #endif /* COMPAT_43 */ #ifndef _SYS_SYSPROTO_H_ struct msync_args { void *addr; int len; int flags; }; #endif /* * MPSAFE */ int msync(td, uap) struct thread *td; struct msync_args *uap; { vm_offset_t addr; vm_size_t size, pageoff; int flags; vm_map_t map; int rv; addr = (vm_offset_t) uap->addr; size = uap->len; flags = uap->flags; pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); if (addr + size < addr) return (EINVAL); if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE)) return (EINVAL); map = &td->td_proc->p_vmspace->vm_map; /* * Clean the pages and interpret the return value. */ rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0, (flags & MS_INVALIDATE) != 0); switch (rv) { case KERN_SUCCESS: return (0); case KERN_INVALID_ADDRESS: return (EINVAL); /* Sun returns ENOMEM? */ case KERN_INVALID_ARGUMENT: return (EBUSY); default: return (EINVAL); } } #ifndef _SYS_SYSPROTO_H_ struct munmap_args { void *addr; size_t len; }; #endif /* * MPSAFE */ int munmap(td, uap) struct thread *td; struct munmap_args *uap; { +#ifdef HWPMC_HOOKS + struct pmckern_map_out pkm; + vm_map_entry_t entry; +#endif vm_offset_t addr; vm_size_t size, pageoff; vm_map_t map; addr = (vm_offset_t) uap->addr; size = uap->len; if (size == 0) return (EINVAL); pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); if (addr + size < addr) return (EINVAL); /* * Check for illegal addresses. Watch out for address wrap... */ map = &td->td_proc->p_vmspace->vm_map; if (addr < vm_map_min(map) || addr + size > vm_map_max(map)) return (EINVAL); vm_map_lock(map); /* * Make sure entire range is allocated. */ if (!vm_map_check_protection(map, addr, addr + size, VM_PROT_NONE)) { vm_map_unlock(map); return (EINVAL); } +#ifdef HWPMC_HOOKS + /* + * Inform hwpmc if the address range being unmapped contains + * an executable region. + */ + if (vm_map_lookup_entry(map, addr, &entry)) { + for (; + entry != &map->header && entry->start < addr + size; + entry = entry->next) { + if (vm_map_check_protection(map, entry->start, + entry->end, VM_PROT_EXECUTE) == TRUE) { + pkm.pm_address = (uintptr_t) addr; + pkm.pm_size = (size_t) size; + PMC_CALL_HOOK(td, PMC_FN_MUNMAP, + (void *) &pkm); + break; + } + } + } +#endif /* returns nothing but KERN_SUCCESS anyway */ vm_map_delete(map, addr, addr + size); vm_map_unlock(map); return (0); } #ifndef _SYS_SYSPROTO_H_ struct mprotect_args { const void *addr; size_t len; int prot; }; #endif /* * MPSAFE */ int mprotect(td, uap) struct thread *td; struct mprotect_args *uap; { vm_offset_t addr; vm_size_t size, pageoff; vm_prot_t prot; addr = (vm_offset_t) uap->addr; size = uap->len; prot = uap->prot & VM_PROT_ALL; #if defined(VM_PROT_READ_IS_EXEC) if (prot & VM_PROT_READ) prot |= VM_PROT_EXECUTE; #endif pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); if (addr + size < addr) return (EINVAL); switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr, addr + size, prot, FALSE)) { case KERN_SUCCESS: return (0); case KERN_PROTECTION_FAILURE: return (EACCES); } return (EINVAL); } #ifndef _SYS_SYSPROTO_H_ struct minherit_args { void *addr; size_t len; int inherit; }; #endif /* * MPSAFE */ int minherit(td, uap) struct thread *td; struct minherit_args *uap; { vm_offset_t addr; vm_size_t size, pageoff; vm_inherit_t inherit; addr = (vm_offset_t)uap->addr; size = uap->len; inherit = uap->inherit; pageoff = (addr & PAGE_MASK); addr -= pageoff; size += pageoff; size = (vm_size_t) round_page(size); if (addr + size < addr) return (EINVAL); switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr, addr + size, inherit)) { case KERN_SUCCESS: return (0); case KERN_PROTECTION_FAILURE: return (EACCES); } return (EINVAL); } #ifndef _SYS_SYSPROTO_H_ struct madvise_args { void *addr; size_t len; int behav; }; #endif /* * MPSAFE */ /* ARGSUSED */ int madvise(td, uap) struct thread *td; struct madvise_args *uap; { vm_offset_t start, end; vm_map_t map; struct proc *p; int error; /* * Check for our special case, advising the swap pager we are * "immortal." */ if (uap->behav == MADV_PROTECT) { error = suser(td); if (error == 0) { p = td->td_proc; PROC_LOCK(p); p->p_flag |= P_PROTECTED; PROC_UNLOCK(p); } return (error); } /* * Check for illegal behavior */ if (uap->behav < 0 || uap->behav > MADV_CORE) return (EINVAL); /* * Check for illegal addresses. Watch out for address wrap... Note * that VM_*_ADDRESS are not constants due to casts (argh). */ map = &td->td_proc->p_vmspace->vm_map; if ((vm_offset_t)uap->addr < vm_map_min(map) || (vm_offset_t)uap->addr + uap->len > vm_map_max(map)) return (EINVAL); if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr) return (EINVAL); /* * Since this routine is only advisory, we default to conservative * behavior. */ start = trunc_page((vm_offset_t) uap->addr); end = round_page((vm_offset_t) uap->addr + uap->len); if (vm_map_madvise(map, start, end, uap->behav)) return (EINVAL); return (0); } #ifndef _SYS_SYSPROTO_H_ struct mincore_args { const void *addr; size_t len; char *vec; }; #endif /* * MPSAFE */ /* ARGSUSED */ int mincore(td, uap) struct thread *td; struct mincore_args *uap; { vm_offset_t addr, first_addr; vm_offset_t end, cend; pmap_t pmap; vm_map_t map; char *vec; int error = 0; int vecindex, lastvecindex; vm_map_entry_t current; vm_map_entry_t entry; int mincoreinfo; unsigned int timestamp; /* * Make sure that the addresses presented are valid for user * mode. */ first_addr = addr = trunc_page((vm_offset_t) uap->addr); end = addr + (vm_size_t)round_page(uap->len); map = &td->td_proc->p_vmspace->vm_map; if (end > vm_map_max(map) || end < addr) return (EINVAL); /* * Address of byte vector */ vec = uap->vec; pmap = vmspace_pmap(td->td_proc->p_vmspace); vm_map_lock_read(map); RestartScan: timestamp = map->timestamp; if (!vm_map_lookup_entry(map, addr, &entry)) entry = entry->next; /* * Do this on a map entry basis so that if the pages are not * in the current processes address space, we can easily look * up the pages elsewhere. */ lastvecindex = -1; for (current = entry; (current != &map->header) && (current->start < end); current = current->next) { /* * ignore submaps (for now) or null objects */ if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) || current->object.vm_object == NULL) continue; /* * limit this scan to the current map entry and the * limits for the mincore call */ if (addr < current->start) addr = current->start; cend = current->end; if (cend > end) cend = end; /* * scan this entry one page at a time */ while (addr < cend) { /* * Check pmap first, it is likely faster, also * it can provide info as to whether we are the * one referencing or modifying the page. */ mincoreinfo = pmap_mincore(pmap, addr); if (!mincoreinfo) { vm_pindex_t pindex; vm_ooffset_t offset; vm_page_t m; /* * calculate the page index into the object */ offset = current->offset + (addr - current->start); pindex = OFF_TO_IDX(offset); VM_OBJECT_LOCK(current->object.vm_object); m = vm_page_lookup(current->object.vm_object, pindex); /* * if the page is resident, then gather information about * it. */ if (m != NULL && m->valid != 0) { mincoreinfo = MINCORE_INCORE; vm_page_lock_queues(); if (m->dirty || pmap_is_modified(m)) mincoreinfo |= MINCORE_MODIFIED_OTHER; if ((m->flags & PG_REFERENCED) || pmap_ts_referenced(m)) { vm_page_flag_set(m, PG_REFERENCED); mincoreinfo |= MINCORE_REFERENCED_OTHER; } vm_page_unlock_queues(); } VM_OBJECT_UNLOCK(current->object.vm_object); } /* * subyte may page fault. In case it needs to modify * the map, we release the lock. */ vm_map_unlock_read(map); /* * calculate index into user supplied byte vector */ vecindex = OFF_TO_IDX(addr - first_addr); /* * If we have skipped map entries, we need to make sure that * the byte vector is zeroed for those skipped entries. */ while ((lastvecindex + 1) < vecindex) { error = subyte(vec + lastvecindex, 0); if (error) { error = EFAULT; goto done2; } ++lastvecindex; } /* * Pass the page information to the user */ error = subyte(vec + vecindex, mincoreinfo); if (error) { error = EFAULT; goto done2; } /* * If the map has changed, due to the subyte, the previous * output may be invalid. */ vm_map_lock_read(map); if (timestamp != map->timestamp) goto RestartScan; lastvecindex = vecindex; addr += PAGE_SIZE; } } /* * subyte may page fault. In case it needs to modify * the map, we release the lock. */ vm_map_unlock_read(map); /* * Zero the last entries in the byte vector. */ vecindex = OFF_TO_IDX(end - first_addr); while ((lastvecindex + 1) < vecindex) { error = subyte(vec + lastvecindex, 0); if (error) { error = EFAULT; goto done2; } ++lastvecindex; } /* * If the map has changed, due to the subyte, the previous * output may be invalid. */ vm_map_lock_read(map); if (timestamp != map->timestamp) goto RestartScan; vm_map_unlock_read(map); done2: return (error); } #ifndef _SYS_SYSPROTO_H_ struct mlock_args { const void *addr; size_t len; }; #endif /* * MPSAFE */ int mlock(td, uap) struct thread *td; struct mlock_args *uap; { struct proc *proc; vm_offset_t addr, end, last, start; vm_size_t npages, size; int error; error = suser(td); if (error) return (error); addr = (vm_offset_t)uap->addr; size = uap->len; last = addr + size; start = trunc_page(addr); end = round_page(last); if (last < addr || end < addr) return (EINVAL); npages = atop(end - start); if (npages > vm_page_max_wired) return (ENOMEM); proc = td->td_proc; PROC_LOCK(proc); if (ptoa(npages + pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))) > lim_cur(proc, RLIMIT_MEMLOCK)) { PROC_UNLOCK(proc); return (ENOMEM); } PROC_UNLOCK(proc); if (npages + cnt.v_wire_count > vm_page_max_wired) return (EAGAIN); error = vm_map_wire(&proc->p_vmspace->vm_map, start, end, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); return (error == KERN_SUCCESS ? 0 : ENOMEM); } #ifndef _SYS_SYSPROTO_H_ struct mlockall_args { int how; }; #endif /* * MPSAFE */ int mlockall(td, uap) struct thread *td; struct mlockall_args *uap; { vm_map_t map; int error; map = &td->td_proc->p_vmspace->vm_map; error = 0; if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0)) return (EINVAL); #if 0 /* * If wiring all pages in the process would cause it to exceed * a hard resource limit, return ENOMEM. */ PROC_LOCK(td->td_proc); if (map->size - ptoa(pmap_wired_count(vm_map_pmap(map)) > lim_cur(td->td_proc, RLIMIT_MEMLOCK))) { PROC_UNLOCK(td->td_proc); return (ENOMEM); } PROC_UNLOCK(td->td_proc); #else error = suser(td); if (error) return (error); #endif if (uap->how & MCL_FUTURE) { vm_map_lock(map); vm_map_modflags(map, MAP_WIREFUTURE, 0); vm_map_unlock(map); error = 0; } if (uap->how & MCL_CURRENT) { /* * P1003.1-2001 mandates that all currently mapped pages * will be memory resident and locked (wired) upon return * from mlockall(). vm_map_wire() will wire pages, by * calling vm_fault_wire() for each page in the region. */ error = vm_map_wire(map, vm_map_min(map), vm_map_max(map), VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); error = (error == KERN_SUCCESS ? 0 : EAGAIN); } return (error); } #ifndef _SYS_SYSPROTO_H_ struct munlockall_args { register_t dummy; }; #endif /* * MPSAFE */ int munlockall(td, uap) struct thread *td; struct munlockall_args *uap; { vm_map_t map; int error; map = &td->td_proc->p_vmspace->vm_map; error = suser(td); if (error) return (error); /* Clear the MAP_WIREFUTURE flag from this vm_map. */ vm_map_lock(map); vm_map_modflags(map, 0, MAP_WIREFUTURE); vm_map_unlock(map); /* Forcibly unwire all pages. */ error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map), VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK); return (error); } #ifndef _SYS_SYSPROTO_H_ struct munlock_args { const void *addr; size_t len; }; #endif /* * MPSAFE */ int munlock(td, uap) struct thread *td; struct munlock_args *uap; { vm_offset_t addr, end, last, start; vm_size_t size; int error; error = suser(td); if (error) return (error); addr = (vm_offset_t)uap->addr; size = uap->len; last = addr + size; start = trunc_page(addr); end = round_page(last); if (last < addr || end < addr) return (EINVAL); error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end, VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); return (error == KERN_SUCCESS ? 0 : ENOMEM); } /* * vm_mmap_vnode() * * MPSAFE * * Helper function for vm_mmap. Perform sanity check specific for mmap * operations on vnodes. */ int vm_mmap_vnode(struct thread *td, vm_size_t objsize, vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, struct vnode *vp, vm_ooffset_t foff, vm_object_t *objp) { struct vattr va; void *handle; vm_object_t obj; struct mount *mp; int error, flags, type; int vfslocked; mp = vp->v_mount; vfslocked = VFS_LOCK_GIANT(mp); if ((error = vget(vp, LK_EXCLUSIVE, td)) != 0) { VFS_UNLOCK_GIANT(vfslocked); return (error); } flags = *flagsp; obj = vp->v_object; if (vp->v_type == VREG) { /* * Get the proper underlying object */ if (obj == NULL) { error = EINVAL; goto done; } if (obj->handle != vp) { vput(vp); vp = (struct vnode*)obj->handle; vget(vp, LK_EXCLUSIVE, td); } type = OBJT_VNODE; handle = vp; } else if (vp->v_type == VCHR) { type = OBJT_DEVICE; handle = vp->v_rdev; /* XXX: lack thredref on device */ if(vp->v_rdev->si_devsw->d_flags & D_MMAP_ANON) { *maxprotp = VM_PROT_ALL; *flagsp |= MAP_ANON; error = 0; goto done; } /* * cdevs does not provide private mappings of any kind. */ if ((*maxprotp & VM_PROT_WRITE) == 0 && (prot & PROT_WRITE) != 0) { error = EACCES; goto done; } if (flags & (MAP_PRIVATE|MAP_COPY)) { error = EINVAL; goto done; } /* * Force device mappings to be shared. */ flags |= MAP_SHARED; } else { error = EINVAL; goto done; } if ((error = VOP_GETATTR(vp, &va, td->td_ucred, td))) { goto done; } #ifdef MAC error = mac_check_vnode_mmap(td->td_ucred, vp, prot, flags); if (error != 0) goto done; #endif if ((flags & MAP_SHARED) != 0) { if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) { if (prot & PROT_WRITE) { error = EPERM; goto done; } *maxprotp &= ~VM_PROT_WRITE; } } /* * If it is a regular file without any references * we do not need to sync it. * Adjust object size to be the size of actual file. */ if (vp->v_type == VREG) { objsize = round_page(va.va_size); if (va.va_nlink == 0) flags |= MAP_NOSYNC; } obj = vm_pager_allocate(type, handle, objsize, prot, foff); if (obj == NULL) { error = (type == OBJT_DEVICE ? EINVAL : ENOMEM); goto done; } *objp = obj; *flagsp = flags; vfs_mark_atime(vp, td); done: vput(vp); VFS_UNLOCK_GIANT(vfslocked); return (error); } /* * vm_mmap_cdev() * * MPSAFE * * Helper function for vm_mmap. Perform sanity check specific for mmap * operations on cdevs. */ int vm_mmap_cdev(struct thread *td, vm_size_t objsize, vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp, struct cdev *cdev, vm_ooffset_t foff, vm_object_t *objp) { vm_object_t obj; int flags; flags = *flagsp; /* XXX: lack thredref on device */ if (cdev->si_devsw->d_flags & D_MMAP_ANON) { *maxprotp = VM_PROT_ALL; *flagsp |= MAP_ANON; return (0); } /* * cdevs does not provide private mappings of any kind. */ if ((*maxprotp & VM_PROT_WRITE) == 0 && (prot & PROT_WRITE) != 0) return (EACCES); if (flags & (MAP_PRIVATE|MAP_COPY)) return (EINVAL); /* * Force device mappings to be shared. */ flags |= MAP_SHARED; #ifdef MAC_XXX error = mac_check_cdev_mmap(td->td_ucred, cdev, prot); if (error != 0) return (error); #endif obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, foff); if (obj == NULL) return (EINVAL); *objp = obj; *flagsp = flags; return (0); } /* * vm_mmap() * * MPSAFE * * Internal version of mmap. Currently used by mmap, exec, and sys5 * shared memory. Handle is either a vnode pointer or NULL for MAP_ANON. */ int vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot, vm_prot_t maxprot, int flags, objtype_t handle_type, void *handle, vm_ooffset_t foff) { boolean_t fitit; vm_object_t object; int rv = KERN_SUCCESS; int docow, error; struct thread *td = curthread; if (size == 0) return (0); size = round_page(size); PROC_LOCK(td->td_proc); if (td->td_proc->p_vmspace->vm_map.size + size > lim_cur(td->td_proc, RLIMIT_VMEM)) { PROC_UNLOCK(td->td_proc); return(ENOMEM); } PROC_UNLOCK(td->td_proc); /* * We currently can only deal with page aligned file offsets. * The check is here rather than in the syscall because the * kernel calls this function internally for other mmaping * operations (such as in exec) and non-aligned offsets will * cause pmap inconsistencies...so we want to be sure to * disallow this in all cases. */ if (foff & PAGE_MASK) return (EINVAL); if ((flags & MAP_FIXED) == 0) { fitit = TRUE; *addr = round_page(*addr); } else { if (*addr != trunc_page(*addr)) return (EINVAL); fitit = FALSE; (void) vm_map_remove(map, *addr, *addr + size); } /* * Lookup/allocate object. */ switch (handle_type) { case OBJT_DEVICE: error = vm_mmap_cdev(td, size, prot, &maxprot, &flags, handle, foff, &object); break; case OBJT_VNODE: error = vm_mmap_vnode(td, size, prot, &maxprot, &flags, handle, foff, &object); break; case OBJT_DEFAULT: if (handle == NULL) { error = 0; break; } /* FALLTHROUGH */ default: error = EINVAL; } if (error) return (error); if (flags & MAP_ANON) { object = NULL; docow = 0; /* * Unnamed anonymous regions always start at 0. */ if (handle == 0) foff = 0; } else { docow = MAP_PREFAULT_PARTIAL; } if ((flags & (MAP_ANON|MAP_SHARED)) == 0) docow |= MAP_COPY_ON_WRITE; if (flags & MAP_NOSYNC) docow |= MAP_DISABLE_SYNCER; if (flags & MAP_NOCORE) docow |= MAP_DISABLE_COREDUMP; #if defined(VM_PROT_READ_IS_EXEC) if (prot & VM_PROT_READ) prot |= VM_PROT_EXECUTE; if (maxprot & VM_PROT_READ) maxprot |= VM_PROT_EXECUTE; #endif if (fitit) *addr = pmap_addr_hint(object, *addr, size); if (flags & MAP_STACK) rv = vm_map_stack(map, *addr, size, prot, maxprot, docow | MAP_STACK_GROWS_DOWN); else rv = vm_map_find(map, object, foff, addr, size, fitit, prot, maxprot, docow); if (rv != KERN_SUCCESS) { /* * Lose the object reference. Will destroy the * object if it's an unnamed anonymous mapping * or named anonymous without other references. */ vm_object_deallocate(object); } else if (flags & MAP_SHARED) { /* * Shared memory is also shared with children. */ rv = vm_map_inherit(map, *addr, *addr + size, VM_INHERIT_SHARE); if (rv != KERN_SUCCESS) (void) vm_map_remove(map, *addr, *addr + size); } /* * If the process has requested that all future mappings * be wired, then heed this. */ if ((rv == KERN_SUCCESS) && (map->flags & MAP_WIREFUTURE)) vm_map_wire(map, *addr, *addr + size, VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES); switch (rv) { case KERN_SUCCESS: return (0); case KERN_INVALID_ADDRESS: case KERN_NO_SPACE: return (ENOMEM); case KERN_PROTECTION_FAILURE: return (EACCES); default: return (EINVAL); } } Index: head/usr.sbin/pmcstat/pmcstat.8 =================================================================== --- head/usr.sbin/pmcstat/pmcstat.8 (revision 157143) +++ head/usr.sbin/pmcstat/pmcstat.8 (revision 157144) @@ -1,250 +1,282 @@ -.\" Copyright (c) 2003 Joseph Koshy. All rights reserved. +.\" Copyright (c) 2003-2006 Joseph Koshy. All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" This software is provided by Joseph Koshy ``as is'' and .\" any express or implied warranties, including, but not limited to, the .\" implied warranties of merchantability and fitness for a particular purpose .\" are disclaimed. in no event shall Joseph Koshy be liable .\" for any direct, indirect, incidental, special, exemplary, or consequential .\" damages (including, but not limited to, procurement of substitute goods .\" or services; loss of use, data, or profits; or business interruption) .\" however caused and on any theory of liability, whether in contract, strict .\" liability, or tort (including negligence or otherwise) arising in any way .\" out of the use of this software, even if advised of the possibility of .\" such damage. .\" .\" $FreeBSD$ .\" -.Dd July 1, 2005 +.Dd March 26, 2006 .Os .Dt PMCSTAT 8 .Sh NAME .Nm pmcstat .Nd "performance measurement with performance monitoring hardware" .Sh SYNOPSIS .Nm .Op Fl C .Op Fl D Ar pathname .Op Fl E +.Op Fl M Ar mapfilename .Op Fl O Ar logfilename .Op Fl P Ar event-spec .Op Fl R Ar logfilename .Op Fl S Ar event-spec .Op Fl W .Op Fl c Ar cpu .Op Fl d .Op Fl g -.Op Fl k Ar kernelfile +.Op Fl k Ar kerneldir .Op Fl n Ar rate .Op Fl o Ar outputfile .Op Fl p Ar event-spec +.Op Fl q +.Op Fl r Ar fsroot .Op Fl s Ar event-spec .Op Fl t Ar pid +.Op Fl v .Op Fl w Ar secs .Op Ar command Op Ar args .Sh DESCRIPTION The .Nm utility measures system performance using the facilities provided by .Xr hwpmc 4 . .Pp The .Nm utility can measure both hardware events seen by the system as a whole, and those seen when a specified process is executing on the system's CPUs. If a specific process is being targeted (for example, if the .Fl t Ar pid option is specified, or if a command line is specified using .Ar command ) , then measurement occurs till the target process exits or the .Nm utility is interrupted by the user. If a specific process is not targeted for measurement, then .Nm will perform system-wide measurements till interrupted by the user. .Pp A given invocation of .Nm can mix allocations of system-mode and process-mode PMCs, of both counting and sampling flavors. The values of all counting PMCs are printed in human readable form at regular intervals by .Nm . The output of sampling PMCs may be configured to go to a log file for subsequent offline analysis, or, at the expense of greater overhead, may be configured to be printed in text form on the fly. .Pp Hardware events to measure are specified to .Nm using event specifier strings .Ar event-spec . The syntax of these event specifiers is machine dependent and is documented in .Xr pmc 3 . .Pp A process-mode PMC may be configured to be inheritable by the target process' current and future children. .Sh OPTIONS The following options are available: .Bl -tag -width indent .It Fl C Toggle between showing cumulative or incremental counts for subsequent counting mode PMCs specified on the command line. The default is to show incremental counts. .It Fl D Ar pathname Create files with per-program samples in the directory named by .Ar pathname . The default is to create these files in the current directory. .It Fl E Toggle showing per-process counts at the time a tracked process exits for subsequent process-mode PMCs specified on the command line. This option is useful for mapping the performance characteristics of a complex pipeline of processes when used in conjunction with the .Fl d option. The default is to not to enable per-process tracking. +.It Fl M Ar mapfilename +Write the mapping between executable objects encountered in the event +log and the abbreviated pathnames used for +.Xr gprof 1 +profiles to file +.Ar mapfilename . +If this option is not specified, mapping information is not written. +Argument +.Ar mapfilename +may be a +.Dq Li - +in which case this mapping information is sent to the output +file configured by the +.Fl o +option. .It Fl O Ar logfilename Send logging output to file .Ar logfilename . If this option is not specified and one of the logging options is requested, then .Nm will print a textual form of the logged events to the configured output file. .It Fl P Ar event-spec Allocate a process mode sampling PMC measuring hardware events specified in .Ar event-spec . .It Fl R Ar logfilename Perform offline analysis using sampling data in file .Ar logfilename . .It Fl S Ar event-spec Allocate a system mode sampling PMC measuring hardware events specified in .Ar event-spec . .It Fl W Toggle logging the incremental counts seen by the threads of a tracked process each time they are scheduled on a CPU. This is an experimental feature intended to help analyse the dynamic behaviour of processes in the system. It may incur substantial overhead if enabled. The default is for this feature to be disabled. .It Fl c Ar cpu Set the cpu for subsequent system mode PMCs specified on the command line to .Ar cpu . The default is to allocate system mode PMCs on CPU zero. .It Fl d Toggle between process mode PMCs measuring events for the target process' current and future children or only measuring events for the target process. The default is to measure events for the target process alone. .It Fl g Produce flat execution profiles in a format compatible with .Xr gprof 1 . A separate profile file is generated for each executable object encountered. Profile files are placed in sub-directories named by their PMC event name. -.It Fl k Ar kernelfile -Set the pathname of the kernel to argument -.Ar kernelfile . +.It Fl k Ar kerneldir +Set the pathname of the kernel directory to argument +.Ar kerneldir . +This directory specifies where +.Nm +should look for the kernel and its modules. The default is -.Pa /boot/kernel/kernel . +.Pa /boot/kernel . .It Fl n Ar rate Set the default sampling rate for subsequent sampling mode PMCs specified on the command line. The default is to configure PMCs to sample the CPU's instruction pointer every 65536 events. .It Fl o Ar outputfile Send counter readings and textual representations of logged data to file .Ar outputfile . The default is to send output to .Pa stderr . .It Fl p Ar event-spec Allocate a process mode counting PMC measuring hardware events specified in .Ar event-spec . +.It Fl q +Decrease verbosity. +.It Fl r Ar fsroot +Set the top of the filesystem hierarchy under which executables +are located to argument +.Ar fsroot . +The default is +.Pa / . .It Fl s Ar event-spec Allocate a system mode counting PMC measuring hardware events specified in .Ar event-spec . .It Fl t Ar pid Attach all process mode PMCs allocated to the process with PID .Ar pid . The option is not allowed in conjunction with specifying a command using .Ar command . +.It Fl v +Increase verbosity. .It Fl w Ar secs Print the values of all counting mode PMCs every .Ar secs seconds. The argument .Ar secs may be a fractional value. The default interval is 5 seconds. .El .Pp If .Ar command is specified, it is executed using .Xr execvp 3 . .Sh EXAMPLES To perform system-wide statistical sampling on an AMD Athlon CPU with samples taken every 32768 instruction retirals and data being sampled to file .Pa sample.stat , use: .Dl "pmcstat -O sample.stat -n 32768 -S k7-retired-instructions" .Pp To execute .Nm mozilla and measure the number of data cache misses suffered by it and its children every 12 seconds on an AMD Athlon, use: .Dl "pmcstat -d -w 12 -p k7-dc-misses mozilla" .Pp To collect a system-wide samples driven by processor instructions executed use: .Dl "pmcstat -S instructions -O /tmp/sample.out" .Pp To generate .Xr gprof 1 compatible flat profiles from a sample file use: .Dl "pmcstat -R /tmp/sample.out -g" .Sh DIAGNOSTICS .Ex -std .Sh SEE ALSO .Xr gprof 1 , .Xr execvp 3 , .Xr pmc 3 , .Xr pmclog 3 , .Xr hwpmc 4 , .Xr pmccontrol 8 , .Xr sysctl 8 .Sh HISTORY The .Nm utility first appeared in .Fx 6.0 . It is .Ud .Sh AUTHORS .An Joseph Koshy Aq jkoshy@FreeBSD.org .Sh BUGS On AMD64 platforms .Nm does not yet handle profiles with samples from 32 bit executables. Index: head/usr.sbin/pmcstat/pmcstat.c =================================================================== --- head/usr.sbin/pmcstat/pmcstat.c (revision 157143) +++ head/usr.sbin/pmcstat/pmcstat.c (revision 157144) @@ -1,986 +1,1055 @@ /*- - * Copyright (c) 2003-2005, Joseph Koshy + * Copyright (c) 2003-2006, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include "pmcstat.h" /* * A given invocation of pmcstat(8) can manage multiple PMCs of both * the system-wide and per-process variety. Each of these could be in * 'counting mode' or in 'sampling mode'. * * For 'counting mode' PMCs, pmcstat(8) will periodically issue a * pmc_read() at the configured time interval and print out the value * of the requested PMCs. * * For 'sampling mode' PMCs it can log to a file for offline analysis, * or can analyse sampling data "on the fly", either by converting * samples to printed textual form or by creating gprof(1) compatible * profiles, one per program executed. When creating gprof(1) * profiles it can optionally merge entries from multiple processes * for a given executable into a single profile file. */ /* Globals */ int pmcstat_interrupt = 0; int pmcstat_displayheight = DEFAULT_DISPLAY_HEIGHT; int pmcstat_pipefd[NPIPEFD]; int pmcstat_kq; /* * cleanup */ void pmcstat_cleanup(struct pmcstat_args *a) { struct pmcstat_ev *ev, *tmp; /* release allocated PMCs. */ STAILQ_FOREACH_SAFE(ev, &a->pa_head, ev_next, tmp) if (ev->ev_pmcid != PMC_ID_INVALID) { if (pmc_release(ev->ev_pmcid) < 0) err(EX_OSERR, "ERROR: cannot release pmc " "0x%x \"%s\"", ev->ev_pmcid, ev->ev_name); free(ev->ev_name); free(ev->ev_spec); STAILQ_REMOVE(&a->pa_head, ev, pmcstat_ev, ev_next); free(ev); } /* de-configure the log file if present. */ if (a->pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE)) (void) pmc_configure_logfile(-1); if (a->pa_logparser) { pmclog_close(a->pa_logparser); a->pa_logparser = NULL; } if (a->pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE)) - pmcstat_shutdown_logging(); + pmcstat_shutdown_logging(a); } void pmcstat_start_pmcs(struct pmcstat_args *a) { struct pmcstat_ev *ev; STAILQ_FOREACH(ev, &args.pa_head, ev_next) { assert(ev->ev_pmcid != PMC_ID_INVALID); if (pmc_start(ev->ev_pmcid) < 0) { warn("ERROR: Cannot start pmc 0x%x \"%s\"", ev->ev_pmcid, ev->ev_name); pmcstat_cleanup(a); exit(EX_OSERR); } } } void pmcstat_print_headers(struct pmcstat_args *a) { struct pmcstat_ev *ev; int c; (void) fprintf(a->pa_printfile, PRINT_HEADER_PREFIX); STAILQ_FOREACH(ev, &a->pa_head, ev_next) { if (PMC_IS_SAMPLING_MODE(ev->ev_mode)) continue; c = PMC_IS_SYSTEM_MODE(ev->ev_mode) ? 's' : 'p'; if (ev->ev_fieldskip != 0) { (void) fprintf(a->pa_printfile, "%*s%c/%*s ", ev->ev_fieldskip, "", c, ev->ev_fieldwidth - ev->ev_fieldskip - 2, ev->ev_name); } else (void) fprintf(a->pa_printfile, "%c/%*s ", c, ev->ev_fieldwidth - 2, ev->ev_name); } (void) fflush(a->pa_printfile); } void pmcstat_print_counters(struct pmcstat_args *a) { int extra_width; struct pmcstat_ev *ev; pmc_value_t value; extra_width = sizeof(PRINT_HEADER_PREFIX) - 1; STAILQ_FOREACH(ev, &a->pa_head, ev_next) { /* skip sampling mode counters */ if (PMC_IS_SAMPLING_MODE(ev->ev_mode)) continue; if (pmc_read(ev->ev_pmcid, &value) < 0) err(EX_OSERR, "ERROR: Cannot read pmc " "\"%s\"", ev->ev_name); (void) fprintf(a->pa_printfile, "%*ju ", ev->ev_fieldwidth + extra_width, (uintmax_t) ev->ev_cumulative ? value : (value - ev->ev_saved)); if (ev->ev_cumulative == 0) ev->ev_saved = value; extra_width = 0; } (void) fflush(a->pa_printfile); } /* * Print output */ void pmcstat_print_pmcs(struct pmcstat_args *a) { static int linecount = 0; /* check if we need to print a header line */ if (++linecount > pmcstat_displayheight) { (void) fprintf(a->pa_printfile, "\n"); linecount = 1; } if (linecount == 1) pmcstat_print_headers(a); (void) fprintf(a->pa_printfile, "\n"); pmcstat_print_counters(a); return; } /* * Do process profiling * * If a pid was specified, attach each allocated PMC to the target * process. Otherwise, fork a child and attach the PMCs to the child, * and have the child exec() the target program. */ void pmcstat_setup_process(struct pmcstat_args *a) { char token; struct pmcstat_ev *ev; struct kevent kev; if (a->pa_flags & FLAG_HAS_PID) { STAILQ_FOREACH(ev, &a->pa_head, ev_next) if (pmc_attach(ev->ev_pmcid, a->pa_pid) != 0) err(EX_OSERR, "ERROR: cannot attach pmc \"%s\" to " "process %d", ev->ev_name, (int) a->pa_pid); } else { /* * We need to fork a new process and startup the child * using execvp(). Before doing the exec() the child * process reads its pipe for a token so that the parent * can finish doing its pmc_attach() calls. */ if (pipe(pmcstat_pipefd) < 0) err(EX_OSERR, "ERROR: cannot create pipe"); switch (a->pa_pid = fork()) { case -1: err(EX_OSERR, "ERROR: cannot fork"); /*NOTREACHED*/ case 0: /* child */ /* wait for our parent to signal us */ (void) close(pmcstat_pipefd[WRITEPIPEFD]); if (read(pmcstat_pipefd[READPIPEFD], &token, 1) < 0) err(EX_OSERR, "ERROR (child): cannot read " "token"); (void) close(pmcstat_pipefd[READPIPEFD]); /* exec() the program requested */ execvp(*a->pa_argv, a->pa_argv); /* and if that fails, notify the parent */ kill(getppid(), SIGCHLD); err(EX_OSERR, "ERROR: execvp \"%s\" failed", *a->pa_argv); /*NOTREACHED*/ default: /* parent */ (void) close(pmcstat_pipefd[READPIPEFD]); /* attach all our PMCs to the child */ STAILQ_FOREACH(ev, &args.pa_head, ev_next) if (PMC_IS_VIRTUAL_MODE(ev->ev_mode) && pmc_attach(ev->ev_pmcid, a->pa_pid) != 0) err(EX_OSERR, "ERROR: cannot attach pmc " "\"%s\" to process %d", ev->ev_name, (int) a->pa_pid); } } /* Ask to be notified via a kevent when the target process exits */ EV_SET(&kev, a->pa_pid, EVFILT_PROC, EV_ADD|EV_ONESHOT, NOTE_EXIT, 0, NULL); if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) err(EX_OSERR, "ERROR: cannot monitor child process %d", a->pa_pid); return; } void pmcstat_start_process(struct pmcstat_args *a) { /* nothing to do: target is already running */ if (a->pa_flags & FLAG_HAS_PID) return; /* write token to child to state that we are ready */ if (write(pmcstat_pipefd[WRITEPIPEFD], "+", 1) != 1) err(EX_OSERR, "ERROR: write failed"); (void) close(pmcstat_pipefd[WRITEPIPEFD]); } void pmcstat_show_usage(void) { errx(EX_USAGE, "[options] [commandline]\n" "\t Measure process and/or system performance using hardware\n" "\t performance monitoring counters.\n" "\t Options include:\n" "\t -C\t\t (toggle) show cumulative counts\n" "\t -D path\t create profiles in directory \"path\"\n" "\t -E\t\t (toggle) show counts at process exit\n" + "\t -M file\t print executable/gmon file map to \"file\"\n" "\t -O file\t send log output to \"file\"\n" "\t -P spec\t allocate a process-private sampling PMC\n" "\t -R file\t read events from \"file\"\n" "\t -S spec\t allocate a system-wide sampling PMC\n" "\t -W\t\t (toggle) show counts per context switch\n" "\t -c cpu\t\t set cpu for subsequent system-wide PMCs\n" "\t -d\t\t (toggle) track descendants\n" "\t -g\t\t produce gprof(1) compatible profiles\n" - "\t -k file\t set the path to the kernel\n" + "\t -k dir\t set the path to the kernel\n" "\t -n rate\t set sampling rate\n" "\t -o file\t send print output to \"file\"\n" "\t -p spec\t allocate a process-private counting PMC\n" + "\t -q\t\t suppress verbosity\n" + "\t -r fsroot\t specify FS root directory\n" "\t -s spec\t allocate a system-wide counting PMC\n" "\t -t pid\t\t attach to running process with pid \"pid\"\n" + "\t -v\t\t increase verbosity\n" "\t -w secs\t set printing time interval" ); } /* * Main */ int main(int argc, char **argv) { double interval; int option, npmc, ncpu; int c, check_driver_stats, current_cpu, current_sampling_count; int do_print, do_descendants; int do_logproccsw, do_logprocexit; int pipefd[2]; int use_cumulative_counts; pid_t pid; - char *end; + char *end, *tmp; const char *errmsg; enum pmcstat_state runstate; struct pmc_driverstats ds_start, ds_end; struct pmcstat_ev *ev; struct sigaction sa; struct kevent kev; struct winsize ws; struct stat sb; + char buffer[PATH_MAX]; check_driver_stats = 0; current_cpu = 0; current_sampling_count = DEFAULT_SAMPLE_COUNT; do_descendants = 0; do_logproccsw = 0; do_logprocexit = 0; use_cumulative_counts = 0; args.pa_required = 0; args.pa_flags = 0; + args.pa_verbosity = 1; args.pa_pid = (pid_t) -1; args.pa_logfd = -1; + args.pa_fsroot = ""; + args.pa_kernel = strdup("/boot/kernel"); args.pa_samplesdir = "."; - args.pa_kernel = "/boot/kernel/kernel"; args.pa_printfile = stderr; args.pa_interval = DEFAULT_WAIT_INTERVAL; + args.pa_mapfilename = NULL; STAILQ_INIT(&args.pa_head); bzero(&ds_start, sizeof(ds_start)); bzero(&ds_end, sizeof(ds_end)); ev = NULL; - while ((option = getopt(argc, argv, "CD:EO:P:R:S:Wc:dgk:n:o:p:s:t:w:")) - != -1) + while ((option = getopt(argc, argv, + "CD:EM:O:P:R:S:Wc:dgk:n:o:p:qr:s:t:vw:")) != -1) switch (option) { case 'C': /* cumulative values */ use_cumulative_counts = !use_cumulative_counts; args.pa_required |= FLAG_HAS_COUNTING_PMCS; break; case 'c': /* CPU */ current_cpu = strtol(optarg, &end, 0); if (*end != '\0' || current_cpu < 0) errx(EX_USAGE, "ERROR: Illegal CPU number \"%s\".", optarg); args.pa_required |= FLAG_HAS_SYSTEM_PMCS; break; case 'D': if (stat(optarg, &sb) < 0) err(EX_OSERR, "ERROR: Cannot stat \"%s\"", optarg); if (!S_ISDIR(sb.st_mode)) errx(EX_USAGE, "ERROR: \"%s\" is not a " "directory", optarg); args.pa_samplesdir = optarg; args.pa_flags |= FLAG_HAS_SAMPLESDIR; args.pa_required |= FLAG_DO_GPROF; break; case 'd': /* toggle descendents */ do_descendants = !do_descendants; args.pa_required |= FLAG_HAS_PROCESS_PMCS; break; case 'g': /* produce gprof compatible profiles */ args.pa_flags |= FLAG_DO_GPROF; break; case 'k': /* pathname to the kernel */ - args.pa_kernel = optarg; + free(args.pa_kernel); + args.pa_kernel = strdup(optarg); args.pa_required |= FLAG_DO_GPROF; args.pa_flags |= FLAG_HAS_KERNELPATH; break; case 'E': /* log process exit */ do_logprocexit = !do_logprocexit; args.pa_required |= (FLAG_HAS_PROCESS_PMCS | FLAG_HAS_COUNTING_PMCS | FLAG_HAS_OUTPUT_LOGFILE); break; + case 'M': /* mapfile */ + args.pa_mapfilename = optarg; + break; + case 'p': /* process virtual counting PMC */ case 's': /* system-wide counting PMC */ case 'P': /* process virtual sampling PMC */ case 'S': /* system-wide sampling PMC */ if ((ev = malloc(sizeof(*ev))) == NULL) errx(EX_SOFTWARE, "ERROR: Out of memory."); switch (option) { case 'p': ev->ev_mode = PMC_MODE_TC; break; case 's': ev->ev_mode = PMC_MODE_SC; break; case 'P': ev->ev_mode = PMC_MODE_TS; break; case 'S': ev->ev_mode = PMC_MODE_SS; break; } if (option == 'P' || option == 'p') { args.pa_flags |= FLAG_HAS_PROCESS_PMCS; args.pa_required |= (FLAG_HAS_COMMANDLINE | FLAG_HAS_PID); } if (option == 'P' || option == 'S') { args.pa_flags |= FLAG_HAS_SAMPLING_PMCS; args.pa_required |= (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE); } if (option == 'p' || option == 's') args.pa_flags |= FLAG_HAS_COUNTING_PMCS; if (option == 's' || option == 'S') args.pa_flags |= FLAG_HAS_SYSTEM_PMCS; ev->ev_spec = strdup(optarg); if (option == 'S' || option == 'P') ev->ev_count = current_sampling_count; else ev->ev_count = -1; if (option == 'S' || option == 's') ev->ev_cpu = current_cpu; else ev->ev_cpu = PMC_CPU_ANY; ev->ev_flags = 0; if (do_descendants) ev->ev_flags |= PMC_F_DESCENDANTS; if (do_logprocexit) ev->ev_flags |= PMC_F_LOG_PROCEXIT; if (do_logproccsw) ev->ev_flags |= PMC_F_LOG_PROCCSW; ev->ev_cumulative = use_cumulative_counts; ev->ev_saved = 0LL; ev->ev_pmcid = PMC_ID_INVALID; /* extract event name */ c = strcspn(optarg, ", \t"); ev->ev_name = malloc(c + 1); (void) strncpy(ev->ev_name, optarg, c); *(ev->ev_name + c) = '\0'; STAILQ_INSERT_TAIL(&args.pa_head, ev, ev_next); break; case 'n': /* sampling count */ current_sampling_count = strtol(optarg, &end, 0); if (*end != '\0' || current_sampling_count <= 0) errx(EX_USAGE, "ERROR: Illegal count value \"%s\".", optarg); args.pa_required |= FLAG_HAS_SAMPLING_PMCS; break; case 'o': /* outputfile */ if (args.pa_printfile != NULL) (void) fclose(args.pa_printfile); if ((args.pa_printfile = fopen(optarg, "w")) == NULL) errx(EX_OSERR, "ERROR: cannot open \"%s\" for " "writing.", optarg); args.pa_flags |= FLAG_DO_PRINT; break; case 'O': /* sampling output */ if (args.pa_outputpath) errx(EX_USAGE, "ERROR: option -O may only be " "specified once."); args.pa_outputpath = optarg; args.pa_flags |= FLAG_HAS_OUTPUT_LOGFILE; break; + case 'q': /* quiet mode */ + args.pa_verbosity = 0; + break; + + case 'r': /* root FS path */ + args.pa_fsroot = optarg; + break; + case 'R': /* read an existing log file */ if (args.pa_logparser != NULL) errx(EX_USAGE, "ERROR: option -R may only be " "specified once."); args.pa_inputpath = optarg; if (args.pa_printfile == stderr) args.pa_printfile = stdout; args.pa_flags |= FLAG_READ_LOGFILE; break; case 't': /* target pid */ pid = strtol(optarg, &end, 0); if (*end != '\0' || pid <= 0) errx(EX_USAGE, "ERROR: Illegal pid value " "\"%s\".", optarg); args.pa_flags |= FLAG_HAS_PID; args.pa_required |= FLAG_HAS_PROCESS_PMCS; args.pa_pid = pid; break; + case 'v': /* verbose */ + args.pa_verbosity++; + break; + case 'w': /* wait interval */ interval = strtod(optarg, &end); if (*end != '\0' || interval <= 0) errx(EX_USAGE, "ERROR: Illegal wait interval " "value \"%s\".", optarg); args.pa_flags |= FLAG_HAS_WAIT_INTERVAL; args.pa_required |= FLAG_HAS_COUNTING_PMCS; args.pa_interval = interval; break; case 'W': /* toggle LOG_CSW */ do_logproccsw = !do_logproccsw; args.pa_required |= (FLAG_HAS_PROCESS_PMCS | FLAG_HAS_COUNTING_PMCS | FLAG_HAS_OUTPUT_LOGFILE); break; case '?': default: pmcstat_show_usage(); break; } args.pa_argc = (argc -= optind); args.pa_argv = (argv += optind); if (argc) /* command line present */ args.pa_flags |= FLAG_HAS_COMMANDLINE; /* * Check invocation syntax. */ /* disallow -O and -R together */ if (args.pa_outputpath && args.pa_inputpath) errx(EX_USAGE, "ERROR: options -O and -R are mutually " "exclusive."); if (args.pa_flags & FLAG_READ_LOGFILE) { errmsg = NULL; if (args.pa_flags & FLAG_HAS_COMMANDLINE) errmsg = "a command line specification"; else if (args.pa_flags & FLAG_HAS_PID) errmsg = "option -t"; else if (!STAILQ_EMPTY(&args.pa_head)) errmsg = "a PMC event specification"; if (errmsg) errx(EX_USAGE, "ERROR: option -R may not be used with " "%s.", errmsg); } else if (STAILQ_EMPTY(&args.pa_head)) /* All other uses require a PMC spec. */ pmcstat_show_usage(); /* check for -t pid without a process PMC spec */ if ((args.pa_required & FLAG_HAS_PID) && (args.pa_flags & FLAG_HAS_PROCESS_PMCS) == 0) errx(EX_USAGE, "ERROR: option -t requires a process mode PMC " "to be specified."); /* check for process-mode options without a command or -t pid */ if ((args.pa_required & FLAG_HAS_PROCESS_PMCS) && (args.pa_flags & (FLAG_HAS_COMMANDLINE | FLAG_HAS_PID)) == 0) errx(EX_USAGE, "ERROR: options -d, -E, -p, -P, and -W require " "a command line or target process."); /* check for -p | -P without a target process of some sort */ if ((args.pa_required & (FLAG_HAS_COMMANDLINE | FLAG_HAS_PID)) && (args.pa_flags & (FLAG_HAS_COMMANDLINE | FLAG_HAS_PID)) == 0) errx(EX_USAGE, "ERROR: options -P and -p require a " "target process or a command line."); /* check for process-mode options without a process-mode PMC */ if ((args.pa_required & FLAG_HAS_PROCESS_PMCS) && (args.pa_flags & FLAG_HAS_PROCESS_PMCS) == 0) errx(EX_USAGE, "ERROR: options -d, -E, and -W require a " "process mode PMC to be specified."); /* check for -c cpu and not system mode PMCs */ if ((args.pa_required & FLAG_HAS_SYSTEM_PMCS) && (args.pa_flags & FLAG_HAS_SYSTEM_PMCS) == 0) errx(EX_USAGE, "ERROR: option -c requires at least one " "system mode PMC to be specified."); /* check for counting mode options without a counting PMC */ if ((args.pa_required & FLAG_HAS_COUNTING_PMCS) && (args.pa_flags & FLAG_HAS_COUNTING_PMCS) == 0) errx(EX_USAGE, "ERROR: options -C, -W, -o and -w require at " "least one counting mode PMC to be specified."); /* check for sampling mode options without a sampling PMC spec */ if ((args.pa_required & FLAG_HAS_SAMPLING_PMCS) && (args.pa_flags & FLAG_HAS_SAMPLING_PMCS) == 0) errx(EX_USAGE, "ERROR: options -n and -O require at least " "one sampling mode PMC to be specified."); if ((args.pa_flags & (FLAG_HAS_PID | FLAG_HAS_COMMANDLINE)) == (FLAG_HAS_PID | FLAG_HAS_COMMANDLINE)) errx(EX_USAGE, "ERROR: option -t cannot be specified with a command " "line."); /* check if -g is being used correctly */ if ((args.pa_flags & FLAG_DO_GPROF) && !(args.pa_flags & (FLAG_HAS_SAMPLING_PMCS|FLAG_READ_LOGFILE))) errx(EX_USAGE, "ERROR: option -g requires sampling PMCs or -R " "to be specified."); /* check if -O was spuriously specified */ if ((args.pa_flags & FLAG_HAS_OUTPUT_LOGFILE) && (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) errx(EX_USAGE, "ERROR: option -O is used only with options " "-E, -P, -S and -W."); - /* -D dir and -k kernel path require -g */ + /* -D dir and -k kernel path require -g or -R */ if ((args.pa_flags & FLAG_HAS_KERNELPATH) && - ((args.pa_flags & FLAG_DO_GPROF) == 0)) - errx(EX_USAGE, "ERROR: option -k is only used with -g."); + (args.pa_flags & FLAG_DO_GPROF) == 0 && + (args.pa_flags & FLAG_READ_LOGFILE) == 0) + errx(EX_USAGE, "ERROR: option -k is only used with -g/-R."); if ((args.pa_flags & FLAG_HAS_SAMPLESDIR) && - ((args.pa_flags & FLAG_DO_GPROF) == 0)) - errx(EX_USAGE, "ERROR: option -D is only used with -g."); + (args.pa_flags & FLAG_DO_GPROF) == 0 && + (args.pa_flags & FLAG_READ_LOGFILE) == 0) + errx(EX_USAGE, "ERROR: option -D is only used with -g/-R."); + /* -M mapfile requires -g or -R */ + if (args.pa_mapfilename != NULL && + (args.pa_flags & FLAG_DO_GPROF) == 0 && + (args.pa_flags & FLAG_READ_LOGFILE) == 0) + errx(EX_USAGE, "ERROR: option -M is only used with -g/-R."); + /* * Disallow textual output of sampling PMCs if counting PMCs * have also been asked for, mostly because the combined output * is difficult to make sense of. */ if ((args.pa_flags & FLAG_HAS_COUNTING_PMCS) && (args.pa_flags & FLAG_HAS_SAMPLING_PMCS) && ((args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0)) errx(EX_USAGE, "ERROR: option -O is required if counting and " "sampling PMCs are specified together."); + /* + * Check if "-k kerneldir" was specified, and if whether 'kerneldir' + * actually refers to a a file. If so, use `dirname path` to determine + * the kernel directory. + */ + if (args.pa_flags & FLAG_HAS_KERNELPATH) { + (void) snprintf(buffer, sizeof(buffer), "%s%s", args.pa_fsroot, + args.pa_kernel); + if (stat(buffer, &sb) < 0) + err(EX_OSERR, "ERROR: Cannot locate kernel \"%s\"", + buffer); + if (!S_ISREG(sb.st_mode) && !S_ISDIR(sb.st_mode)) + errx(EX_USAGE, "ERROR: \"%s\": Unsupported file type.", + buffer); + if (!S_ISDIR(sb.st_mode)) { + tmp = args.pa_kernel; + args.pa_kernel = strdup(dirname(args.pa_kernel)); + free(tmp); + (void) snprintf(buffer, sizeof(buffer), "%s%s", + args.pa_fsroot, args.pa_kernel); + if (stat(buffer, &sb) < 0) + err(EX_OSERR, "ERROR: Cannot stat \"%s\"", + buffer); + if (!S_ISDIR(sb.st_mode)) + errx(EX_USAGE, "ERROR: \"%s\" is not a " + "directory.", buffer); + } + } + /* if we've been asked to process a log file, do that and exit */ if (args.pa_flags & FLAG_READ_LOGFILE) { /* * Print the log in textual form if we haven't been * asked to generate gmon.out files. */ if ((args.pa_flags & FLAG_DO_GPROF) == 0) args.pa_flags |= FLAG_DO_PRINT; pmcstat_initialize_logging(&args); - if ((args.pa_logfd = pmcstat_open(args.pa_inputpath, + if ((args.pa_logfd = pmcstat_open_log(args.pa_inputpath, PMCSTAT_OPEN_FOR_READ)) < 0) err(EX_OSERR, "ERROR: Cannot open \"%s\" for " "reading", args.pa_inputpath); if ((args.pa_logparser = pmclog_open(args.pa_logfd)) == NULL) err(EX_OSERR, "ERROR: Cannot create parser"); pmcstat_process_log(&args); + pmcstat_shutdown_logging(&args); exit(EX_OK); } /* otherwise, we've been asked to collect data */ if (pmc_init() < 0) err(EX_UNAVAILABLE, "ERROR: Initialization of the pmc(3) library failed"); if ((ncpu = pmc_ncpu()) < 0) err(EX_OSERR, "ERROR: Cannot determine the number CPUs " "on the system"); if ((npmc = pmc_npmc(0)) < 0) /* assume all CPUs are identical */ err(EX_OSERR, "ERROR: Cannot determine the number of PMCs " "on CPU %d", 0); /* Allocate a kqueue */ if ((pmcstat_kq = kqueue()) < 0) err(EX_OSERR, "ERROR: Cannot allocate kqueue"); /* * Configure the specified log file or setup a default log * consumer via a pipe. */ if (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) { if (args.pa_outputpath) { - if ((args.pa_logfd = pmcstat_open(args.pa_outputpath, + if ((args.pa_logfd = + pmcstat_open_log(args.pa_outputpath, PMCSTAT_OPEN_FOR_WRITE)) < 0) err(EX_OSERR, "ERROR: Cannot open \"%s\" for " "writing", args.pa_outputpath); } else { /* * process the log on the fly by reading it in * through a pipe. */ if (pipe(pipefd) < 0) err(EX_OSERR, "ERROR: pipe(2) failed"); if (fcntl(pipefd[READPIPEFD], F_SETFL, O_NONBLOCK) < 0) err(EX_OSERR, "ERROR: fcntl(2) failed"); EV_SET(&kev, pipefd[READPIPEFD], EVFILT_READ, EV_ADD, 0, 0, NULL); if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) err(EX_OSERR, "ERROR: Cannot register kevent"); args.pa_logfd = pipefd[WRITEPIPEFD]; args.pa_flags |= (FLAG_HAS_PIPE | FLAG_DO_PRINT); args.pa_logparser = pmclog_open(pipefd[READPIPEFD]); } if (pmc_configure_logfile(args.pa_logfd) < 0) err(EX_OSERR, "ERROR: Cannot configure log file"); } /* remember to check for driver errors if we are sampling or logging */ check_driver_stats = (args.pa_flags & FLAG_HAS_SAMPLING_PMCS) || (args.pa_flags & FLAG_HAS_OUTPUT_LOGFILE); /* * Allocate PMCs. */ STAILQ_FOREACH(ev, &args.pa_head, ev_next) { if (pmc_allocate(ev->ev_spec, ev->ev_mode, ev->ev_flags, ev->ev_cpu, &ev->ev_pmcid) < 0) err(EX_OSERR, "ERROR: Cannot allocate %s-mode pmc with " "specification \"%s\"", PMC_IS_SYSTEM_MODE(ev->ev_mode) ? "system" : "process", ev->ev_spec); if (PMC_IS_SAMPLING_MODE(ev->ev_mode) && pmc_set(ev->ev_pmcid, ev->ev_count) < 0) err(EX_OSERR, "ERROR: Cannot set sampling count " "for PMC \"%s\"", ev->ev_name); } /* compute printout widths */ STAILQ_FOREACH(ev, &args.pa_head, ev_next) { int counter_width; int display_width; int header_width; (void) pmc_width(ev->ev_pmcid, &counter_width); header_width = strlen(ev->ev_name) + 2; /* prefix '%c|' */ display_width = (int) floor(counter_width / 3.32193) + 1; if (header_width > display_width) { ev->ev_fieldskip = 0; ev->ev_fieldwidth = header_width; } else { ev->ev_fieldskip = display_width - header_width; ev->ev_fieldwidth = display_width; } } /* * If our output is being set to a terminal, register a handler * for window size changes. */ if (isatty(fileno(args.pa_printfile))) { if (ioctl(fileno(args.pa_printfile), TIOCGWINSZ, &ws) < 0) err(EX_OSERR, "ERROR: Cannot determine window size"); pmcstat_displayheight = ws.ws_row - 1; EV_SET(&kev, SIGWINCH, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL); if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) err(EX_OSERR, "ERROR: Cannot register kevent for " "SIGWINCH"); } EV_SET(&kev, SIGINT, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL); if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) err(EX_OSERR, "ERROR: Cannot register kevent for SIGINT"); EV_SET(&kev, SIGIO, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL); if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) err(EX_OSERR, "ERROR: Cannot register kevent for SIGIO"); /* * An exec() failure of a forked child is signalled by the * child sending the parent a SIGCHLD. We don't register an * actual signal handler for SIGCHLD, but instead use our * kqueue to pick up the signal. */ EV_SET(&kev, SIGCHLD, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL); if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) err(EX_OSERR, "ERROR: Cannot register kevent for SIGCHLD"); /* setup a timer if we have counting mode PMCs needing to be printed */ if ((args.pa_flags & FLAG_HAS_COUNTING_PMCS) && (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) { EV_SET(&kev, 0, EVFILT_TIMER, EV_ADD, 0, args.pa_interval * 1000, NULL); if (kevent(pmcstat_kq, &kev, 1, NULL, 0, NULL) < 0) err(EX_OSERR, "ERROR: Cannot register kevent for " "timer"); } /* attach PMCs to the target process, starting it if specified */ if (args.pa_flags & (FLAG_HAS_PID | FLAG_HAS_COMMANDLINE)) pmcstat_setup_process(&args); if (check_driver_stats && pmc_get_driver_stats(&ds_start) < 0) err(EX_OSERR, "ERROR: Cannot retrieve driver statistics"); /* start the pmcs */ pmcstat_start_pmcs(&args); /* start the (commandline) process if needed */ if (args.pa_flags & FLAG_HAS_COMMANDLINE) pmcstat_start_process(&args); /* initialize logging if printing the configured log */ if ((args.pa_flags & FLAG_DO_PRINT) && (args.pa_flags & (FLAG_HAS_PIPE | FLAG_HAS_OUTPUT_LOGFILE))) pmcstat_initialize_logging(&args); /* Handle SIGINT using the kqueue loop */ sa.sa_handler = SIG_IGN; sa.sa_flags = 0; (void) sigemptyset(&sa.sa_mask); if (sigaction(SIGINT, &sa, NULL) < 0) err(EX_OSERR, "ERROR: Cannot install signal handler"); /* * loop till either the target process (if any) exits, or we * are killed by a SIGINT. */ runstate = PMCSTAT_RUNNING; do_print = 0; do { if ((c = kevent(pmcstat_kq, NULL, 0, &kev, 1, NULL)) <= 0) { if (errno != EINTR) err(EX_OSERR, "ERROR: kevent failed"); else continue; } if (kev.flags & EV_ERROR) errc(EX_OSERR, kev.data, "ERROR: kevent failed"); switch (kev.filter) { case EVFILT_PROC: /* target has exited */ if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE)) runstate = pmcstat_close_log(&args); else runstate = PMCSTAT_FINISHED; do_print = 1; break; case EVFILT_READ: /* log file data is present */ runstate = pmcstat_process_log(&args); break; case EVFILT_SIGNAL: if (kev.ident == SIGCHLD) { /* * The child process sends us a * SIGCHLD if its exec() failed. We * wait for it to exit and then exit * ourselves. */ (void) wait(&c); runstate = PMCSTAT_FINISHED; } else if (kev.ident == SIGIO) { /* * We get a SIGIO if a PMC loses all * of its targets, or if logfile * writes encounter an error. */ if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE)) { runstate = pmcstat_close_log(&args); if (args.pa_flags & (FLAG_DO_PRINT|FLAG_DO_GPROF)) pmcstat_process_log(&args); } do_print = 1; /* print PMCs at exit */ runstate = PMCSTAT_FINISHED; } else if (kev.ident == SIGINT) { /* Kill the child process if we started it */ if (args.pa_flags & FLAG_HAS_COMMANDLINE) if (kill(args.pa_pid, SIGINT) != 0) err(EX_OSERR, "ERROR: cannot " "signal child process"); runstate = PMCSTAT_FINISHED; } else if (kev.ident == SIGWINCH) { if (ioctl(fileno(args.pa_printfile), TIOCGWINSZ, &ws) < 0) err(EX_OSERR, "ERROR: Cannot determine " "window size"); pmcstat_displayheight = ws.ws_row - 1; } else assert(0); break; case EVFILT_TIMER: /* print out counting PMCs */ do_print = 1; break; } if (do_print && (args.pa_required & FLAG_HAS_OUTPUT_LOGFILE) == 0) { pmcstat_print_pmcs(&args); if (runstate == PMCSTAT_FINISHED && /* final newline */ (args.pa_flags & FLAG_DO_PRINT) == 0) (void) fprintf(args.pa_printfile, "\n"); do_print = 0; } } while (runstate != PMCSTAT_FINISHED); /* flush any pending log entries */ if (args.pa_flags & (FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE)) pmc_flush_logfile(); pmcstat_cleanup(&args); + free(args.pa_kernel); + /* check if the driver lost any samples or events */ if (check_driver_stats) { if (pmc_get_driver_stats(&ds_end) < 0) err(EX_OSERR, "ERROR: Cannot retrieve driver " "statistics"); - if (ds_start.pm_intr_bufferfull != ds_end.pm_intr_bufferfull) + if (ds_start.pm_intr_bufferfull != ds_end.pm_intr_bufferfull && + args.pa_verbosity > 0) warnx("WARNING: some samples were dropped. Please " "consider tuning the \"kern.hwpmc.nsamples\" " "tunable."); if (ds_start.pm_buffer_requests_failed != - ds_end.pm_buffer_requests_failed) + ds_end.pm_buffer_requests_failed && + args.pa_verbosity > 0) warnx("WARNING: some events were discarded. Please " "consider tuning the \"kern.hwpmc.nbuffers\" " "tunable."); } exit(EX_OK); } Index: head/usr.sbin/pmcstat/pmcstat.h =================================================================== --- head/usr.sbin/pmcstat/pmcstat.h (revision 157143) +++ head/usr.sbin/pmcstat/pmcstat.h (revision 157144) @@ -1,127 +1,128 @@ /*- - * Copyright (c) 2005, Joseph Koshy + * Copyright (c) 2005-2006, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _PMCSTAT_H_ #define _PMCSTAT_H_ #define FLAG_HAS_PID 0x00000001 /* explicit pid */ #define FLAG_HAS_WAIT_INTERVAL 0x00000002 /* -w secs */ #define FLAG_HAS_OUTPUT_LOGFILE 0x00000004 /* -O file or pipe */ #define FLAG_HAS_COMMANDLINE 0x00000008 /* command */ #define FLAG_HAS_SAMPLING_PMCS 0x00000010 /* -S or -P */ #define FLAG_HAS_COUNTING_PMCS 0x00000020 /* -s or -p */ #define FLAG_HAS_PROCESS_PMCS 0x00000040 /* -P or -p */ #define FLAG_HAS_SYSTEM_PMCS 0x00000080 /* -S or -s */ #define FLAG_HAS_PIPE 0x00000100 /* implicit log */ #define FLAG_READ_LOGFILE 0x00000200 /* -R file */ #define FLAG_DO_GPROF 0x00000400 /* -g */ #define FLAG_HAS_SAMPLESDIR 0x00000800 /* -D dir */ #define FLAG_HAS_KERNELPATH 0x00001000 /* -k kernel */ #define FLAG_DO_PRINT 0x00002000 /* -o */ #define DEFAULT_SAMPLE_COUNT 65536 #define DEFAULT_WAIT_INTERVAL 5.0 #define DEFAULT_DISPLAY_HEIGHT 23 #define DEFAULT_BUFFER_SIZE 4096 #define PRINT_HEADER_PREFIX "# " #define READPIPEFD 0 #define WRITEPIPEFD 1 #define NPIPEFD 2 #define PMCSTAT_OPEN_FOR_READ 0 #define PMCSTAT_OPEN_FOR_WRITE 1 #define PMCSTAT_DEFAULT_NW_HOST "localhost" #define PMCSTAT_DEFAULT_NW_PORT "9000" #define PMCSTAT_NHASH 256 #define PMCSTAT_HASH_MASK 0xFF #define PMCSTAT_LDD_COMMAND "/usr/bin/ldd" #define PMCSTAT_PRINT_ENTRY(A,T,...) do { \ fprintf((A)->pa_printfile, "%-8s", T); \ fprintf((A)->pa_printfile, " " __VA_ARGS__); \ fprintf((A)->pa_printfile, "\n"); \ } while (0) enum pmcstat_state { PMCSTAT_FINISHED = 0, PMCSTAT_EXITING = 1, PMCSTAT_RUNNING = 2 }; struct pmcstat_ev { STAILQ_ENTRY(pmcstat_ev) ev_next; char *ev_spec; /* event specification */ char *ev_name; /* (derived) event name */ enum pmc_mode ev_mode; /* desired mode */ int ev_count; /* associated count if in sampling mode */ int ev_cpu; /* specific cpu if requested */ int ev_flags; /* PMC_F_* */ int ev_cumulative; /* show cumulative counts */ int ev_fieldwidth; /* print width */ int ev_fieldskip; /* #leading spaces */ pmc_value_t ev_saved; /* saved value for incremental counts */ pmc_id_t ev_pmcid; /* allocated ID */ }; struct pmcstat_args { int pa_flags; /* argument flags */ int pa_required; /* required features */ + int pa_verbosity; /* verbosity level */ pid_t pa_pid; /* attached to pid */ FILE *pa_printfile; /* where to send printed output */ int pa_logfd; /* output log file */ char *pa_inputpath; /* path to input log */ char *pa_outputpath; /* path to output log */ void *pa_logparser; /* log file parser */ - const char *pa_kernel; /* pathname of the kernel */ + const char *pa_fsroot; /* FS root where executables reside */ + char *pa_kernel; /* pathname of the kernel */ const char *pa_samplesdir; /* directory for profile files */ + const char *pa_mapfilename;/* mapfile name */ double pa_interval; /* printing interval in seconds */ int pa_argc; char **pa_argv; STAILQ_HEAD(, pmcstat_ev) pa_head; } args; /* Function prototypes */ void pmcstat_cleanup(struct pmcstat_args *_a); int pmcstat_close_log(struct pmcstat_args *_a); void pmcstat_initialize_logging(struct pmcstat_args *_a); -int pmcstat_open(const char *_p, int _mode); +int pmcstat_open_log(const char *_p, int _mode); void pmcstat_print_counters(struct pmcstat_args *_a); void pmcstat_print_headers(struct pmcstat_args *_a); void pmcstat_print_pmcs(struct pmcstat_args *_a); void pmcstat_setup_process(struct pmcstat_args *_a); void pmcstat_show_usage(void); -void pmcstat_shutdown_logging(void); +void pmcstat_shutdown_logging(struct pmcstat_args *_a); void pmcstat_start_pmcs(struct pmcstat_args *_a); void pmcstat_start_process(struct pmcstat_args *_a); int pmcstat_process_log(struct pmcstat_args *_a); -int pmcstat_print_log(struct pmcstat_args *_a); -int pmcstat_convert_log(struct pmcstat_args *_a); #endif /* _PMCSTAT_H_ */ Index: head/usr.sbin/pmcstat/pmcstat_log.c =================================================================== --- head/usr.sbin/pmcstat/pmcstat_log.c (revision 157143) +++ head/usr.sbin/pmcstat/pmcstat_log.c (revision 157144) @@ -1,1353 +1,1790 @@ /*- - * Copyright (c) 2005, Joseph Koshy + * Copyright (c) 2005-2006, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include -__FBSDID("$FreeBSD$"); - /* - * Transform a hwpmc(4) log into human readable form and into gprof(1) - * compatible profiles. - * - * Each executable object encountered in the log gets one 'gmon.out' - * profile per PMC. We currently track: - * - program executables - * - shared libraries loaded by the runtime loader - * - the runtime loader itself - * - the kernel. - * We do not track shared objects mapped in by dlopen() yet (this - * needs additional support from hwpmc(4)). - * - * 'gmon.out' profiles generated for a given sampling PMC are - * aggregates of all the samples for that particular executable - * object. + * Transform a hwpmc(4) log into human readable form, and into + * gprof(1) compatible profiles. */ +#include +__FBSDID("$FreeBSD$"); + #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pmcstat.h" #define min(A,B) ((A) < (B) ? (A) : (B)) #define max(A,B) ((A) > (B) ? (A) : (B)) /* - * A simple implementation of interned strings. Each interned string - * is assigned a unique address, so that subsequent string compares - * can be done by a simple pointer comparision instead of with - * strcmp(). + * PUBLIC INTERFACES + * + * pmcstat_initialize_logging() initialize this module, called first + * pmcstat_shutdown_logging() orderly shutdown, called last + * pmcstat_open_log() open an eventlog for processing + * pmcstat_process_log() print/convert an event log + * pmcstat_close_log() finish processing an event log + * + * IMPLEMENTATION OF GMON OUTPUT + * + * We correlate each 'sample' seen in the event log back to an + * executable object in the system. Executable objects include: + * - program executables, + * - shared libraries loaded by the runtime loader, + * - dlopen()'ed objects loaded by the program, + * - the runtime loader itself, + * - the kernel and kernel modules. + * + * Each such executable object gets one 'gmon.out' profile, per PMC in + * use. Creation of 'gmon.out' profiles is done lazily. The + * 'gmon.out' profiles generated for a given sampling PMC are + * aggregates of all the samples for that particular executable + * object. + * + * Each process that we know about is treated as a set of regions that + * map to executable objects. Processes are described by + * 'pmcstat_process' structures. Executable objects are tracked by + * 'pmcstat_image' structures. The kernel and kernel modules are + * common to all processes (they reside at the same virtual addresses + * for all processes). Individual processes can have their text + * segments and shared libraries loaded at process-specific locations. + * + * A given executable object can be in use by multiple processes + * (e.g., libc.so) and loaded at a different address in each. + * pmcstat_pcmap structures track per-image mappings. + * + * The sample log could have samples from multiple PMCs; we + * generate one 'gmon.out' profile per PMC. */ -struct pmcstat_string { - LIST_ENTRY(pmcstat_string) ps_next; /* hash link */ - int ps_len; - int ps_hash; - const char *ps_string; -}; -static LIST_HEAD(,pmcstat_string) pmcstat_string_hash[PMCSTAT_NHASH]; +typedef const void *pmcstat_interned_string; /* * 'pmcstat_pmcrecord' is a mapping from PMC ids to human-readable * names. */ struct pmcstat_pmcrecord { LIST_ENTRY(pmcstat_pmcrecord) pr_next; - pmc_id_t pr_pmcid; - const char *pr_pmcname; + pmc_id_t pr_pmcid; + pmcstat_interned_string pr_pmcname; }; static LIST_HEAD(,pmcstat_pmcrecord) pmcstat_pmcs = LIST_HEAD_INITIALIZER(&pmcstat_pmcs); /* * struct pmcstat_gmonfile tracks a given 'gmon.out' file. These * files are mmap()'ed in as needed. */ struct pmcstat_gmonfile { LIST_ENTRY(pmcstat_gmonfile) pgf_next; /* list of entries */ + int pgf_overflow; /* whether a count overflowed */ pmc_id_t pgf_pmcid; /* id of the associated pmc */ size_t pgf_nbuckets; /* #buckets in this gmon.out */ - const char *pgf_name; /* pathname of gmon.out file */ + pmcstat_interned_string pgf_name; /* pathname of gmon.out file */ size_t pgf_ndatabytes; /* number of bytes mapped */ void *pgf_gmondata; /* pointer to mmap'ed data */ }; -static TAILQ_HEAD(,pmcstat_gmonfile) pmcstat_gmonfiles = - TAILQ_HEAD_INITIALIZER(pmcstat_gmonfiles); - /* * A 'pmcstat_image' structure describes an executable program on - * disk. 'pi_internedpath' is a cookie representing the pathname of + * disk. 'pi_execpath' is a cookie representing the pathname of * the executable. 'pi_start' and 'pi_end' are the least and greatest * virtual addresses for the text segments in the executable. * 'pi_gmonlist' contains a linked list of gmon.out files associated * with this image. */ enum pmcstat_image_type { - PMCSTAT_IMAGE_UNKNOWN = 0, - PMCSTAT_IMAGE_ELF, - PMCSTAT_IMAGE_AOUT + PMCSTAT_IMAGE_UNKNOWN = 0, /* never looked at the image */ + PMCSTAT_IMAGE_INDETERMINABLE, /* can't tell what the image is */ + PMCSTAT_IMAGE_ELF32, /* ELF 32 bit object */ + PMCSTAT_IMAGE_ELF64, /* ELF 64 bit object */ + PMCSTAT_IMAGE_AOUT /* AOUT object */ }; struct pmcstat_image { LIST_ENTRY(pmcstat_image) pi_next; /* hash link */ TAILQ_ENTRY(pmcstat_image) pi_lru; /* LRU list */ - const char *pi_internedpath; /* cookie */ - const char *pi_samplename; /* sample path name */ + pmcstat_interned_string pi_execpath;/* cookie */ + pmcstat_interned_string pi_samplename; /* sample path name */ enum pmcstat_image_type pi_type; /* executable type */ + + /* + * Executables have pi_start and pi_end; these are zero + * for shared libraries. + */ uintfptr_t pi_start; /* start address (inclusive) */ uintfptr_t pi_end; /* end address (exclusive) */ uintfptr_t pi_entry; /* entry address */ - int pi_isdynamic; /* whether a dynamic object */ - const char *pi_dynlinkerpath; /* path in .interp section */ + uintfptr_t pi_vaddr; /* virtual address where loaded */ + int pi_isdynamic; /* whether a dynamic + * object */ + int pi_iskernelmodule; + pmcstat_interned_string pi_dynlinkerpath; /* path in .interp */ + /* + * An image can be associated with one or more gmon.out files; + * one per PMC. + */ LIST_HEAD(,pmcstat_gmonfile) pi_gmlist; }; +/* + * All image descriptors are kept in a hash table. + */ static LIST_HEAD(,pmcstat_image) pmcstat_image_hash[PMCSTAT_NHASH]; static TAILQ_HEAD(,pmcstat_image) pmcstat_image_lru = TAILQ_HEAD_INITIALIZER(pmcstat_image_lru); +/* + * A 'pmcstat_pcmap' structure maps a virtual address range to an + * underlying 'pmcstat_image' descriptor. + */ struct pmcstat_pcmap { TAILQ_ENTRY(pmcstat_pcmap) ppm_next; uintfptr_t ppm_lowpc; uintfptr_t ppm_highpc; struct pmcstat_image *ppm_image; }; /* - * A 'pmcstat_process' structure tracks processes. + * A 'pmcstat_process' structure models processes. Each process is + * associated with a set of pmcstat_pcmap structures that map + * addresses inside it to executable objects. This set is implemented + * as a list, kept sorted in ascending order of mapped addresses. + * + * 'pp_pid' holds the pid of the process. When a process exits, the + * 'pp_isactive' field is set to zero, but the process structure is + * not immediately reclaimed because there may still be samples in the + * log for this process. */ struct pmcstat_process { LIST_ENTRY(pmcstat_process) pp_next; /* hash-next */ pid_t pp_pid; /* associated pid */ int pp_isactive; /* whether active */ uintfptr_t pp_entryaddr; /* entry address */ TAILQ_HEAD(,pmcstat_pcmap) pp_map; /* address range map */ }; +#define PMCSTAT_ALLOCATE 1 + +/* + * All process descriptors are kept in a hash table. + */ static LIST_HEAD(,pmcstat_process) pmcstat_process_hash[PMCSTAT_NHASH]; static struct pmcstat_process *pmcstat_kernproc; /* kernel 'process' */ +/* Misc. statistics */ +static struct pmcstat_stats { + int ps_exec_aout; /* # a.out executables seen */ + int ps_exec_elf; /* # elf executables seen */ + int ps_exec_errors; /* # errors processing executables */ + int ps_exec_indeterminable; /* # unknown executables seen */ + int ps_samples_total; /* total number of samples processed */ + int ps_samples_unknown_offset; /* #samples not in any map */ + int ps_samples_indeterminable; /* #samples in indeterminable images */ +} pmcstat_stats; + /* * Prototypes */ static void pmcstat_gmon_create_file(struct pmcstat_gmonfile *_pgf, struct pmcstat_image *_image); -static const char *pmcstat_gmon_create_name(const char *_sd, +static pmcstat_interned_string pmcstat_gmon_create_name(const char *_sd, struct pmcstat_image *_img, pmc_id_t _pmcid); static void pmcstat_gmon_map_file(struct pmcstat_gmonfile *_pgf); static void pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *_pgf); -static struct pmcstat_image *pmcstat_image_from_path(const char *_path); -static enum pmcstat_image_type pmcstat_image_get_type(const char *_p); -static void pmcstat_image_get_elf_params(struct pmcstat_image *_image); +static void pmcstat_image_determine_type(struct pmcstat_image *_image, + struct pmcstat_args *_a); +static struct pmcstat_image *pmcstat_image_from_path(pmcstat_interned_string + _path, int _iskernelmodule); +static void pmcstat_image_get_aout_params(struct pmcstat_image *_image, + struct pmcstat_args *_a); +static void pmcstat_image_get_elf_params(struct pmcstat_image *_image, + struct pmcstat_args *_a); static void pmcstat_image_increment_bucket(struct pmcstat_pcmap *_pcm, uintfptr_t _pc, pmc_id_t _pmcid, struct pmcstat_args *_a); static void pmcstat_image_link(struct pmcstat_process *_pp, - struct pmcstat_image *_i, uintfptr_t _lpc, uintfptr_t _hpc); + struct pmcstat_image *_i, uintfptr_t _lpc); -static void pmcstat_pmcid_add(pmc_id_t _pmcid, const char *_name, - struct pmcstat_args *_a); +static void pmcstat_pmcid_add(pmc_id_t _pmcid, + pmcstat_interned_string _name, struct pmcstat_args *_a); static const char *pmcstat_pmcid_to_name(pmc_id_t _pmcid); -static void pmcstat_process_add_elf_image(struct pmcstat_process *_pp, - const char *_path, uintfptr_t _entryaddr); +static void pmcstat_process_aout_exec(struct pmcstat_process *_pp, + struct pmcstat_image *_image, uintfptr_t _entryaddr, + struct pmcstat_args *_a); +static void pmcstat_process_elf_exec(struct pmcstat_process *_pp, + struct pmcstat_image *_image, uintfptr_t _entryaddr, + struct pmcstat_args *_a); static void pmcstat_process_exec(struct pmcstat_process *_pp, - const char *_path, uintfptr_t _entryaddr); -static struct pmcstat_process *pmcstat_process_lookup(pid_t _pid, int _allocate); + pmcstat_interned_string _path, uintfptr_t _entryaddr, + struct pmcstat_args *_ao); +static struct pmcstat_process *pmcstat_process_lookup(pid_t _pid, + int _allocate); static struct pmcstat_pcmap *pmcstat_process_find_map( struct pmcstat_process *_p, uintfptr_t _pc); static int pmcstat_string_compute_hash(const char *_string); -static const char *pmcstat_string_intern(const char *_s); -static struct pmcstat_string *pmcstat_string_lookup(const char *_s); +static void pmcstat_string_initialize(void); +static pmcstat_interned_string pmcstat_string_intern(const char *_s); +static pmcstat_interned_string pmcstat_string_lookup(const char *_s); +static int pmcstat_string_lookup_hash(pmcstat_interned_string _is); +static void pmcstat_string_shutdown(void); +static const char *pmcstat_string_unintern(pmcstat_interned_string _is); /* + * A simple implementation of interned strings. Each interned string + * is assigned a unique address, so that subsequent string compares + * can be done by a simple pointer comparision instead of using + * strcmp(). This speeds up hash table lookups and saves memory if + * duplicate strings are the norm. + */ +struct pmcstat_string { + LIST_ENTRY(pmcstat_string) ps_next; /* hash link */ + int ps_len; + int ps_hash; + char *ps_string; +}; + +static LIST_HEAD(,pmcstat_string) pmcstat_string_hash[PMCSTAT_NHASH]; + +/* + * Compute a 'hash' value for a string. + */ + +static int +pmcstat_string_compute_hash(const char *s) +{ + int hash; + + for (hash = 0; *s; s++) + hash ^= *s; + + return (hash & PMCSTAT_HASH_MASK); +} + +/* + * Intern a copy of string 's', and return a pointer to the + * interned structure. + */ + +static pmcstat_interned_string +pmcstat_string_intern(const char *s) +{ + struct pmcstat_string *ps; + const struct pmcstat_string *cps; + int hash, len; + + if ((cps = pmcstat_string_lookup(s)) != NULL) + return (cps); + + hash = pmcstat_string_compute_hash(s); + len = strlen(s); + + if ((ps = malloc(sizeof(*ps))) == NULL) + err(EX_OSERR, "ERROR: Could not intern string"); + ps->ps_len = len; + ps->ps_hash = hash; + ps->ps_string = strdup(s); + LIST_INSERT_HEAD(&pmcstat_string_hash[hash], ps, ps_next); + return ((pmcstat_interned_string) ps); +} + +static const char * +pmcstat_string_unintern(pmcstat_interned_string str) +{ + const char *s; + + s = ((const struct pmcstat_string *) str)->ps_string; + return (s); +} + +static pmcstat_interned_string +pmcstat_string_lookup(const char *s) +{ + struct pmcstat_string *ps; + int hash, len; + + hash = pmcstat_string_compute_hash(s); + len = strlen(s); + + LIST_FOREACH(ps, &pmcstat_string_hash[hash], ps_next) + if (ps->ps_len == len && ps->ps_hash == hash && + strcmp(ps->ps_string, s) == 0) + return (ps); + return (NULL); +} + +static int +pmcstat_string_lookup_hash(pmcstat_interned_string s) +{ + const struct pmcstat_string *ps; + + ps = (const struct pmcstat_string *) s; + return (ps->ps_hash); +} + +/* + * Initialize the string interning facility. + */ + +static void +pmcstat_string_initialize(void) +{ + int i; + + for (i = 0; i < PMCSTAT_NHASH; i++) + LIST_INIT(&pmcstat_string_hash[i]); +} + +/* + * Destroy the string table, free'ing up space. + */ + +static void +pmcstat_string_shutdown(void) +{ + int i; + struct pmcstat_string *ps, *pstmp; + + for (i = 0; i < PMCSTAT_NHASH; i++) + LIST_FOREACH_SAFE(ps, &pmcstat_string_hash[i], ps_next, + pstmp) { + LIST_REMOVE(ps, ps_next); + free(ps->ps_string); + free(ps); + } +} + +/* * Create a gmon.out file and size it. */ static void pmcstat_gmon_create_file(struct pmcstat_gmonfile *pgf, struct pmcstat_image *image) { int fd; size_t count; struct gmonhdr gm; + const char *pathname; char buffer[DEFAULT_BUFFER_SIZE]; - if ((fd = open(pgf->pgf_name, O_RDWR|O_NOFOLLOW|O_CREAT, + pathname = pmcstat_string_unintern(pgf->pgf_name); + if ((fd = open(pathname, O_RDWR|O_NOFOLLOW|O_CREAT, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)) < 0) - err(EX_OSERR, "ERROR: Cannot open \"%s\"", pgf->pgf_name); + err(EX_OSERR, "ERROR: Cannot open \"%s\"", pathname); gm.lpc = image->pi_start; gm.hpc = image->pi_end; gm.ncnt = (pgf->pgf_nbuckets * sizeof(HISTCOUNTER)) + sizeof(struct gmonhdr); gm.version = GMONVERSION; gm.profrate = 0; /* use ticks */ gm.histcounter_type = 0; /* compatibility with moncontrol() */ gm.spare[0] = gm.spare[1] = 0; /* Write out the gmon header */ if (write(fd, &gm, sizeof(gm)) < 0) goto error; /* Zero fill the samples[] array */ (void) memset(buffer, 0, sizeof(buffer)); count = pgf->pgf_ndatabytes - sizeof(struct gmonhdr); while (count > sizeof(buffer)) { if (write(fd, &buffer, sizeof(buffer)) < 0) goto error; count -= sizeof(buffer); } if (write(fd, &buffer, count) < 0) goto error; + /* TODO size the arc table */ + (void) close(fd); return; error: - err(EX_OSERR, "ERROR: Cannot write \"%s\"", pgf->pgf_name); + err(EX_OSERR, "ERROR: Cannot write \"%s\"", pathname); } -const char * +/* + * Determine the full pathname of a gmon.out file for a given + * (image,pmcid) combination. Return the interned string. + */ + +pmcstat_interned_string pmcstat_gmon_create_name(const char *samplesdir, struct pmcstat_image *image, pmc_id_t pmcid) { const char *pmcname; char fullpath[PATH_MAX]; pmcname = pmcstat_pmcid_to_name(pmcid); (void) snprintf(fullpath, sizeof(fullpath), - "%s/%s/%s", samplesdir, pmcname, image->pi_samplename); + "%s/%s/%s", samplesdir, pmcname, + pmcstat_string_unintern(image->pi_samplename)); - return pmcstat_string_intern(fullpath); + return (pmcstat_string_intern(fullpath)); } +/* + * Mmap in a gmon.out file for processing. + */ + static void pmcstat_gmon_map_file(struct pmcstat_gmonfile *pgf) { int fd; + const char *pathname; + pathname = pmcstat_string_unintern(pgf->pgf_name); + /* the gmon.out file must already exist */ - if ((fd = open(pgf->pgf_name, O_RDWR | O_NOFOLLOW, 0)) < 0) - err(EX_OSERR, "ERROR: cannot open \"%s\"", - pgf->pgf_name); + if ((fd = open(pathname, O_RDWR | O_NOFOLLOW, 0)) < 0) + err(EX_OSERR, "ERROR: cannot open \"%s\"", pathname); pgf->pgf_gmondata = mmap(NULL, pgf->pgf_ndatabytes, PROT_READ|PROT_WRITE, MAP_NOSYNC|MAP_SHARED, fd, 0); if (pgf->pgf_gmondata == MAP_FAILED) - /* XXX unmap a few files and try again? */ - err(EX_OSERR, "ERROR: cannot map \"%s\"", pgf->pgf_name); + err(EX_OSERR, "ERROR: cannot map \"%s\"", pathname); (void) close(fd); } /* - * Unmap the data mapped from a gmon.out file. + * Unmap a gmon.out file after sync'ing its data to disk. */ static void pmcstat_gmon_unmap_file(struct pmcstat_gmonfile *pgf) { (void) msync(pgf->pgf_gmondata, pgf->pgf_ndatabytes, MS_SYNC); (void) munmap(pgf->pgf_gmondata, pgf->pgf_ndatabytes); pgf->pgf_gmondata = NULL; } +/* + * Determine whether a given executable image is an A.OUT object, and + * if so, fill in its parameters from the text file. + * Sets image->pi_type. + */ + static void -pmcstat_image_get_elf_params(struct pmcstat_image *image) +pmcstat_image_get_aout_params(struct pmcstat_image *image, + struct pmcstat_args *a) { + int fd; + ssize_t nbytes; + struct exec ex; + const char *path; + char buffer[PATH_MAX]; + + path = pmcstat_string_unintern(image->pi_execpath); + assert(path != NULL); + + if (image->pi_iskernelmodule) + errx(EX_SOFTWARE, "ERROR: a.out kernel modules are " + "unsupported \"%s\"", path); + + (void) snprintf(buffer, sizeof(buffer), "%s%s", + a->pa_fsroot, path); + + if ((fd = open(buffer, O_RDONLY, 0)) < 0 || + (nbytes = read(fd, &ex, sizeof(ex))) < 0) { + warn("WARNING: Cannot determine type of \"%s\"", path); + image->pi_type = PMCSTAT_IMAGE_INDETERMINABLE; + if (fd != -1) + (void) close(fd); + return; + } + + (void) close(fd); + + if ((unsigned) nbytes != sizeof(ex) || + N_BADMAG(ex)) + return; + + image->pi_type = PMCSTAT_IMAGE_AOUT; + + /* TODO: the rest of a.out processing */ + + return; +} + +/* + * Examine an ELF file to determine the size of its text segment. + * Sets image->pi_type if anything conclusive can be determined about + * this image. + */ + +static void +pmcstat_image_get_elf_params(struct pmcstat_image *image, + struct pmcstat_args *a) +{ int fd, i; - struct stat st; + const char *path; void *mapbase; uintfptr_t minva, maxva; const Elf_Ehdr *h; const Elf_Phdr *ph; const Elf_Shdr *sh; #if defined(__amd64__) const Elf32_Ehdr *h32; const Elf32_Phdr *ph32; const Elf32_Shdr *sh32; #endif - const char *path; + enum pmcstat_image_type image_type; + struct stat st; + char buffer[PATH_MAX]; assert(image->pi_type == PMCSTAT_IMAGE_UNKNOWN); minva = ~(uintfptr_t) 0; maxva = (uintfptr_t) 0; - path = image->pi_internedpath; + path = pmcstat_string_unintern(image->pi_execpath); - if ((fd = open(path, O_RDONLY, 0)) < 0) - err(EX_OSERR, "ERROR: Cannot open \"%s\"", path); + assert(path != NULL); - if (fstat(fd, &st) < 0) - err(EX_OSERR, "ERROR: Cannot stat \"%s\"", path); + /* + * Look for kernel modules under FSROOT/KERNELPATH/NAME, + * and user mode executable objects under FSROOT/PATHNAME. + */ + if (image->pi_iskernelmodule) + (void) snprintf(buffer, sizeof(buffer), "%s%s/%s", + a->pa_fsroot, a->pa_kernel, path); + else + (void) snprintf(buffer, sizeof(buffer), "%s%s", + a->pa_fsroot, path); - if ((mapbase = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0)) == - MAP_FAILED) - err(EX_OSERR, "ERROR: Cannot mmap \"%s\"", path); + if ((fd = open(buffer, O_RDONLY, 0)) < 0 || + fstat(fd, &st) < 0 || + (mapbase = mmap(0, st.st_size, PROT_READ, MAP_SHARED, + fd, 0)) == MAP_FAILED) { + warn("WARNING: Cannot determine type of \"%s\"", buffer); + image->pi_type = PMCSTAT_IMAGE_INDETERMINABLE; + if (fd != -1) + (void) close(fd); + return; + } (void) close(fd); + /* Punt on non-ELF objects */ h = (const Elf_Ehdr *) mapbase; if (!IS_ELF(*h)) - err(EX_SOFTWARE, "ERROR: \"%s\" not an ELF file", path); + return; - /* we only handle executable objects */ - if (h->e_type != ET_EXEC && h->e_type != ET_DYN) - err(EX_DATAERR, "ERROR: Unknown file type for \"%s\"", - image->pi_internedpath); + /* + * We only handle executable ELF objects and kernel + * modules. + */ + if (h->e_type != ET_EXEC && h->e_type != ET_DYN && + !(image->pi_iskernelmodule && h->e_type == ET_REL)) + return; + image->pi_isdynamic = 0; + image->pi_dynlinkerpath = NULL; + image->pi_vaddr = 0; + #define GET_VA(H, SH, MINVA, MAXVA) do { \ for (i = 0; i < (H)->e_shnum; i++) \ if ((SH)[i].sh_flags & SHF_EXECINSTR) { \ (MINVA) = min((MINVA),(SH)[i].sh_addr); \ (MAXVA) = max((MAXVA),(SH)[i].sh_addr + \ (SH)[i].sh_size); \ } \ } while (0) #define GET_PHDR_INFO(H, PH, IMAGE) do { \ for (i = 0; i < (H)->e_phnum; i++) { \ switch ((PH)[i].p_type) { \ case PT_DYNAMIC: \ image->pi_isdynamic = 1; \ break; \ case PT_INTERP: \ image->pi_dynlinkerpath = \ pmcstat_string_intern( \ - (char *) mapbase + \ + (char *) mapbase + \ (PH)[i].p_offset); \ break; \ + case PT_LOAD: \ + if ((PH)[i].p_offset == 0) \ + image->pi_vaddr = \ + (PH)[i].p_vaddr; \ + break; \ } \ } \ } while (0) - image->pi_type = PMCSTAT_IMAGE_ELF; - image->pi_isdynamic = 0; - image->pi_dynlinkerpath = NULL; - switch (h->e_machine) { case EM_386: case EM_486: #if defined(__amd64__) /* a 32 bit executable */ h32 = (const Elf32_Ehdr *) h; sh32 = (const Elf32_Shdr *)((uintptr_t) mapbase + h32->e_shoff); GET_VA(h32, sh32, minva, maxva); image->pi_entry = h32->e_entry; if (h32->e_type == ET_EXEC) { ph32 = (const Elf32_Phdr *)((uintptr_t) mapbase + h32->e_phoff); GET_PHDR_INFO(h32, ph32, image); } + image_type = PMCSTAT_IMAGE_ELF32; break; #endif default: sh = (const Elf_Shdr *)((uintptr_t) mapbase + h->e_shoff); GET_VA(h, sh, minva, maxva); image->pi_entry = h->e_entry; if (h->e_type == ET_EXEC) { ph = (const Elf_Phdr *)((uintptr_t) mapbase + h->e_phoff); GET_PHDR_INFO(h, ph, image); } + image_type = PMCSTAT_IMAGE_ELF64; break; } #undef GET_PHDR_INFO #undef GET_VA image->pi_start = minva; - image->pi_end = maxva; + image->pi_end = maxva; + image->pi_type = image_type; if (munmap(mapbase, st.st_size) < 0) err(EX_OSERR, "ERROR: Cannot unmap \"%s\"", path); + return; +} +/* + * Given an image descriptor, determine whether it is an ELF, or AOUT. + * If no handler claims the image, set its type to 'INDETERMINABLE'. + */ + +static void +pmcstat_image_determine_type(struct pmcstat_image *image, + struct pmcstat_args *a) +{ + assert(image->pi_type == PMCSTAT_IMAGE_UNKNOWN); + + /* Try each kind of handler in turn */ + if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) + pmcstat_image_get_elf_params(image, a); + if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) + pmcstat_image_get_aout_params(image, a); + + /* + * Otherwise, remember that we tried to determine + * the object's type and had failed. + */ + if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) + image->pi_type = PMCSTAT_IMAGE_INDETERMINABLE; } /* * Locate an image descriptor given an interned path, adding a fresh * descriptor to the cache if necessary. This function also finds a * suitable name for this image's sample file. + * + * We defer filling in the file format specific parts of the image + * structure till the time we actually see a sample that would fall + * into this image. */ static struct pmcstat_image * -pmcstat_image_from_path(const char *internedpath) +pmcstat_image_from_path(pmcstat_interned_string internedpath, + int iskernelmodule) { int count, hash, nlen; struct pmcstat_image *pi; char *sn; char name[NAME_MAX]; - hash = pmcstat_string_compute_hash(internedpath); + hash = pmcstat_string_lookup_hash(internedpath); - /* Look for an existing entry. */ + /* First, look for an existing entry. */ LIST_FOREACH(pi, &pmcstat_image_hash[hash], pi_next) - if (pi->pi_internedpath == internedpath) { + if (pi->pi_execpath == internedpath && + pi->pi_iskernelmodule == iskernelmodule) { /* move descriptor to the head of the lru list */ TAILQ_REMOVE(&pmcstat_image_lru, pi, pi_lru); TAILQ_INSERT_HEAD(&pmcstat_image_lru, pi, pi_lru); - return pi; + return (pi); } /* * Allocate a new entry and place at the head of the hash and * LRU lists. */ pi = malloc(sizeof(*pi)); if (pi == NULL) - return NULL; + return (NULL); pi->pi_type = PMCSTAT_IMAGE_UNKNOWN; - pi->pi_internedpath = internedpath; + pi->pi_execpath = internedpath; pi->pi_start = ~0; pi->pi_entry = ~0; pi->pi_end = 0; + pi->pi_iskernelmodule = iskernelmodule; /* * Look for a suitable name for the sample files associated * with this image: if `basename(path)`+".gmon" is available, * we use that, otherwise we try iterating through * `basename(path)`+ "~" + NNN + ".gmon" till we get a free * entry. */ - if ((sn = basename(internedpath)) == NULL) - err(EX_OSERR, "ERROR: Cannot process \"%s\"", internedpath); + if ((sn = basename(pmcstat_string_unintern(internedpath))) == NULL) + err(EX_OSERR, "ERROR: Cannot process \"%s\"", + pmcstat_string_unintern(internedpath)); nlen = strlen(sn); - nlen = min(nlen, (int) sizeof(name) - 6); /* ".gmon\0" */ + nlen = min(nlen, (int) (sizeof(name) - sizeof(".gmon"))); snprintf(name, sizeof(name), "%.*s.gmon", nlen, sn); + /* try use the unabridged name first */ if (pmcstat_string_lookup(name) == NULL) pi->pi_samplename = pmcstat_string_intern(name); else { + /* + * Otherwise use a prefix from the original name and + * upto 3 digits. + */ nlen = strlen(sn); - nlen = min(nlen, (int) sizeof(name)-10); /* "~ddd.gmon\0" */ + nlen = min(nlen, (int) (sizeof(name)-sizeof("~NNN.gmon"))); count = 0; do { - count++; - snprintf(name, sizeof(name), "%.*s~%3.3d", + if (++count > 999) + errx(EX_CANTCREAT, "ERROR: cannot create a gmon " + "file for \"%s\"", name); + snprintf(name, sizeof(name), "%.*s~%3.3d.gmon", nlen, sn, count); if (pmcstat_string_lookup(name) == NULL) { pi->pi_samplename = pmcstat_string_intern(name); count = 0; } } while (count > 0); } + LIST_INIT(&pi->pi_gmlist); LIST_INSERT_HEAD(&pmcstat_image_hash[hash], pi, pi_next); TAILQ_INSERT_HEAD(&pmcstat_image_lru, pi, pi_lru); - return pi; + return (pi); } /* - * Given an open file, determine its file type. - */ - -static enum pmcstat_image_type -pmcstat_image_get_type(const char *path) -{ - int fd; - Elf_Ehdr eh; - struct exec ex; - ssize_t nbytes; - char buffer[DEFAULT_BUFFER_SIZE]; - - if ((fd = open(path, O_RDONLY)) < 0) - err(EX_OSERR, "ERROR: Cannot open \"%s\"", path); - - nbytes = max(sizeof(eh), sizeof(ex)); - if ((nbytes = pread(fd, buffer, nbytes, 0)) < 0) - err(EX_OSERR, "ERROR: Cannot read \"%s\"", path); - - (void) close(fd); - - /* check if its an ELF file */ - if ((unsigned) nbytes >= sizeof(Elf_Ehdr)) { - bcopy(buffer, &eh, sizeof(eh)); - if (IS_ELF(eh)) - return PMCSTAT_IMAGE_ELF; - } - - /* Look for an A.OUT header */ - if ((unsigned) nbytes >= sizeof(struct exec)) { - bcopy(buffer, &ex, sizeof(ex)); - if (!N_BADMAG(ex)) - return PMCSTAT_IMAGE_AOUT; - } - - return PMCSTAT_IMAGE_UNKNOWN; -} - -/* * Increment the bucket in the gmon.out file corresponding to 'pmcid' * and 'pc'. */ static void pmcstat_image_increment_bucket(struct pmcstat_pcmap *map, uintfptr_t pc, pmc_id_t pmcid, struct pmcstat_args *a) { struct pmcstat_image *image; struct pmcstat_gmonfile *pgf; uintfptr_t bucket; HISTCOUNTER *hc; assert(pc >= map->ppm_lowpc && pc < map->ppm_highpc); + image = map->ppm_image; + /* - * Find the gmon file corresponding to 'pmcid', creating it if - * needed. + * If this is the first time we are seeing a sample for + * this executable image, try determine its parameters. */ + if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) + pmcstat_image_determine_type(image, a); - image = map->ppm_image; + assert(image->pi_type != PMCSTAT_IMAGE_UNKNOWN); + /* Ignore samples in images that we know nothing about. */ + if (image->pi_type == PMCSTAT_IMAGE_INDETERMINABLE) { + pmcstat_stats.ps_samples_indeterminable++; + return; + } + + /* + * Find the gmon file corresponding to 'pmcid', creating it if + * needed. + */ LIST_FOREACH(pgf, &image->pi_gmlist, pgf_next) if (pgf->pgf_pmcid == pmcid) break; /* If we don't have a gmon.out file for this PMCid, create one */ if (pgf == NULL) { if ((pgf = calloc(1, sizeof(*pgf))) == NULL) err(EX_OSERR, "ERROR:"); pgf->pgf_gmondata = NULL; /* mark as unmapped */ pgf->pgf_name = pmcstat_gmon_create_name(a->pa_samplesdir, image, pmcid); pgf->pgf_pmcid = pmcid; assert(image->pi_end > image->pi_start); pgf->pgf_nbuckets = (image->pi_end - image->pi_start) / FUNCTION_ALIGNMENT; /* see */ pgf->pgf_ndatabytes = sizeof(struct gmonhdr) + pgf->pgf_nbuckets * sizeof(HISTCOUNTER); pmcstat_gmon_create_file(pgf, image); LIST_INSERT_HEAD(&image->pi_gmlist, pgf, pgf_next); } /* * Map the gmon file in if needed. It may have been mapped * out under memory pressure. */ if (pgf->pgf_gmondata == NULL) pmcstat_gmon_map_file(pgf); + assert(pgf->pgf_gmondata != NULL); + + /* + * + */ + bucket = (pc - map->ppm_lowpc) / FUNCTION_ALIGNMENT; assert(bucket < pgf->pgf_nbuckets); hc = (HISTCOUNTER *) ((uintptr_t) pgf->pgf_gmondata + sizeof(struct gmonhdr)); /* saturating add */ - if (hc[bucket] < 0xFFFF) + if (hc[bucket] < 0xFFFFU) /* XXX tie this to sizeof(HISTCOUNTER) */ hc[bucket]++; - + else /* mark that an overflow occurred */ + pgf->pgf_overflow = 1; } /* - * Record the fact that PC values from 'lowpc' to 'highpc' come from + * Record the fact that PC values from 'start' to 'end' come from * image 'image'. */ static void pmcstat_image_link(struct pmcstat_process *pp, struct pmcstat_image *image, - uintfptr_t lowpc, uintfptr_t highpc) + uintfptr_t start) { struct pmcstat_pcmap *pcm, *pcmnew; + uintfptr_t offset; + assert(image->pi_type != PMCSTAT_IMAGE_UNKNOWN && + image->pi_type != PMCSTAT_IMAGE_INDETERMINABLE); + if ((pcmnew = malloc(sizeof(*pcmnew))) == NULL) - err(EX_OSERR, "ERROR: "); + err(EX_OSERR, "ERROR: Cannot create a map entry"); - pcmnew->ppm_lowpc = lowpc; - pcmnew->ppm_highpc = highpc; + /* + * Adjust the map entry to only cover the text portion + * of the object. + */ + + offset = start - image->pi_vaddr; + pcmnew->ppm_lowpc = image->pi_start + offset; + pcmnew->ppm_highpc = image->pi_end + offset; pcmnew->ppm_image = image; + assert(pcmnew->ppm_lowpc < pcmnew->ppm_highpc); + + /* Overlapped mmap()'s are assumed to never occur. */ TAILQ_FOREACH(pcm, &pp->pp_map, ppm_next) - if (pcm->ppm_lowpc < lowpc) + if (pcm->ppm_lowpc >= pcmnew->ppm_highpc) break; if (pcm == NULL) TAILQ_INSERT_TAIL(&pp->pp_map, pcmnew, ppm_next); else TAILQ_INSERT_BEFORE(pcm, pcmnew, ppm_next); } /* + * Unmap images in the range [start..end) associated with process + * 'pp'. + */ + +static void +pmcstat_image_unmap(struct pmcstat_process *pp, uintfptr_t start, + uintfptr_t end) +{ + struct pmcstat_pcmap *pcm, *pcmtmp, *pcmnew; + + assert(pp != NULL); + assert(start < end); + + /* + * Cases: + * - we could have the range completely in the middle of an + * existing pcmap; in this case we have to split the pcmap + * structure into two (i.e., generate a 'hole'). + * - we could have the range covering multiple pcmaps; these + * will have to be removed. + * - we could have either 'start' or 'end' falling in the + * middle of a pcmap; in this case shorten the entry. + */ + + TAILQ_FOREACH_SAFE(pcm, &pp->pp_map, ppm_next, pcmtmp) { + assert(pcm->ppm_lowpc < pcm->ppm_highpc); + if (pcm->ppm_highpc <= start) + continue; + if (pcm->ppm_lowpc > end) + return; + if (pcm->ppm_lowpc >= start && pcm->ppm_highpc <= end) { + /* + * The current pcmap is completely inside the + * unmapped range: remove it entirely. + */ + TAILQ_REMOVE(&pp->pp_map, pcm, ppm_next); + free(pcm); + } else if (pcm->ppm_lowpc < start && pcm->ppm_highpc > end) { + /* + * Split this pcmap into two; curtail the + * current map to end at [start-1], and start + * the new one at [end]. + */ + if ((pcmnew = malloc(sizeof(*pcmnew))) == NULL) + err(EX_OSERR, "ERROR: Cannot split a map " + "entry"); + + pcmnew->ppm_image = pcm->ppm_image; + + pcmnew->ppm_lowpc = end; + pcmnew->ppm_highpc = pcm->ppm_highpc; + + pcm->ppm_highpc = start; + + TAILQ_INSERT_AFTER(&pp->pp_map, pcm, pcmnew, ppm_next); + + return; + } else if (pcm->ppm_lowpc < start) + pcm->ppm_lowpc = start; + else if (pcm->ppm_highpc > end) + pcm->ppm_highpc = end; + else + assert(0); + } +} + +/* * Add a {pmcid,name} mapping. */ static void -pmcstat_pmcid_add(pmc_id_t pmcid, const char *name, struct pmcstat_args *a) +pmcstat_pmcid_add(pmc_id_t pmcid, pmcstat_interned_string ps, + struct pmcstat_args *a) { struct pmcstat_pmcrecord *pr; struct stat st; char fullpath[PATH_MAX]; LIST_FOREACH(pr, &pmcstat_pmcs, pr_next) if (pr->pr_pmcid == pmcid) { - pr->pr_pmcname = name; + pr->pr_pmcname = ps; return; } if ((pr = malloc(sizeof(*pr))) == NULL) err(EX_OSERR, "ERROR: Cannot allocate pmc record"); pr->pr_pmcid = pmcid; - pr->pr_pmcname = name; + pr->pr_pmcname = ps; LIST_INSERT_HEAD(&pmcstat_pmcs, pr, pr_next); (void) snprintf(fullpath, sizeof(fullpath), "%s/%s", a->pa_samplesdir, - name); + pmcstat_string_unintern(ps)); /* If the path name exists, it should be a directory */ if (stat(fullpath, &st) == 0 && S_ISDIR(st.st_mode)) return; if (mkdir(fullpath, S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH) < 0) err(EX_OSERR, "ERROR: Cannot create directory \"%s\"", fullpath); } /* - * Given a pmcid in use, find its human-readable name, or a + * Given a pmcid in use, find its human-readable name. */ static const char * pmcstat_pmcid_to_name(pmc_id_t pmcid) { struct pmcstat_pmcrecord *pr; char fullpath[PATH_MAX]; LIST_FOREACH(pr, &pmcstat_pmcs, pr_next) if (pr->pr_pmcid == pmcid) - return pr->pr_pmcname; + return (pmcstat_string_unintern(pr->pr_pmcname)); /* create a default name and add this entry */ if ((pr = malloc(sizeof(*pr))) == NULL) err(EX_OSERR, "ERROR: "); pr->pr_pmcid = pmcid; (void) snprintf(fullpath, sizeof(fullpath), "%X", (unsigned int) pmcid); pr->pr_pmcname = pmcstat_string_intern(fullpath); LIST_INSERT_HEAD(&pmcstat_pmcs, pr, pr_next); - return pr->pr_pmcname; + return (pmcstat_string_unintern(pr->pr_pmcname)); } /* - * Associate an ELF image with a process. Argument 'path' names the - * executable while 'fd' is an already open descriptor to it. + * Associate an AOUT image with a process. */ static void -pmcstat_process_add_elf_image(struct pmcstat_process *pp, const char *path, - uintfptr_t entryaddr) +pmcstat_process_aout_exec(struct pmcstat_process *pp, + struct pmcstat_image *image, uintfptr_t entryaddr, + struct pmcstat_args *a) { - size_t linelen; - FILE *rf; - char *line; - uintmax_t libstart; - struct pmcstat_image *image, *rtldimage; - char libname[PATH_MAX], libpath[PATH_MAX]; - char command[PATH_MAX + sizeof(PMCSTAT_LDD_COMMAND) + 1]; + (void) pp; + (void) image; + (void) entryaddr; + (void) a; + /* TODO Implement a.out handling */ +} - /* Look up path in the cache. */ - if ((image = pmcstat_image_from_path(path)) == NULL) - return; +/* + * Associate an ELF image with a process. + */ - if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) - pmcstat_image_get_elf_params(image); +static void +pmcstat_process_elf_exec(struct pmcstat_process *pp, + struct pmcstat_image *image, uintfptr_t entryaddr, + struct pmcstat_args *a) +{ + uintmax_t libstart; + struct pmcstat_image *rtldimage; + assert(image->pi_type == PMCSTAT_IMAGE_ELF32 || + image->pi_type == PMCSTAT_IMAGE_ELF64); + /* Create a map entry for the base executable. */ - pmcstat_image_link(pp, image, image->pi_start, image->pi_end); + pmcstat_image_link(pp, image, image->pi_vaddr); /* * For dynamically linked executables we need to: * (a) find where the dynamic linker was mapped to for this * process, * (b) find all the executable objects that the dynamic linker * brought in. */ + if (image->pi_isdynamic) { /* * The runtime loader gets loaded just after the maximum * possible heap address. Like so: * * [ TEXT DATA BSS HEAP -->*RTLD SHLIBS <--STACK] * ^ ^ * 0 VM_MAXUSER_ADDRESS + * * The exact address where the loader gets mapped in * will vary according to the size of the executable * and the limits on the size of the process'es data * segment at the time of exec(). The entry address * recorded at process exec time corresponds to the * 'start' address inside the dynamic linker. From * this we can figure out the address where the * runtime loader's file object had been mapped to. */ - rtldimage = pmcstat_image_from_path(image->pi_dynlinkerpath); - if (rtldimage == NULL) - err(EX_OSERR, "ERROR: Cannot find image for " - "\"%s\"", image->pi_dynlinkerpath); + rtldimage = pmcstat_image_from_path(image->pi_dynlinkerpath, + 0); + if (rtldimage == NULL) { + warnx("WARNING: Cannot find image for \"%s\".", + pmcstat_string_unintern(image->pi_dynlinkerpath)); + pmcstat_stats.ps_exec_errors++; + return; + } + if (rtldimage->pi_type == PMCSTAT_IMAGE_UNKNOWN) - pmcstat_image_get_elf_params(rtldimage); + pmcstat_image_get_elf_params(rtldimage, a); - libstart = entryaddr - rtldimage->pi_entry; - pmcstat_image_link(pp, rtldimage, libstart, - libstart + rtldimage->pi_end - rtldimage->pi_start); - - /* Process all other objects loaded by this executable. */ - (void) snprintf(command, sizeof(command), "%s %s", - PMCSTAT_LDD_COMMAND, path); - - if ((rf = popen(command, "r")) == NULL) - err(EX_OSERR, "ERROR: Cannot create pipe"); - - (void) fgetln(rf, &linelen); - - while (!feof(rf) && !ferror(rf)) { - - if ((line = fgetln(rf, &linelen)) == NULL) - continue; - line[linelen-1] = '\0'; - - libstart = 0; - libpath[0] = libname[0] = '\0'; - if (sscanf(line, "%s \"%[^\"]\" %jx", - libname, libpath, &libstart) != 3) - continue; - - if (libstart == 0) { - warnx("WARNING: object \"%s\" was not found " - "for program \"%s\".", libname, path); - continue; - } - - image = pmcstat_image_from_path( - pmcstat_string_intern(libpath)); - if (image == NULL) - err(EX_OSERR, "ERROR: Cannot process " - "\"%s\"", libpath); - - if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) - pmcstat_image_get_elf_params(image); - - pmcstat_image_link(pp, image, libstart + image->pi_start, - libstart + image->pi_end); + if (rtldimage->pi_type != PMCSTAT_IMAGE_ELF32 && + rtldimage->pi_type != PMCSTAT_IMAGE_ELF64) { + warnx("WARNING: rtld not an ELF object \"%s\".", + pmcstat_string_unintern(image->pi_dynlinkerpath)); + return; } - (void) pclose(rf); - + libstart = entryaddr - rtldimage->pi_entry; + pmcstat_image_link(pp, rtldimage, libstart); } } /* * Find the process descriptor corresponding to a PID. If 'allocate' * is zero, we return a NULL if a pid descriptor could not be found or * a process descriptor process. If 'allocate' is non-zero, then we * will attempt to allocate a fresh process descriptor. Zombie * process descriptors are only removed if a fresh allocation for the * same PID is requested. */ static struct pmcstat_process * pmcstat_process_lookup(pid_t pid, int allocate) { uint32_t hash; struct pmcstat_pcmap *ppm, *ppmtmp; struct pmcstat_process *pp, *pptmp; hash = (uint32_t) pid & PMCSTAT_HASH_MASK; /* simplicity wins */ LIST_FOREACH_SAFE(pp, &pmcstat_process_hash[hash], pp_next, pptmp) if (pp->pp_pid == pid) { /* Found a descriptor, check and process zombies */ - if (allocate && !pp->pp_isactive) { + if (allocate && pp->pp_isactive == 0) { /* remove maps */ TAILQ_FOREACH_SAFE(ppm, &pp->pp_map, ppm_next, ppmtmp) { TAILQ_REMOVE(&pp->pp_map, ppm, ppm_next); free(ppm); } /* remove process entry */ LIST_REMOVE(pp, pp_next); free(pp); break; } - return pp; + return (pp); } if (!allocate) - return NULL; + return (NULL); if ((pp = malloc(sizeof(*pp))) == NULL) err(EX_OSERR, "ERROR: Cannot allocate pid descriptor"); pp->pp_pid = pid; pp->pp_isactive = 1; TAILQ_INIT(&pp->pp_map); LIST_INSERT_HEAD(&pmcstat_process_hash[hash], pp, pp_next); - return pp; + return (pp); } /* * Associate an image and a process. */ static void -pmcstat_process_exec(struct pmcstat_process *pp, const char *path, - uintfptr_t entryaddr) +pmcstat_process_exec(struct pmcstat_process *pp, + pmcstat_interned_string path, uintfptr_t entryaddr, + struct pmcstat_args *a) { - enum pmcstat_image_type filetype; struct pmcstat_image *image; - if ((image = pmcstat_image_from_path(path)) == NULL) + if ((image = pmcstat_image_from_path(path, 0)) == NULL) { + pmcstat_stats.ps_exec_errors++; return; + } if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) - filetype = pmcstat_image_get_type(path); - else - filetype = image->pi_type; + pmcstat_image_determine_type(image, a); - switch (filetype) { - case PMCSTAT_IMAGE_ELF: - pmcstat_process_add_elf_image(pp, path, entryaddr); + assert(image->pi_type != PMCSTAT_IMAGE_UNKNOWN); + + switch (image->pi_type) { + case PMCSTAT_IMAGE_ELF32: + case PMCSTAT_IMAGE_ELF64: + pmcstat_stats.ps_exec_elf++; + pmcstat_process_elf_exec(pp, image, entryaddr, a); break; case PMCSTAT_IMAGE_AOUT: + pmcstat_stats.ps_exec_aout++; + pmcstat_process_aout_exec(pp, image, entryaddr, a); break; + case PMCSTAT_IMAGE_INDETERMINABLE: + pmcstat_stats.ps_exec_indeterminable++; + break; + default: err(EX_SOFTWARE, "ERROR: Unsupported executable type for " - "\"%s\"", path); + "\"%s\"", pmcstat_string_unintern(path)); } } /* * Find the map entry associated with process 'p' at PC value 'pc'. */ static struct pmcstat_pcmap * pmcstat_process_find_map(struct pmcstat_process *p, uintfptr_t pc) { struct pmcstat_pcmap *ppm; - TAILQ_FOREACH(ppm, &p->pp_map, ppm_next) - if (pc >= ppm->ppm_lowpc && pc < ppm->ppm_highpc) - return ppm; + TAILQ_FOREACH(ppm, &p->pp_map, ppm_next) { + if (pc >= ppm->ppm_lowpc && pc < ppm->ppm_highpc) + return (ppm); + if (pc < ppm->ppm_lowpc) + return (NULL); + } - return NULL; + return (NULL); } -/* - * Compute a 'hash' value for a string. - */ static int -pmcstat_string_compute_hash(const char *s) -{ - int hash; - - for (hash = 0; *s; s++) - hash ^= *s; - - return hash & PMCSTAT_HASH_MASK; -} - -/* - * Intern a copy of string 's', and return a pointer to it. - */ - -static const char * -pmcstat_string_intern(const char *s) -{ - struct pmcstat_string *ps; - int hash, len; - - hash = pmcstat_string_compute_hash(s); - len = strlen(s); - - if ((ps = pmcstat_string_lookup(s)) != NULL) - return ps->ps_string; - - if ((ps = malloc(sizeof(*ps))) == NULL) - err(EX_OSERR, "ERROR: Could not intern string"); - ps->ps_len = len; - ps->ps_hash = hash; - ps->ps_string = strdup(s); - LIST_INSERT_HEAD(&pmcstat_string_hash[hash], ps, ps_next); - return ps->ps_string; -} - -static struct pmcstat_string * -pmcstat_string_lookup(const char *s) -{ - struct pmcstat_string *ps; - int hash, len; - - hash = pmcstat_string_compute_hash(s); - len = strlen(s); - - LIST_FOREACH(ps, &pmcstat_string_hash[hash], ps_next) - if (ps->ps_len == len && ps->ps_hash == hash && - strcmp(ps->ps_string, s) == 0) - return ps; - return NULL; -} - -/* - * Public Interfaces. - */ - -/* - * Close a logfile, after first flushing all in-module queued data. - */ - -int -pmcstat_close_log(struct pmcstat_args *a) -{ - if (pmc_flush_logfile() < 0 || - pmc_configure_logfile(-1) < 0) - err(EX_OSERR, "ERROR: logging failed"); - a->pa_flags &= ~(FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE); - return a->pa_flags & FLAG_HAS_PIPE ? PMCSTAT_EXITING : - PMCSTAT_FINISHED; -} - - -int pmcstat_convert_log(struct pmcstat_args *a) { uintfptr_t pc; + pid_t pid; + struct pmcstat_image *image; struct pmcstat_process *pp, *ppnew; struct pmcstat_pcmap *ppm, *ppmtmp; struct pmclog_ev ev; - const char *image_path; + pmcstat_interned_string image_path; while (pmclog_read(a->pa_logparser, &ev) == 0) { assert(ev.pl_state == PMCLOG_OK); switch (ev.pl_type) { - case PMCLOG_TYPE_MAPPINGCHANGE: + case PMCLOG_TYPE_INITIALIZE: + if ((ev.pl_u.pl_i.pl_version & 0xFF000000) != + PMC_VERSION_MAJOR << 24 && a->pa_verbosity > 0) + warnx("WARNING: Log version 0x%x does not " + "match compiled version 0x%x.", + ev.pl_u.pl_i.pl_version, + PMC_VERSION_MAJOR); + break; + case PMCLOG_TYPE_MAP_IN: /* * Introduce an address range mapping for a - * process. + * userland process or the kernel (pid == -1). + * + * We always allocate a process descriptor so + * that subsequent samples seen for this + * address range are mapped to the current + * object being mapped in. */ + pid = ev.pl_u.pl_mi.pl_pid; + if (pid == -1) + pp = pmcstat_kernproc; + else + pp = pmcstat_process_lookup(pid, + PMCSTAT_ALLOCATE); + + assert(pp != NULL); + + image_path = pmcstat_string_intern(ev.pl_u.pl_mi. + pl_pathname); + image = pmcstat_image_from_path(image_path, pid == -1); + if (image->pi_type == PMCSTAT_IMAGE_UNKNOWN) + pmcstat_image_determine_type(image, a); + if (image->pi_type != PMCSTAT_IMAGE_INDETERMINABLE) + pmcstat_image_link(pp, image, + ev.pl_u.pl_mi.pl_start); break; + case PMCLOG_TYPE_MAP_OUT: + /* + * Remove an address map. + */ + pid = ev.pl_u.pl_mo.pl_pid; + if (pid == -1) + pp = pmcstat_kernproc; + else + pp = pmcstat_process_lookup(pid, 0); + + if (pp == NULL) /* unknown process */ + break; + + pmcstat_image_unmap(pp, ev.pl_u.pl_mo.pl_start, + ev.pl_u.pl_mo.pl_end); + break; + case PMCLOG_TYPE_PCSAMPLE: /* * We bring in the gmon file for the image * currently associated with the PMC & pid * pair and increment the appropriate entry * bin inside this. */ + pmcstat_stats.ps_samples_total++; + pc = ev.pl_u.pl_s.pl_pc; - pp = pmcstat_process_lookup(ev.pl_u.pl_s.pl_pid, 1); + pp = pmcstat_process_lookup(ev.pl_u.pl_s.pl_pid, + PMCSTAT_ALLOCATE); if ((ppm = pmcstat_process_find_map(pp, pc)) == NULL && (ppm = pmcstat_process_find_map(pmcstat_kernproc, - pc)) == NULL) - break; /* unknown process,offset pair */ + pc)) == NULL) { /* unknown process,offset pair */ + pmcstat_stats.ps_samples_unknown_offset++; + break; + } pmcstat_image_increment_bucket(ppm, pc, ev.pl_u.pl_s.pl_pmcid, a); break; case PMCLOG_TYPE_PMCALLOCATE: /* * Record the association pmc id between this * PMC and its name. */ pmcstat_pmcid_add(ev.pl_u.pl_a.pl_pmcid, pmcstat_string_intern(ev.pl_u.pl_a.pl_evname), a); break; case PMCLOG_TYPE_PROCEXEC: /* * Change the executable image associated with * a process. */ - pp = pmcstat_process_lookup(ev.pl_u.pl_x.pl_pid, 1); + pp = pmcstat_process_lookup(ev.pl_u.pl_x.pl_pid, + PMCSTAT_ALLOCATE); /* delete the current process map */ TAILQ_FOREACH_SAFE(ppm, &pp->pp_map, ppm_next, ppmtmp) { TAILQ_REMOVE(&pp->pp_map, ppm, ppm_next); free(ppm); } - /* locate the descriptor for the new 'base' image */ + /* associate this process image */ image_path = pmcstat_string_intern( ev.pl_u.pl_x.pl_pathname); - - /* link to the new image */ + assert(image_path != NULL); pmcstat_process_exec(pp, image_path, - ev.pl_u.pl_x.pl_entryaddr); + ev.pl_u.pl_x.pl_entryaddr, a); break; case PMCLOG_TYPE_PROCEXIT: /* * Due to the way the log is generated, the * last few samples corresponding to a process * may appear in the log after the process * exit event is recorded. Thus we keep the * process' descriptor and associated data * structures around, but mark the process as * having exited. */ pp = pmcstat_process_lookup(ev.pl_u.pl_e.pl_pid, 0); if (pp == NULL) break; - pp->pp_isactive = 0; /* make a zombie */ + pp->pp_isactive = 0; /* mark as a zombie */ break; case PMCLOG_TYPE_SYSEXIT: pp = pmcstat_process_lookup(ev.pl_u.pl_se.pl_pid, 0); if (pp == NULL) break; pp->pp_isactive = 0; /* make a zombie */ break; case PMCLOG_TYPE_PROCFORK: /* - * If we had been tracking 'oldpid', then clone - * its pid descriptor. + * Allocate a process descriptor for the new + * (child) process. */ + ppnew = + pmcstat_process_lookup(ev.pl_u.pl_f.pl_newpid, + PMCSTAT_ALLOCATE); + + /* + * If we had been tracking the parent, clone + * its address maps. + */ pp = pmcstat_process_lookup(ev.pl_u.pl_f.pl_oldpid, 0); if (pp == NULL) break; - - ppnew = - pmcstat_process_lookup(ev.pl_u.pl_f.pl_newpid, 1); - - /* copy the old process' address maps */ TAILQ_FOREACH(ppm, &pp->pp_map, ppm_next) pmcstat_image_link(ppnew, ppm->ppm_image, - ppm->ppm_lowpc, ppm->ppm_highpc); + ppm->ppm_lowpc); break; default: /* other types of entries are not relevant */ break; } } if (ev.pl_state == PMCLOG_EOF) - return PMCSTAT_FINISHED; + return (PMCSTAT_FINISHED); else if (ev.pl_state == PMCLOG_REQUIRE_DATA) - return PMCSTAT_RUNNING; + return (PMCSTAT_RUNNING); err(EX_DATAERR, "ERROR: event parsing failed (record %jd, " "offset 0x%jx)", (uintmax_t) ev.pl_count + 1, ev.pl_offset); } - /* - * Open a log file, for reading or writing. - * - * The function returns the fd of a successfully opened log or -1 in - * case of failure. - */ - -int -pmcstat_open(const char *path, int mode) -{ - int fd; - - /* - * If 'path' is "-" then open one of stdin or stdout depending - * on the value of 'mode'. Otherwise, treat 'path' as a file - * name and open that. - */ - if (path[0] == '-' && path[1] == '\0') - fd = (mode == PMCSTAT_OPEN_FOR_READ) ? 0 : 1; - else - fd = open(path, mode == PMCSTAT_OPEN_FOR_READ ? - O_RDONLY : (O_WRONLY|O_CREAT|O_TRUNC), - S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); - - return fd; -} - -/* * Print log entries as text. */ -int +static int pmcstat_print_log(struct pmcstat_args *a) { struct pmclog_ev ev; while (pmclog_read(a->pa_logparser, &ev) == 0) { assert(ev.pl_state == PMCLOG_OK); switch (ev.pl_type) { case PMCLOG_TYPE_CLOSELOG: PMCSTAT_PRINT_ENTRY(a,"closelog",); break; case PMCLOG_TYPE_DROPNOTIFY: PMCSTAT_PRINT_ENTRY(a,"drop",); break; case PMCLOG_TYPE_INITIALIZE: PMCSTAT_PRINT_ENTRY(a,"initlog","0x%x \"%s\"", ev.pl_u.pl_i.pl_version, pmc_name_of_cputype(ev.pl_u.pl_i.pl_arch)); break; - case PMCLOG_TYPE_MAPPINGCHANGE: - PMCSTAT_PRINT_ENTRY(a,"mapping","%s %d %p %p \"%s\"", - ev.pl_u.pl_m.pl_type == PMCLOG_MAPPING_INSERT ? - "insert" : "delete", - ev.pl_u.pl_m.pl_pid, - (void *) ev.pl_u.pl_m.pl_start, - (void *) ev.pl_u.pl_m.pl_end, - ev.pl_u.pl_m.pl_pathname); + case PMCLOG_TYPE_MAP_IN: + PMCSTAT_PRINT_ENTRY(a,"map-in","%d %p \"%s\"", + ev.pl_u.pl_mi.pl_pid, + (void *) ev.pl_u.pl_mi.pl_start, + ev.pl_u.pl_mi.pl_pathname); break; + case PMCLOG_TYPE_MAP_OUT: + PMCSTAT_PRINT_ENTRY(a,"map-out","%d %p %p", + ev.pl_u.pl_mo.pl_pid, + (void *) ev.pl_u.pl_mo.pl_start, + (void *) ev.pl_u.pl_mo.pl_end); + break; case PMCLOG_TYPE_PCSAMPLE: PMCSTAT_PRINT_ENTRY(a,"sample","0x%x %d %p %c", ev.pl_u.pl_s.pl_pmcid, ev.pl_u.pl_s.pl_pid, (void *) ev.pl_u.pl_s.pl_pc, ev.pl_u.pl_s.pl_usermode ? 'u' : 's'); break; case PMCLOG_TYPE_PMCALLOCATE: PMCSTAT_PRINT_ENTRY(a,"allocate","0x%x \"%s\" 0x%x", ev.pl_u.pl_a.pl_pmcid, ev.pl_u.pl_a.pl_evname, ev.pl_u.pl_a.pl_flags); break; case PMCLOG_TYPE_PMCATTACH: PMCSTAT_PRINT_ENTRY(a,"attach","0x%x %d \"%s\"", ev.pl_u.pl_t.pl_pmcid, ev.pl_u.pl_t.pl_pid, ev.pl_u.pl_t.pl_pathname); break; case PMCLOG_TYPE_PMCDETACH: PMCSTAT_PRINT_ENTRY(a,"detach","0x%x %d", ev.pl_u.pl_d.pl_pmcid, ev.pl_u.pl_d.pl_pid); break; case PMCLOG_TYPE_PROCCSW: PMCSTAT_PRINT_ENTRY(a,"cswval","0x%x %d %jd", ev.pl_u.pl_c.pl_pmcid, ev.pl_u.pl_c.pl_pid, ev.pl_u.pl_c.pl_value); break; case PMCLOG_TYPE_PROCEXEC: PMCSTAT_PRINT_ENTRY(a,"exec","0x%x %d %p \"%s\"", ev.pl_u.pl_x.pl_pmcid, ev.pl_u.pl_x.pl_pid, (void *) ev.pl_u.pl_x.pl_entryaddr, ev.pl_u.pl_x.pl_pathname); break; case PMCLOG_TYPE_PROCEXIT: PMCSTAT_PRINT_ENTRY(a,"exitval","0x%x %d %jd", ev.pl_u.pl_e.pl_pmcid, ev.pl_u.pl_e.pl_pid, ev.pl_u.pl_e.pl_value); break; case PMCLOG_TYPE_PROCFORK: PMCSTAT_PRINT_ENTRY(a,"fork","%d %d", ev.pl_u.pl_f.pl_oldpid, ev.pl_u.pl_f.pl_newpid); break; case PMCLOG_TYPE_USERDATA: PMCSTAT_PRINT_ENTRY(a,"userdata","0x%x", ev.pl_u.pl_u.pl_userdata); break; case PMCLOG_TYPE_SYSEXIT: PMCSTAT_PRINT_ENTRY(a,"exit","%d", ev.pl_u.pl_se.pl_pid); break; default: fprintf(a->pa_printfile, "unknown %d", ev.pl_type); } } if (ev.pl_state == PMCLOG_EOF) - return PMCSTAT_FINISHED; + return (PMCSTAT_FINISHED); else if (ev.pl_state == PMCLOG_REQUIRE_DATA) - return PMCSTAT_RUNNING; + return (PMCSTAT_RUNNING); err(EX_DATAERR, "ERROR: event parsing failed " "(record %jd, offset 0x%jx)", (uintmax_t) ev.pl_count + 1, ev.pl_offset); /*NOTREACHED*/ } /* + * Public Interfaces. + */ + +/* + * Close a logfile, after first flushing all in-module queued data. + */ + +int +pmcstat_close_log(struct pmcstat_args *a) +{ + if (pmc_flush_logfile() < 0 || + pmc_configure_logfile(-1) < 0) + err(EX_OSERR, "ERROR: logging failed"); + a->pa_flags &= ~(FLAG_HAS_OUTPUT_LOGFILE | FLAG_HAS_PIPE); + return (a->pa_flags & FLAG_HAS_PIPE ? PMCSTAT_EXITING : + PMCSTAT_FINISHED); +} + + + +/* + * Open a log file, for reading or writing. + * + * The function returns the fd of a successfully opened log or -1 in + * case of failure. + */ + +int +pmcstat_open_log(const char *path, int mode) +{ + int fd; + + /* + * If 'path' is "-" then open one of stdin or stdout depending + * on the value of 'mode'. Otherwise, treat 'path' as a file + * name and open that. + */ + if (path[0] == '-' && path[1] == '\0') + fd = (mode == PMCSTAT_OPEN_FOR_READ) ? 0 : 1; + else + fd = open(path, mode == PMCSTAT_OPEN_FOR_READ ? + O_RDONLY : (O_WRONLY|O_CREAT|O_TRUNC), + S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); + + return (fd); +} + +/* * Process a log file in offline analysis mode. */ int pmcstat_process_log(struct pmcstat_args *a) { /* * If gprof style profiles haven't been asked for, just print the * log to the current output file. */ if (a->pa_flags & FLAG_DO_PRINT) - return pmcstat_print_log(a); + return (pmcstat_print_log(a)); else /* convert the log to gprof compatible profiles */ - return pmcstat_convert_log(a); + return (pmcstat_convert_log(a)); } +/* + * Initialize module. + */ + void pmcstat_initialize_logging(struct pmcstat_args *a) { int i; - const char *kernpath; - struct pmcstat_image *img; + (void) a; + /* use a convenient format for 'ldd' output */ if (setenv("LD_TRACE_LOADED_OBJECTS_FMT1","%o \"%p\" %x\n",1) != 0) - goto error; + err(EX_OSERR, "ERROR: Cannot setenv"); /* Initialize hash tables */ + pmcstat_string_initialize(); for (i = 0; i < PMCSTAT_NHASH; i++) { LIST_INIT(&pmcstat_image_hash[i]); LIST_INIT(&pmcstat_process_hash[i]); - LIST_INIT(&pmcstat_string_hash[i]); } - /* create a fake 'process' entry for the kernel with pid == -1 */ - if ((pmcstat_kernproc = pmcstat_process_lookup((pid_t) -1, 1)) == NULL) - goto error; - - if ((kernpath = pmcstat_string_intern(a->pa_kernel)) == NULL) - goto error; - - img = pmcstat_image_from_path(kernpath); - - pmcstat_image_get_elf_params(img); - pmcstat_image_link(pmcstat_kernproc, img, img->pi_start, img->pi_end); - - return; - - error: - err(EX_OSERR, "ERROR: Cannot initialize logging"); + /* + * Create a fake 'process' entry for the kernel with pid -1. + * hwpmc(4) will subsequently inform us about where the kernel + * and any loaded kernel modules are mapped. + */ + if ((pmcstat_kernproc = pmcstat_process_lookup((pid_t) -1, + PMCSTAT_ALLOCATE)) == NULL) + err(EX_OSERR, "ERROR: Cannot initialize logging"); } +/* + * Shutdown module. + */ + void -pmcstat_shutdown_logging(void) +pmcstat_shutdown_logging(struct pmcstat_args *a) { int i; + FILE *mf; struct pmcstat_gmonfile *pgf, *pgftmp; struct pmcstat_image *pi, *pitmp; struct pmcstat_process *pp, *pptmp; - struct pmcstat_string *ps, *pstmp; + /* determine where to send the map file */ + mf = NULL; + if (a->pa_mapfilename != NULL) + mf = (strcmp(a->pa_mapfilename, "-") == 0) ? + a->pa_printfile : fopen(a->pa_mapfilename, "w"); + + if (mf == NULL && a->pa_flags & FLAG_DO_GPROF && + a->pa_verbosity >= 2) + mf = a->pa_printfile; + + if (mf) + (void) fprintf(mf, "MAP:\n"); + for (i = 0; i < PMCSTAT_NHASH; i++) { LIST_FOREACH_SAFE(pi, &pmcstat_image_hash[i], pi_next, pitmp) { /* flush gmon.out data to disk */ LIST_FOREACH_SAFE(pgf, &pi->pi_gmlist, pgf_next, pgftmp) { - pmcstat_gmon_unmap_file(pgf); - LIST_REMOVE(pgf, pgf_next); - free(pgf); + pmcstat_gmon_unmap_file(pgf); + LIST_REMOVE(pgf, pgf_next); + + if (pgf->pgf_overflow && a->pa_verbosity >= 1) + warnx("WARNING: profile \"%s\" " + "overflowed.", + pmcstat_string_unintern(pgf->pgf_name)); + free(pgf); } + if (mf) + (void) fprintf(mf, " \"%s\" -> \"%s\"\n", + pmcstat_string_unintern(pi->pi_execpath), + pmcstat_string_unintern(pi->pi_samplename)); LIST_REMOVE(pi, pi_next); free(pi); } LIST_FOREACH_SAFE(pp, &pmcstat_process_hash[i], pp_next, pptmp) { LIST_REMOVE(pp, pp_next); free(pp); } - LIST_FOREACH_SAFE(ps, &pmcstat_string_hash[i], ps_next, - pstmp) { - LIST_REMOVE(ps, ps_next); - free(ps); - } } + + pmcstat_string_shutdown(); + + /* + * Print errors unless -q was specified. Print all statistics + * if verbosity > 1. + */ +#define PRINT(N,V,A) do { \ + if (pmcstat_stats.ps_##V || (A)->pa_verbosity >= 2) \ + (void) fprintf((A)->pa_printfile, " %-40s %d\n",\ + N, pmcstat_stats.ps_##V); \ + } while (0) + + if (a->pa_verbosity >= 1 && a->pa_flags & FLAG_DO_GPROF) { + (void) fprintf(a->pa_printfile, "CONVERSION STATISTICS:\n"); + PRINT("#exec/a.out", exec_aout, a); + PRINT("#exec/elf", exec_elf, a); + PRINT("#exec/unknown", exec_indeterminable, a); + PRINT("#exec handling errors", exec_errors, a); + PRINT("#samples/total", samples_total, a); + PRINT("#samples/unclaimed", samples_unknown_offset, a); + PRINT("#samples/unknown-object", samples_indeterminable, a); + } + + if (mf) + (void) fclose(mf); }