Index: head/tools/debugscripts/kld_deb.py =================================================================== --- head/tools/debugscripts/kld_deb.py (revision 291040) +++ head/tools/debugscripts/kld_deb.py (revision 291041) @@ -1,142 +1,143 @@ #!/usr/local/bin/python # # Copyright 2004 John-Mark Gurney # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ +from __future__ import print_function import sys import os import popen2 import re gdb_cmd = 'kgdb %(p)s/kernel.debug %(core)s | tee /tmp/gdb.log' #GDB regex filenamere = re.compile(r'filename\s+=\s+0x[0-9a-f]+\s("(?P[^"]+)"|(?P<[^>]*>))', re.M) addressre = re.compile(r'address\s+=\s+(?P0x[0-9a-f]+)', re.M) nextre = re.compile(r'tqe_next\s+=\s+(?P0x[0-9a-f]+)', re.M) printre = re.compile(r'\$\d+\s+=\s+') #Paths to search for ko's/debugs kld_debug_paths = [] if len(sys.argv[1:]) < 2: - print 'Usage: prog []' + print('Usage: prog []') sys.exit(1) #Get the base modules path pfs = sys.argv[1].split('/') try: i = 0 while 1: i = i + pfs[i:].index('sys') + 1 except: pass if i == -1: sys.stderr.write("No sys dir in kernel source path: %s\n" % sys.argv[1]) sys.exit(0) kld_debug_paths.append('/'.join(pfs[:i] + ['modules'])) kld_debug_paths.append(sys.argv[1]) #kld_debug_paths.append(sys.argv[3:]) gdb_cmd = gdb_cmd % {'p': sys.argv[1], 'core': sys.argv[2] } #Start gdb gdb = popen2.popen4(gdb_cmd) def searchfor(inp, re, j = 0, l = None): """searchfor(inp, re, j, l): Searches for regex re in inp. It will automatically add more lines. If j is set, the lines will be joined together. l can provide a starting line to help search against. Return value is a tuple of the last line, and the match if any.""" ret = None if not l: l = inp.readline() ret = re.search(l) while l and not ret: if j: l += inp.readline() else: l = inp.readline() ret = re.search(l) return (l, ret) def get_addresses(inp, out): """get_addresses(inp, out): It will search for addresses from gdb. inp and out, are the gdb input and output respectively. Return value is a list of tuples. The tuples contain the filename and the address the filename was loaded.""" addr = [] nxad = 1 while nxad: if nxad == 1: out.write("print linker_files.tqh_first[0]\n") else: out.write("print *(struct linker_file *)%d\n" % nxad) out.flush() l = searchfor(inp, printre)[0] l, fn = searchfor(inp, filenamere, 1, l) if not fn.group('fn'): sys.stderr.write("got error: %s\n" % fn.group('error')) nxad = 0 else: l, ad = searchfor(inp, addressre, 1, l) l, nx = searchfor(inp, nextre, 1, l) addr.append((fn.group('fn'), long(ad.group('ad'), 16))) nxad = long(nx.group('ad'), 16) return addr #Get the addresses addr = get_addresses(gdb[0], gdb[1]) #Pass through the resulting addresses, skipping the kernel. for i in addr[1:]: for j in kld_debug_paths: #Try .debug first. p = popen2.popen4('find %s -type f -name "%s.debug"' % (j, i[0]))[0].read().strip() if p: break #Try just .ko if .debug wasn't found. p = popen2.popen4('find %s -type f -name "%s"' % (j, i[0]))[0].read().strip() if p: break if not p: #Tell our user that we couldn't find it. a = i[1] sys.stderr.write("Can't find module: %s (addr: %d + header)\n" % (i[0], a)) - print '#add-symbol-file ', a, '#add header' + print('#add-symbol-file ', a, '#add header') continue #j = popen2.popen4('objdump --section-headers /boot/kernel/%s | grep "\.text"' % i[0])[0].read().strip().split() #Output the necessary information j = popen2.popen4('objdump --section-headers "%s" | grep "\.text"' % p)[0].read().strip().split() try: a = int(j[5], 16) - print 'add-symbol-file', p, i[1] + a + print('add-symbol-file', p, i[1] + a) except IndexError: sys.stderr.write('Bad file: %s, address: %d\n' % (i[0], i[1])) Index: head/tools/regression/netinet/ip_id_period/ip_id_period.py =================================================================== --- head/tools/regression/netinet/ip_id_period/ip_id_period.py (revision 291040) +++ head/tools/regression/netinet/ip_id_period/ip_id_period.py (revision 291041) @@ -1,76 +1,77 @@ # Copyright (C) 2008 Michael J. Silbersack. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice unmodified, this list of conditions, and the following # disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # $FreeBSD$ # # This is a regression test to verify the proper behavior of IP ID generation # code. It will push 200000 packets, then report back what the min and max # periods it saw for different IDs were. +from __future__ import print_function import os import signal import subprocess import time if os.path.exists('results.pcap'): os.remove('results.pcap') tcpdump = subprocess.Popen('tcpdump -n -i lo0 -w results.pcap icmp', shell=True) time.sleep(1) # Give tcpdump time to start os.system('sysctl net.inet.icmp.icmplim=0') os.system('ping -q -i .001 -c 100000 127.0.0.1') time.sleep(3) # Give tcpdump time to catch up os.kill(tcpdump.pid, signal.SIGTERM) os.system('tcpdump -n -v -r results.pcap > results.txt') id_lastseen = {} id_minperiod = {} count = 0 for line in open('results.txt').readlines(): id = int(line.split(' id ')[1].split(',')[0]) if id in id_lastseen: period = count - id_lastseen[id] if id not in id_minperiod or period < id_minperiod[id]: id_minperiod[id] = period id_lastseen[id] = count count += 1 sorted_minperiod = list(zip(*reversed(list(zip(*list(id_minperiod.items())))))) sorted_minperiod.sort() print("Lowest 10 ID periods detected:") x = 0 while x < 10: id_tuple = sorted_minperiod.pop(0) print("id: %d period: %d" % (id_tuple[1], id_tuple[0])) x += 1 print("Highest 10 ID periods detected:") x = 0 while x < 10: id_tuple = sorted_minperiod.pop() print("id: %d period: %d" % (id_tuple[1], id_tuple[0])) x += 1 Index: head/tools/sched/schedgraph.py =================================================================== --- head/tools/sched/schedgraph.py (revision 291040) +++ head/tools/sched/schedgraph.py (revision 291041) @@ -1,1640 +1,1641 @@ #!/usr/local/bin/python # Copyright (c) 2002-2003, 2009, Jeffrey Roberson # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice unmodified, this list of conditions, and the following # disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # $FreeBSD$ +from __future__ import print_function import sys import re import random from Tkinter import * # To use: # - Install the ports/x11-toolkits/py-tkinter package; e.g. # portinstall x11-toolkits/py-tkinter package # - Add KTR_SCHED to KTR_COMPILE and KTR_MASK in your KERNCONF; e.g. # options KTR # options KTR_ENTRIES=32768 # options KTR_COMPILE=(KTR_SCHED) # options KTR_MASK=(KTR_SCHED) # - It is encouraged to increase KTR_ENTRIES size to gather enough # information for analysis; e.g. # options KTR_ENTRIES=262144 # as 32768 entries may only correspond to a second or two of profiling # data depending on your workload. # - Rebuild kernel with proper changes to KERNCONF and boot new kernel. # - Run your workload to be profiled. # - While the workload is continuing (i.e. before it finishes), disable # KTR tracing by setting 'sysctl debug.ktr.mask=0'. This is necessary # to avoid a race condition while running ktrdump, i.e. the KTR ring buffer # will cycle a bit while ktrdump runs, and this confuses schedgraph because # the timestamps appear to go backwards at some point. Stopping KTR logging # while the workload is still running is to avoid wasting log entries on # "idle" time at the end. # - Dump the trace to a file: 'ktrdump -ct > ktr.out' # - Run the python script: 'python schedgraph.py ktr.out' optionally provide # your cpu frequency in ghz: 'python schedgraph.py ktr.out 2.4' # # To do: # Add a per-source summary display # "Vertical rule" to help relate data in different rows # Mouse-over popup of full thread/event/row label (currently truncated) # More visible anchors for popup event windows # # BUGS: 1) Only 8 CPUs are supported, more CPUs require more choices of # colours to represent them ;-) eventcolors = [ ("count", "red"), ("running", "green"), ("idle", "grey"), ("spinning", "red"), ("yielding", "yellow"), ("swapped", "violet"), ("suspended", "purple"), ("iwait", "grey"), ("sleep", "blue"), ("blocked", "dark red"), ("runq add", "yellow"), ("runq rem", "yellow"), ("thread exit", "grey"), ("proc exit", "grey"), ("lock acquire", "blue"), ("lock contest", "purple"), ("failed lock try", "red"), ("lock release", "grey"), ("statclock", "black"), ("prio", "black"), ("lend prio", "black"), ("wokeup", "black") ] cpucolors = [ ("CPU 0", "light grey"), ("CPU 1", "dark grey"), ("CPU 2", "light blue"), ("CPU 3", "light pink"), ("CPU 4", "blanched almond"), ("CPU 5", "slate grey"), ("CPU 6", "tan"), ("CPU 7", "thistle"), ("CPU 8", "white") ] colors = [ "white", "thistle", "blanched almond", "tan", "chartreuse", "dark red", "red", "pale violet red", "pink", "light pink", "dark orange", "orange", "coral", "light coral", "goldenrod", "gold", "yellow", "light yellow", "dark green", "green", "light green", "light sea green", "dark blue", "blue", "light blue", "steel blue", "light slate blue", "dark violet", "violet", "purple", "blue violet", "dark grey", "slate grey", "light grey", "black", ] colors.sort() ticksps = None status = None colormap = None ktrfile = None clockfreq = None sources = [] lineno = -1 Y_BORDER = 10 X_BORDER = 10 Y_COUNTER = 80 Y_EVENTSOURCE = 10 XY_POINT = 4 class Colormap: def __init__(self, table): self.table = table self.map = {} for entry in table: self.map[entry[0]] = entry[1] def lookup(self, name): try: color = self.map[name] except: color = colors[random.randrange(0, len(colors))] - print "Picking random color", color, "for", name + print("Picking random color", color, "for", name) self.map[name] = color self.table.append((name, color)) return (color) def ticks2sec(ticks): ticks = float(ticks) ns = float(ticksps) / 1000000000 ticks /= ns if (ticks < 1000): return ("%.2fns" % ticks) ticks /= 1000 if (ticks < 1000): return ("%.2fus" % ticks) ticks /= 1000 if (ticks < 1000): return ("%.2fms" % ticks) ticks /= 1000 return ("%.2fs" % ticks) class Scaler(Frame): def __init__(self, master, target): Frame.__init__(self, master) self.scale = None self.target = target self.label = Label(self, text="Ticks per pixel") self.label.pack(side=LEFT) self.resolution = 100 self.setmax(10000) def scaleset(self, value): self.target.scaleset(int(value)) def set(self, value): self.scale.set(value) def setmax(self, value): # # We can't reconfigure the to_ value so we delete the old # window and make a new one when we resize. # if (self.scale != None): self.scale.pack_forget() self.scale.destroy() self.scale = Scale(self, command=self.scaleset, from_=100, to_=value, orient=HORIZONTAL, resolution=self.resolution) self.scale.pack(fill="both", expand=1) self.scale.set(self.target.scaleget()) class Status(Frame): def __init__(self, master): Frame.__init__(self, master) self.label = Label(self, bd=1, relief=SUNKEN, anchor=W) self.label.pack(fill="both", expand=1) self.clear() def set(self, str): self.label.config(text=str) def clear(self): self.label.config(text="") def startup(self, str): self.set(str) root.update() class ColorConf(Frame): def __init__(self, master, name, color): Frame.__init__(self, master) if (graph.getstate(name) == "hidden"): enabled = 0 else: enabled = 1 self.name = name self.color = StringVar() self.color_default = color self.color_current = color self.color.set(color) self.enabled = IntVar() self.enabled_default = enabled self.enabled_current = enabled self.enabled.set(enabled) self.draw() def draw(self): self.label = Label(self, text=self.name, anchor=W) self.sample = Canvas(self, width=24, height=24, bg='grey') self.rect = self.sample.create_rectangle(0, 0, 24, 24, fill=self.color.get()) self.list = OptionMenu(self, self.color, command=self.setcolor, *colors) self.checkbox = Checkbutton(self, text="enabled", variable=self.enabled) self.label.grid(row=0, column=0, sticky=E+W) self.sample.grid(row=0, column=1) self.list.grid(row=0, column=2, sticky=E+W) self.checkbox.grid(row=0, column=3) self.columnconfigure(0, weight=1) self.columnconfigure(2, minsize=150) def setcolor(self, color): self.color.set(color) self.sample.itemconfigure(self.rect, fill=color) def apply(self): cchange = 0 echange = 0 if (self.color_current != self.color.get()): cchange = 1 if (self.enabled_current != self.enabled.get()): echange = 1 self.color_current = self.color.get() self.enabled_current = self.enabled.get() if (echange != 0): if (self.enabled_current): graph.setcolor(self.name, self.color_current) else: graph.hide(self.name) return if (cchange != 0): graph.setcolor(self.name, self.color_current) def revert(self): self.setcolor(self.color_default) self.enabled.set(self.enabled_default) class ColorConfigure(Toplevel): def __init__(self, table, name): Toplevel.__init__(self) self.resizable(0, 0) self.title(name) self.items = LabelFrame(self, text="Item Type") self.buttons = Frame(self) self.drawbuttons() self.items.grid(row=0, column=0, sticky=E+W) self.columnconfigure(0, weight=1) self.buttons.grid(row=1, column=0, sticky=E+W) self.types = [] self.irow = 0 for type in table: color = graph.getcolor(type[0]) if (color != ""): self.additem(type[0], color) self.bind("", self.destroycb) def destroycb(self, event): self.destroy() def additem(self, name, color): item = ColorConf(self.items, name, color) self.types.append(item) item.grid(row=self.irow, column=0, sticky=E+W) self.irow += 1 def drawbuttons(self): self.apply = Button(self.buttons, text="Apply", command=self.apress) self.default = Button(self.buttons, text="Revert", command=self.rpress) self.apply.grid(row=0, column=0, sticky=E+W) self.default.grid(row=0, column=1, sticky=E+W) self.buttons.columnconfigure(0, weight=1) self.buttons.columnconfigure(1, weight=1) def apress(self): for item in self.types: item.apply() def rpress(self): for item in self.types: item.revert() class SourceConf(Frame): def __init__(self, master, source): Frame.__init__(self, master) if (source.hidden == 1): enabled = 0 else: enabled = 1 self.source = source self.name = source.name self.enabled = IntVar() self.enabled_default = enabled self.enabled_current = enabled self.enabled.set(enabled) self.draw() def draw(self): self.label = Label(self, text=self.name, anchor=W) self.checkbox = Checkbutton(self, text="enabled", variable=self.enabled) self.label.grid(row=0, column=0, sticky=E+W) self.checkbox.grid(row=0, column=1) self.columnconfigure(0, weight=1) def changed(self): if (self.enabled_current != self.enabled.get()): return 1 return 0 def apply(self): self.enabled_current = self.enabled.get() def revert(self): self.enabled.set(self.enabled_default) def check(self): self.enabled.set(1) def uncheck(self): self.enabled.set(0) class SourceConfigure(Toplevel): def __init__(self): Toplevel.__init__(self) self.resizable(0, 0) self.title("Source Configuration") self.items = [] self.iframe = Frame(self) self.iframe.grid(row=0, column=0, sticky=E+W) f = LabelFrame(self.iframe, bd=4, text="Sources") self.items.append(f) self.buttons = Frame(self) self.items[0].grid(row=0, column=0, sticky=E+W) self.columnconfigure(0, weight=1) self.sconfig = [] self.irow = 0 self.icol = 0 for source in sources: self.addsource(source) self.drawbuttons() self.buttons.grid(row=1, column=0, sticky=W) self.bind("", self.destroycb) def destroycb(self, event): self.destroy() def addsource(self, source): if (self.irow > 30): self.icol += 1 self.irow = 0 c = self.icol f = LabelFrame(self.iframe, bd=4, text="Sources") f.grid(row=0, column=c, sticky=N+E+W) self.items.append(f) item = SourceConf(self.items[self.icol], source) self.sconfig.append(item) item.grid(row=self.irow, column=0, sticky=E+W) self.irow += 1 def drawbuttons(self): self.apply = Button(self.buttons, text="Apply", command=self.apress) self.default = Button(self.buttons, text="Revert", command=self.rpress) self.checkall = Button(self.buttons, text="Check All", command=self.cpress) self.uncheckall = Button(self.buttons, text="Uncheck All", command=self.upress) self.checkall.grid(row=0, column=0, sticky=W) self.uncheckall.grid(row=0, column=1, sticky=W) self.apply.grid(row=0, column=2, sticky=W) self.default.grid(row=0, column=3, sticky=W) self.buttons.columnconfigure(0, weight=1) self.buttons.columnconfigure(1, weight=1) self.buttons.columnconfigure(2, weight=1) self.buttons.columnconfigure(3, weight=1) def apress(self): disable_sources = [] enable_sources = [] for item in self.sconfig: if (item.changed() == 0): continue if (item.enabled.get() == 1): enable_sources.append(item.source) else: disable_sources.append(item.source) if (len(disable_sources)): graph.sourcehidelist(disable_sources) if (len(enable_sources)): graph.sourceshowlist(enable_sources) for item in self.sconfig: item.apply() def rpress(self): for item in self.sconfig: item.revert() def cpress(self): for item in self.sconfig: item.check() def upress(self): for item in self.sconfig: item.uncheck() # Reverse compare of second member of the tuple def cmp_counts(x, y): return y[1] - x[1] class SourceStats(Toplevel): def __init__(self, source): self.source = source Toplevel.__init__(self) self.resizable(0, 0) self.title(source.name + " statistics") self.evframe = LabelFrame(self, text="Event Count, Duration, Avg Duration") self.evframe.grid(row=0, column=0, sticky=E+W) eventtypes={} for event in self.source.events: if (event.type == "pad"): continue duration = event.duration if (event.name in eventtypes): (c, d) = eventtypes[event.name] c += 1 d += duration eventtypes[event.name] = (c, d) else: eventtypes[event.name] = (1, duration) events = [] for k, v in eventtypes.iteritems(): (c, d) = v events.append((k, c, d)) events.sort(cmp=cmp_counts) ypos = 0 for event in events: (name, c, d) = event Label(self.evframe, text=name, bd=1, relief=SUNKEN, anchor=W, width=30).grid( row=ypos, column=0, sticky=W+E) Label(self.evframe, text=str(c), bd=1, relief=SUNKEN, anchor=W, width=10).grid( row=ypos, column=1, sticky=W+E) Label(self.evframe, text=ticks2sec(d), bd=1, relief=SUNKEN, width=10).grid( row=ypos, column=2, sticky=W+E) if (d and c): d /= c else: d = 0 Label(self.evframe, text=ticks2sec(d), bd=1, relief=SUNKEN, width=10).grid( row=ypos, column=3, sticky=W+E) ypos += 1 self.bind("", self.destroycb) def destroycb(self, event): self.destroy() class SourceContext(Menu): def __init__(self, event, source): self.source = source Menu.__init__(self, tearoff=0, takefocus=0) self.add_command(label="hide", command=self.hide) self.add_command(label="hide group", command=self.hidegroup) self.add_command(label="stats", command=self.stats) self.tk_popup(event.x_root-3, event.y_root+3) def hide(self): graph.sourcehide(self.source) def hidegroup(self): grouplist = [] for source in sources: if (source.group == self.source.group): grouplist.append(source) graph.sourcehidelist(grouplist) def show(self): graph.sourceshow(self.source) def stats(self): SourceStats(self.source) class EventView(Toplevel): def __init__(self, event, canvas): Toplevel.__init__(self) self.resizable(0, 0) self.title("Event") self.event = event self.buttons = Frame(self) self.buttons.grid(row=0, column=0, sticky=E+W) self.frame = Frame(self) self.frame.grid(row=1, column=0, sticky=N+S+E+W) self.canvas = canvas self.drawlabels() self.drawbuttons() event.displayref(canvas) self.bind("", self.destroycb) self.bind("", self.destroycb) def destroycb(self, event): self.unbind("") if (self.event != None): self.event.displayunref(self.canvas) self.event = None self.destroy() def clearlabels(self): for label in self.frame.grid_slaves(): label.grid_remove() def drawlabels(self): ypos = 0 labels = self.event.labels() while (len(labels) < 7): labels.append(("", "")) for label in labels: name, value = label linked = 0 if (name == "linkedto"): linked = 1 l = Label(self.frame, text=name, bd=1, width=15, relief=SUNKEN, anchor=W) if (linked): fgcolor = "blue" else: fgcolor = "black" r = Label(self.frame, text=value, bd=1, relief=SUNKEN, anchor=W, fg=fgcolor) l.grid(row=ypos, column=0, sticky=E+W) r.grid(row=ypos, column=1, sticky=E+W) if (linked): r.bind("", self.linkpress) ypos += 1 self.frame.columnconfigure(1, minsize=80) def drawbuttons(self): self.back = Button(self.buttons, text="<", command=self.bpress) self.forw = Button(self.buttons, text=">", command=self.fpress) self.new = Button(self.buttons, text="new", command=self.npress) self.back.grid(row=0, column=0, sticky=E+W) self.forw.grid(row=0, column=1, sticky=E+W) self.new.grid(row=0, column=2, sticky=E+W) self.buttons.columnconfigure(2, weight=1) def newevent(self, event): self.event.displayunref(self.canvas) self.clearlabels() self.event = event self.event.displayref(self.canvas) self.drawlabels() def npress(self): EventView(self.event, self.canvas) def bpress(self): prev = self.event.prev() if (prev == None): return while (prev.type == "pad"): prev = prev.prev() if (prev == None): return self.newevent(prev) def fpress(self): next = self.event.next() if (next == None): return while (next.type == "pad"): next = next.next() if (next == None): return self.newevent(next) def linkpress(self, wevent): event = self.event.getlinked() if (event != None): self.newevent(event) class Event: def __init__(self, source, name, cpu, timestamp, attrs): self.source = source self.name = name self.cpu = cpu self.timestamp = int(timestamp) self.attrs = attrs self.idx = None self.item = None self.dispcnt = 0 self.duration = 0 self.recno = lineno def status(self): statstr = self.name + " " + self.source.name statstr += " on: cpu" + str(self.cpu) statstr += " at: " + str(self.timestamp) statstr += " attributes: " for i in range(0, len(self.attrs)): attr = self.attrs[i] statstr += attr[0] + ": " + str(attr[1]) if (i != len(self.attrs) - 1): statstr += ", " status.set(statstr) def labels(self): return [("Source", self.source.name), ("Event", self.name), ("CPU", self.cpu), ("Timestamp", self.timestamp), ("KTR Line ", self.recno) ] + self.attrs def mouseenter(self, canvas): self.displayref(canvas) self.status() def mouseexit(self, canvas): self.displayunref(canvas) status.clear() def mousepress(self, canvas): EventView(self, canvas) def draw(self, canvas, xpos, ypos, item): self.item = item if (item != None): canvas.items[item] = self def move(self, canvas, x, y): if (self.item == None): return; canvas.move(self.item, x, y); def next(self): return self.source.eventat(self.idx + 1) def nexttype(self, type): next = self.next() while (next != None and next.type != type): next = next.next() return (next) def prev(self): return self.source.eventat(self.idx - 1) def displayref(self, canvas): if (self.dispcnt == 0): canvas.itemconfigure(self.item, width=2) self.dispcnt += 1 def displayunref(self, canvas): self.dispcnt -= 1 if (self.dispcnt == 0): canvas.itemconfigure(self.item, width=0) canvas.tag_raise("point", "state") def getlinked(self): for attr in self.attrs: if (attr[0] != "linkedto"): continue source = ktrfile.findid(attr[1]) return source.findevent(self.timestamp) return None class PointEvent(Event): type = "point" def __init__(self, source, name, cpu, timestamp, attrs): Event.__init__(self, source, name, cpu, timestamp, attrs) def draw(self, canvas, xpos, ypos): color = colormap.lookup(self.name) l = canvas.create_oval(xpos - XY_POINT, ypos, xpos + XY_POINT, ypos - (XY_POINT * 2), fill=color, width=0, tags=("event", self.type, self.name, self.source.tag)) Event.draw(self, canvas, xpos, ypos, l) return xpos class StateEvent(Event): type = "state" def __init__(self, source, name, cpu, timestamp, attrs): Event.__init__(self, source, name, cpu, timestamp, attrs) def draw(self, canvas, xpos, ypos): next = self.nexttype("state") if (next == None): return (xpos) self.duration = duration = next.timestamp - self.timestamp self.attrs.insert(0, ("duration", ticks2sec(duration))) color = colormap.lookup(self.name) if (duration < 0): duration = 0 - print "Unsynchronized timestamp" - print self.cpu, self.timestamp - print next.cpu, next.timestamp + print("Unsynchronized timestamp") + print(self.cpu, self.timestamp) + print(next.cpu, next.timestamp) delta = duration / canvas.ratio l = canvas.create_rectangle(xpos, ypos, xpos + delta, ypos - 10, fill=color, width=0, tags=("event", self.type, self.name, self.source.tag)) Event.draw(self, canvas, xpos, ypos, l) return (xpos + delta) class CountEvent(Event): type = "count" def __init__(self, source, count, cpu, timestamp, attrs): count = int(count) self.count = count Event.__init__(self, source, "count", cpu, timestamp, attrs) def draw(self, canvas, xpos, ypos): next = self.nexttype("count") if (next == None): return (xpos) color = colormap.lookup("count") self.duration = duration = next.timestamp - self.timestamp if (duration < 0): duration = 0 - print "Unsynchronized timestamp" - print self.cpu, self.timestamp - print next.cpu, next.timestamp + print("Unsynchronized timestamp") + print(self.cpu, self.timestamp) + print(next.cpu, next.timestamp) self.attrs.insert(0, ("count", self.count)) self.attrs.insert(1, ("duration", ticks2sec(duration))) delta = duration / canvas.ratio yhight = self.source.yscale() * self.count l = canvas.create_rectangle(xpos, ypos - yhight, xpos + delta, ypos, fill=color, width=0, tags=("event", self.type, self.name, self.source.tag)) Event.draw(self, canvas, xpos, ypos, l) return (xpos + delta) class PadEvent(StateEvent): type = "pad" def __init__(self, source, cpu, timestamp, last=0): if (last): cpu = source.events[len(source.events) -1].cpu else: cpu = source.events[0].cpu StateEvent.__init__(self, source, "pad", cpu, timestamp, []) def draw(self, canvas, xpos, ypos): next = self.next() if (next == None): return (xpos) duration = next.timestamp - self.timestamp delta = duration / canvas.ratio Event.draw(self, canvas, xpos, ypos, None) return (xpos + delta) # Sort function for start y address def source_cmp_start(x, y): return x.y - y.y class EventSource: def __init__(self, group, id): self.name = id self.events = [] self.cpuitems = [] self.group = group self.y = 0 self.item = None self.hidden = 0 self.tag = group + id def __cmp__(self, other): if (other == None): return -1 if (self.group == other.group): return cmp(self.name, other.name) return cmp(self.group, other.group) # It is much faster to append items to a list then to insert them # at the beginning. As a result, we add events in reverse order # and then swap the list during fixup. def fixup(self): self.events.reverse() def addevent(self, event): self.events.append(event) def addlastevent(self, event): self.events.insert(0, event) def draw(self, canvas, ypos): xpos = 10 cpux = 10 cpu = self.events[1].cpu for i in range(0, len(self.events)): self.events[i].idx = i for event in self.events: if (event.cpu != cpu and event.cpu != -1): self.drawcpu(canvas, cpu, cpux, xpos, ypos) cpux = xpos cpu = event.cpu xpos = event.draw(canvas, xpos, ypos) self.drawcpu(canvas, cpu, cpux, xpos, ypos) def drawname(self, canvas, ypos): self.y = ypos ypos = ypos - (self.ysize() / 2) self.item = canvas.create_text(X_BORDER, ypos, anchor="w", text=self.name) return (self.item) def drawcpu(self, canvas, cpu, fromx, tox, ypos): cpu = "CPU " + str(cpu) color = cpucolormap.lookup(cpu) # Create the cpu background colors default to hidden l = canvas.create_rectangle(fromx, ypos - self.ysize() - canvas.bdheight, tox, ypos + canvas.bdheight, fill=color, width=0, tags=("cpubg", cpu, self.tag), state="hidden") self.cpuitems.append(l) def move(self, canvas, xpos, ypos): canvas.move(self.tag, xpos, ypos) def movename(self, canvas, xpos, ypos): self.y += ypos canvas.move(self.item, xpos, ypos) def ysize(self): return (Y_EVENTSOURCE) def eventat(self, i): if (i >= len(self.events) or i < 0): return (None) event = self.events[i] return (event) def findevent(self, timestamp): for event in self.events: if (event.timestamp >= timestamp and event.type != "pad"): return (event) return (None) class Counter(EventSource): # # Store a hash of counter groups that keeps the max value # for a counter in this group for scaling purposes. # groups = {} def __init__(self, group, id): try: Counter.cnt = Counter.groups[group] except: Counter.groups[group] = 0 EventSource.__init__(self, group, id) def fixup(self): for event in self.events: if (event.type != "count"): continue; count = int(event.count) if (count > Counter.groups[self.group]): Counter.groups[self.group] = count EventSource.fixup(self) def ymax(self): return (Counter.groups[self.group]) def ysize(self): return (Y_COUNTER) def yscale(self): return (self.ysize() / self.ymax()) class KTRFile: def __init__(self, file): self.timestamp_f = None self.timestamp_l = None self.locks = {} self.ticks = {} self.load = {} self.crit = {} self.stathz = 0 self.eventcnt = 0 self.taghash = {} self.parse(file) self.fixup() global ticksps ticksps = self.ticksps() span = self.timespan() ghz = float(ticksps) / 1000000000.0 # # Update the title with some stats from the file # titlestr = "SchedGraph: " titlestr += ticks2sec(span) + " at %.3f ghz, " % ghz titlestr += str(len(sources)) + " event sources, " titlestr += str(self.eventcnt) + " events" root.title(titlestr) def parse(self, file): try: ifp = open(file) except: - print "Can't open", file + print("Can't open", file) sys.exit(1) # quoteexp matches a quoted string, no escaping quoteexp = "\"([^\"]*)\"" # # commaexp matches a quoted string OR the string up # to the first ',' # commaexp = "(?:" + quoteexp + "|([^,]+))" # # colonstr matches a quoted string OR the string up # to the first ':' # colonexp = "(?:" + quoteexp + "|([^:]+))" # # Match various manditory parts of the KTR string this is # fairly inflexible until you get to attributes to make # parsing faster. # hdrexp = "\s*(\d+)\s+(\d+)\s+(\d+)\s+" groupexp = "KTRGRAPH group:" + quoteexp + ", " idexp = "id:" + quoteexp + ", " typeexp = "([^:]+):" + commaexp + ", " attribexp = "attributes: (.*)" # # Matches optional attributes in the KTR string. This # tolerates more variance as the users supply these values. # attrexp = colonexp + "\s*:\s*(?:" + commaexp + ", (.*)|" attrexp += quoteexp +"|(.*))" # Precompile regexp ktrre = re.compile(hdrexp + groupexp + idexp + typeexp + attribexp) attrre = re.compile(attrexp) global lineno lineno = 0 for line in ifp.readlines(): lineno += 1 if ((lineno % 2048) == 0): status.startup("Parsing line " + str(lineno)) m = ktrre.match(line); if (m == None): - print "Can't parse", lineno, line, + print("Can't parse", lineno, line, end=' ') continue; (index, cpu, timestamp, group, id, type, dat, dat1, attrstring) = m.groups(); if (dat == None): dat = dat1 if (self.checkstamp(timestamp) == 0): - print "Bad timestamp at", lineno, ":", - print cpu, timestamp + print("Bad timestamp at", lineno, ":", end=' ') + print(cpu, timestamp) continue # # Build the table of optional attributes # attrs = [] while (attrstring != None): m = attrre.match(attrstring.strip()) if (m == None): break; # # Name may or may not be quoted. # # For val we have four cases: # 1) quotes followed by comma and more # attributes. # 2) no quotes followed by comma and more # attributes. # 3) no more attributes or comma with quotes. # 4) no more attributes or comma without quotes. # (name, name1, val, val1, attrstring, end, end1) = m.groups(); if (name == None): name = name1 if (end == None): end = end1 if (val == None): val = val1 if (val == None): val = end if (name == "stathz"): self.setstathz(val, cpu) attrs.append((name, val)) args = (dat, cpu, timestamp, attrs) e = self.makeevent(group, id, type, args) if (e == None): - print "Unknown type", type, lineno, line, + print("Unknown type", type, lineno, line, end=' ') def makeevent(self, group, id, type, args): e = None source = self.makeid(group, id, type) if (type == "state"): e = StateEvent(source, *args) elif (type == "counter"): e = CountEvent(source, *args) elif (type == "point"): e = PointEvent(source, *args) if (e != None): self.eventcnt += 1 source.addevent(e); return e def setstathz(self, val, cpu): self.stathz = int(val) cpu = int(cpu) try: ticks = self.ticks[cpu] except: self.ticks[cpu] = 0 self.ticks[cpu] += 1 def checkstamp(self, timestamp): timestamp = int(timestamp) if (self.timestamp_f == None): self.timestamp_f = timestamp; if (self.timestamp_l != None and timestamp -2048> self.timestamp_l): return (0) self.timestamp_l = timestamp; return (1) def makeid(self, group, id, type): tag = group + id if (tag in self.taghash): return self.taghash[tag] if (type == "counter"): source = Counter(group, id) else: source = EventSource(group, id) sources.append(source) self.taghash[tag] = source return (source) def findid(self, id): for source in sources: if (source.name == id): return source return (None) def timespan(self): return (self.timestamp_f - self.timestamp_l); def ticksps(self): oneghz = 1000000000 # Use user supplied clock first if (clockfreq != None): return int(clockfreq * oneghz) # Check for a discovered clock if (self.stathz != 0): return (self.timespan() / self.ticks[0]) * int(self.stathz) # Pretend we have a 1ns clock - print "WARNING: No clock discovered and no frequency ", - print "specified via the command line." - print "Using fake 1ghz clock" + print("WARNING: No clock discovered and no frequency ", end=' ') + print("specified via the command line.") + print("Using fake 1ghz clock") return (oneghz); def fixup(self): for source in sources: e = PadEvent(source, -1, self.timestamp_l) source.addevent(e) e = PadEvent(source, -1, self.timestamp_f, last=1) source.addlastevent(e) source.fixup() sources.sort() class SchedNames(Canvas): def __init__(self, master, display): self.display = display self.parent = master self.bdheight = master.bdheight self.items = {} self.ysize = 0 self.lines = [] Canvas.__init__(self, master, width=120, height=display["height"], bg='grey', scrollregion=(0, 0, 50, 100)) def moveline(self, cur_y, y): for line in self.lines: (x0, y0, x1, y1) = self.coords(line) if (cur_y != y0): continue self.move(line, 0, y) return def draw(self): status.startup("Drawing names") ypos = 0 self.configure(scrollregion=(0, 0, self["width"], self.display.ysize())) for source in sources: l = self.create_line(0, ypos, self["width"], ypos, width=1, fill="black", tags=("all","sources")) self.lines.append(l) ypos += self.bdheight ypos += source.ysize() t = source.drawname(self, ypos) self.items[t] = source ypos += self.bdheight self.ysize = ypos self.create_line(0, ypos, self["width"], ypos, width=1, fill="black", tags=("all",)) self.bind("", self.master.mousepress); self.bind("", self.master.mousepressright); self.bind("", self.master.mouserelease); self.bind("", self.master.mousemotion); def updatescroll(self): self.configure(scrollregion=(0, 0, self["width"], self.display.ysize())) class SchedDisplay(Canvas): def __init__(self, master): self.ratio = 1 self.parent = master self.bdheight = master.bdheight self.items = {} self.lines = [] Canvas.__init__(self, master, width=800, height=500, bg='grey', scrollregion=(0, 0, 800, 500)) def prepare(self): # # Compute a ratio to ensure that the file's timespan fits into # 2^31. Although python may handle larger values for X # values, the Tk internals do not. # self.ratio = (ktrfile.timespan() - 1) / 2**31 + 1 def draw(self): ypos = 0 xsize = self.xsize() for source in sources: status.startup("Drawing " + source.name) l = self.create_line(0, ypos, xsize, ypos, width=1, fill="black", tags=("all",)) self.lines.append(l) ypos += self.bdheight ypos += source.ysize() source.draw(self, ypos) ypos += self.bdheight self.tag_raise("point", "state") self.tag_lower("cpubg", ALL) self.create_line(0, ypos, xsize, ypos, width=1, fill="black", tags=("lines",)) self.tag_bind("event", "", self.mouseenter) self.tag_bind("event", "", self.mouseexit) self.bind("", self.mousepress) self.bind("", self.master.mousepressright); self.bind("", self.wheelup) self.bind("", self.wheeldown) self.bind("", self.master.mouserelease); self.bind("", self.master.mousemotion); def moveline(self, cur_y, y): for line in self.lines: (x0, y0, x1, y1) = self.coords(line) if (cur_y != y0): continue self.move(line, 0, y) return def mouseenter(self, event): item, = self.find_withtag(CURRENT) self.items[item].mouseenter(self) def mouseexit(self, event): item, = self.find_withtag(CURRENT) self.items[item].mouseexit(self) def mousepress(self, event): # Find out what's beneath us items = self.find_withtag(CURRENT) if (len(items) == 0): self.master.mousepress(event) return # Only grab mouse presses for things with event tags. item = items[0] tags = self.gettags(item) for tag in tags: if (tag == "event"): self.items[item].mousepress(self) return # Leave the rest to the master window self.master.mousepress(event) def wheeldown(self, event): self.parent.display_yview("scroll", 1, "units") def wheelup(self, event): self.parent.display_yview("scroll", -1, "units") def xsize(self): return ((ktrfile.timespan() / self.ratio) + (X_BORDER * 2)) def ysize(self): ysize = 0 for source in sources: if (source.hidden == 1): continue ysize += self.parent.sourcesize(source) return ysize def scaleset(self, ratio): if (ktrfile == None): return oldratio = self.ratio xstart, xend = self.xview() midpoint = xstart + ((xend - xstart) / 2) self.ratio = ratio self.updatescroll() self.scale(ALL, 0, 0, float(oldratio) / ratio, 1) xstart, xend = self.xview() xsize = (xend - xstart) / 2 self.xview_moveto(midpoint - xsize) def updatescroll(self): self.configure(scrollregion=(0, 0, self.xsize(), self.ysize())) def scaleget(self): return self.ratio def getcolor(self, tag): return self.itemcget(tag, "fill") def getstate(self, tag): return self.itemcget(tag, "state") def setcolor(self, tag, color): self.itemconfigure(tag, state="normal", fill=color) def hide(self, tag): self.itemconfigure(tag, state="hidden") class GraphMenu(Frame): def __init__(self, master): Frame.__init__(self, master, bd=2, relief=RAISED) self.conf = Menubutton(self, text="Configure") self.confmenu = Menu(self.conf, tearoff=0) self.confmenu.add_command(label="Event Colors", command=self.econf) self.confmenu.add_command(label="CPU Colors", command=self.cconf) self.confmenu.add_command(label="Source Configure", command=self.sconf) self.conf["menu"] = self.confmenu self.conf.pack(side=LEFT) def econf(self): ColorConfigure(eventcolors, "Event Display Configuration") def cconf(self): ColorConfigure(cpucolors, "CPU Background Colors") def sconf(self): SourceConfigure() class SchedGraph(Frame): def __init__(self, master): Frame.__init__(self, master) self.menu = None self.names = None self.display = None self.scale = None self.status = None self.bdheight = Y_BORDER self.clicksource = None self.lastsource = None self.pack(expand=1, fill="both") self.buildwidgets() self.layout() self.bind_all("", self.quitcb) def quitcb(self, event): self.quit() def buildwidgets(self): global status self.menu = GraphMenu(self) self.display = SchedDisplay(self) self.names = SchedNames(self, self.display) self.scale = Scaler(self, self.display) status = self.status = Status(self) self.scrollY = Scrollbar(self, orient="vertical", command=self.display_yview) self.display.scrollX = Scrollbar(self, orient="horizontal", command=self.display.xview) self.display["xscrollcommand"] = self.display.scrollX.set self.display["yscrollcommand"] = self.scrollY.set self.names["yscrollcommand"] = self.scrollY.set def layout(self): self.columnconfigure(1, weight=1) self.rowconfigure(1, weight=1) self.menu.grid(row=0, column=0, columnspan=3, sticky=E+W) self.names.grid(row=1, column=0, sticky=N+S) self.display.grid(row=1, column=1, sticky=W+E+N+S) self.scrollY.grid(row=1, column=2, sticky=N+S) self.display.scrollX.grid(row=2, column=0, columnspan=2, sticky=E+W) self.scale.grid(row=3, column=0, columnspan=3, sticky=E+W) self.status.grid(row=4, column=0, columnspan=3, sticky=E+W) def draw(self): self.master.update() self.display.prepare() self.names.draw() self.display.draw() self.status.startup("") # # Configure scale related values # scalemax = ktrfile.timespan() / int(self.display["width"]) width = int(root.geometry().split('x')[0]) self.constwidth = width - int(self.display["width"]) self.scale.setmax(scalemax) self.scale.set(scalemax) self.display.xview_moveto(0) self.bind("", self.resize) def mousepress(self, event): self.clicksource = self.sourceat(event.y) def mousepressright(self, event): source = self.sourceat(event.y) if (source == None): return SourceContext(event, source) def mouserelease(self, event): if (self.clicksource == None): return newsource = self.sourceat(event.y) if (self.clicksource != newsource): self.sourceswap(self.clicksource, newsource) self.clicksource = None self.lastsource = None def mousemotion(self, event): if (self.clicksource == None): return newsource = self.sourceat(event.y) # # If we get a None source they moved off the page. # swapsource() can't handle moving multiple items so just # pretend we never clicked on anything to begin with so the # user can't mouseover a non-contiguous area. # if (newsource == None): self.clicksource = None self.lastsource = None return if (newsource == self.lastsource): return; self.lastsource = newsource if (newsource != self.clicksource): self.sourceswap(self.clicksource, newsource) # These are here because this object controls layout def sourcestart(self, source): return source.y - self.bdheight - source.ysize() def sourceend(self, source): return source.y + self.bdheight def sourcesize(self, source): return (self.bdheight * 2) + source.ysize() def sourceswap(self, source1, source2): # Sort so we always know which one is on top. if (source2.y < source1.y): swap = source1 source1 = source2 source2 = swap # Only swap adjacent sources if (self.sourceend(source1) != self.sourcestart(source2)): return # Compute start coordinates and target coordinates y1 = self.sourcestart(source1) y2 = self.sourcestart(source2) y1targ = y1 + self.sourcesize(source2) y2targ = y1 # # If the sizes are not equal, adjust the start of the lower # source to account for the lost/gained space. # if (source1.ysize() != source2.ysize()): diff = source2.ysize() - source1.ysize() self.names.moveline(y2, diff); self.display.moveline(y2, diff) source1.move(self.display, 0, y1targ - y1) source2.move(self.display, 0, y2targ - y2) source1.movename(self.names, 0, y1targ - y1) source2.movename(self.names, 0, y2targ - y2) def sourcepicky(self, source): if (source.hidden == 0): return self.sourcestart(source) # Revert to group based sort sources.sort() prev = None for s in sources: if (s == source): break if (s.hidden == 0): prev = s if (prev == None): newy = 0 else: newy = self.sourcestart(prev) + self.sourcesize(prev) return newy def sourceshow(self, source): if (source.hidden == 0): return; newy = self.sourcepicky(source) off = newy - self.sourcestart(source) self.sourceshiftall(newy-1, self.sourcesize(source)) self.sourceshift(source, off) source.hidden = 0 # # Optimized source show of multiple entries that only moves each # existing entry once. Doing sourceshow() iteratively is too # expensive due to python's canvas.move(). # def sourceshowlist(self, srclist): srclist.sort(cmp=source_cmp_start) startsize = [] for source in srclist: if (source.hidden == 0): srclist.remove(source) startsize.append((self.sourcepicky(source), self.sourcesize(source))) sources.sort(cmp=source_cmp_start, reverse=True) self.status.startup("Updating display..."); for source in sources: if (source.hidden == 1): continue nstart = self.sourcestart(source) size = 0 for hidden in startsize: (start, sz) = hidden if (start <= nstart or start+sz <= nstart): size += sz self.sourceshift(source, size) idx = 0 size = 0 for source in srclist: (newy, sz) = startsize[idx] off = (newy + size) - self.sourcestart(source) self.sourceshift(source, off) source.hidden = 0 size += sz idx += 1 self.updatescroll() self.status.set("") # # Optimized source hide of multiple entries that only moves each # remaining entry once. Doing sourcehide() iteratively is too # expensive due to python's canvas.move(). # def sourcehidelist(self, srclist): srclist.sort(cmp=source_cmp_start) sources.sort(cmp=source_cmp_start) startsize = [] off = len(sources) * 100 self.status.startup("Updating display..."); for source in srclist: if (source.hidden == 1): srclist.remove(source) # # Remember our old position so we can sort things # below us when we're done. # startsize.append((self.sourcestart(source), self.sourcesize(source))) self.sourceshift(source, off) source.hidden = 1 idx = 0 size = 0 for hidden in startsize: (start, sz) = hidden size += sz if (idx + 1 < len(startsize)): (stop, sz) = startsize[idx+1] else: stop = self.display.ysize() idx += 1 for source in sources: nstart = self.sourcestart(source) if (nstart < start or source.hidden == 1): continue if (nstart >= stop): break; self.sourceshift(source, -size) self.updatescroll() self.status.set("") def sourcehide(self, source): if (source.hidden == 1): return; # Move it out of the visible area off = len(sources) * 100 start = self.sourcestart(source) self.sourceshift(source, off) self.sourceshiftall(start, -self.sourcesize(source)) source.hidden = 1 def sourceshift(self, source, off): start = self.sourcestart(source) source.move(self.display, 0, off) source.movename(self.names, 0, off) self.names.moveline(start, off); self.display.moveline(start, off) # # We update the idle tasks to shrink the dirtied area so # it does not always include the entire screen. # self.names.update_idletasks() self.display.update_idletasks() def sourceshiftall(self, start, off): self.status.startup("Updating display..."); for source in sources: nstart = self.sourcestart(source) if (nstart < start): continue; self.sourceshift(source, off) self.updatescroll() self.status.set("") def sourceat(self, ypos): (start, end) = self.names.yview() starty = start * float(self.names.ysize) ypos += starty for source in sources: if (source.hidden == 1): continue; yend = self.sourceend(source) ystart = self.sourcestart(source) if (ypos >= ystart and ypos <= yend): return source return None def display_yview(self, *args): self.names.yview(*args) self.display.yview(*args) def resize(self, *args): width = int(root.geometry().split('x')[0]) scalemax = ktrfile.timespan() / (width - self.constwidth) self.scale.setmax(scalemax) def updatescroll(self): self.names.updatescroll() self.display.updatescroll() def setcolor(self, tag, color): self.display.setcolor(tag, color) def hide(self, tag): self.display.hide(tag) def getcolor(self, tag): return self.display.getcolor(tag) def getstate(self, tag): return self.display.getstate(tag) if (len(sys.argv) != 2 and len(sys.argv) != 3): - print "usage:", sys.argv[0], " [clock freq in ghz]" + print("usage:", sys.argv[0], " [clock freq in ghz]") sys.exit(1) if (len(sys.argv) > 2): clockfreq = float(sys.argv[2]) root = Tk() root.title("SchedGraph") colormap = Colormap(eventcolors) cpucolormap = Colormap(cpucolors) graph = SchedGraph(root) ktrfile = KTRFile(sys.argv[1]) graph.draw() root.mainloop() Index: head/tools/tools/shlib-compat/shlib-compat.py =================================================================== --- head/tools/tools/shlib-compat/shlib-compat.py (revision 291040) +++ head/tools/tools/shlib-compat/shlib-compat.py (revision 291041) @@ -1,1156 +1,1157 @@ #!/usr/bin/env python #- # Copyright (c) 2010 Gleb Kurtsou # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ +from __future__ import print_function import os import sys import re import optparse class Config(object): version = '0.1' # controlled by user verbose = 0 dump = False no_dump = False version_filter = None symbol_filter = None alias_prefixes = [] # misc opts objdump = 'objdump' dwarfdump = 'dwarfdump' # debug cmpcache_enabled = True dwarfcache_enabled = True w_alias = True w_cached = False w_symbol = True class FileConfig(object): filename = None out = sys.stdout def init(self, outname): if outname and outname != '-': self.out = open(outname, "w") origfile = FileConfig() newfile = FileConfig() exclude_sym_default = [ '^__bss_start$', '^_edata$', '^_end$', '^_fini$', '^_init$', ] @classmethod def init(cls): cls.version_filter = StrFilter() cls.symbol_filter = StrFilter() class App(object): result_code = 0 def warn(cond, msg): if cond: - print >> sys.stderr, "WARN: " + msg + print("WARN: " + msg, file=sys.stderr) # {{{ misc class StrFilter(object): def __init__(self): self.exclude = [] self.include = [] def compile(self): self.re_exclude = [ re.compile(x) for x in self.exclude ] self.re_include = [ re.compile(x) for x in self.include ] def match(self, s): if len(self.re_include): matched = False for r in self.re_include: if r.match(s): matched = True break if not matched: return False for r in self.re_exclude: if r.match(s): return False return True class Cache(object): class CacheStats(object): def __init__(self): self.hit = 0 self.miss = 0 def show(self, name): total = self.hit + self.miss if total == 0: ratio = '(undef)' else: ratio = '%f' % (self.hit/float(total)) return '%s cache stats: hit: %d; miss: %d; ratio: %s' % \ (name, self.hit, self.miss, ratio) def __init__(self, enabled=True, stats=None): self.enabled = enabled self.items = {} if stats == None: self.stats = Cache.CacheStats() else: self.stats = stats def get(self, id): if self.enabled and id in self.items: self.stats.hit += 1 return self.items[id] else: self.stats.miss += 1 return None def put(self, id, obj): if self.enabled: if id in self.items and obj is not self.items[id]: #raise ValueError("Item is already cached: %d (%s, %s)" % # (id, self.items[id], obj)) warn(Config.w_cached, "Item is already cached: %d (%s, %s)" % \ (id, self.items[id], obj)) self.items[id] = obj def replace(self, id, obj): if self.enabled: assert id in self.items self.items[id] = obj class ListDiff(object): def __init__(self, orig, new): self.orig = set(orig) self.new = set(new) self.common = self.orig & self.new self.added = self.new - self.common self.removed = self.orig - self.common class PrettyPrinter(object): def __init__(self): self.stack = [] def run_nested(self, obj): ex = obj._pp_ex(self) self.stack.append(ex) def run(self, obj): self._result = obj._pp(self) return self._result def nested(self): return sorted(set(self.stack)) def result(self): return self._result; # }}} #{{{ symbols and version maps class Symbol(object): def __init__(self, name, offset, version, lib): self.name = name self.offset = offset self.version = version self.lib = lib self.definition = None @property def name_ver(self): return self.name + '@' + self.version def __repr__(self): return "Symbol(%s, 0x%x, %s)" % (self.name, self.offset, self.version) class CommonSymbol(object): def __init__(self, origsym, newsym): if origsym.name != newsym.name or origsym.version != newsym.version: raise RuntimeError("Symbols have different names: %s", [origsym, newsym]) self.origsym = origsym self.newsym = newsym self.name = newsym.name self.version = newsym.version def __repr__(self): return "CommonSymbol(%s, %s)" % (self.name, self.version) class SymbolAlias(object): def __init__(self, alias, prefix, offset): assert alias.startswith(prefix) self.alias = alias self.name = alias[len(prefix):] self.offset = offset def __repr__(self): return "SymbolAlias(%s, 0x%x)" % (self.alias, self.offset) class VersionMap(object): def __init__(self, name): self.name = name self.symbols = {} def append(self, symbol): if (symbol.name in self.symbols): raise ValueError("Symbol is already defined %s@%s" % (symbol.name, self.name)) self.symbols[symbol.name] = symbol def names(self): return self.symbols.keys() def __repr__(self): return repr(self.symbols.values()) # }}} # {{{ types and definitions class Def(object): _is_alias = False def __init__(self, id, name, **kwargs): self.id = id self.name = name self.attrs = kwargs def __getattr__(self, attr): if attr not in self.attrs: raise AttributeError('%s in %s' % (attr, str(self))) return self.attrs[attr] def _name_opt(self, default=''): if not self.name: return default return self.name def _alias(self): if self._is_alias: return self.type._alias() return self def __cmp__(self, other): # TODO assert 'self' and 'other' belong to different libraries #print 'cmp defs: %s, %s' % (self, other) a = self._alias() try: b = other._alias() except AttributeError: return 1 r = cmp(a.__class__, b.__class__) if r == 0: if a.id != 0 and b.id != 0: ind = (long(a.id) << 32) + b.id r = Dwarf.cmpcache.get(ind) if r != None: return r else: ind = 0 r = cmp(a.attrs, b.attrs) if ind != 0: Dwarf.cmpcache.put(ind, r) else: r = 0 #raise RuntimeError('Comparing different classes: %s, %s' % # (a.__class__.__name__, b.__class__.__name__)) return r def __repr__(self): p = [] if hasattr(self, 'name'): p.append("name=%s" % self.name) for (k, v) in self.attrs.items(): if isinstance(v, Def): v = v.__class__.__name__ + '(...)' p.append("%s=%s" % (k, v)) return self.__class__.__name__ + '(' + ', '.join(p) + ')' def _mapval(self, param, vals): if param not in vals.keys(): raise NotImplementedError("Invalid value '%s': %s" % (param, str(self))) return vals[param] def _pp_ex(self, pp): raise NotImplementedError('Extended pretty print not implemeted: %s' % str(self)) def _pp(self, pp): raise NotImplementedError('Pretty print not implemeted: %s' % str(self)) class AnonymousDef(Def): def __init__(self, id, **kwargs): Def.__init__(self, id, None, **kwargs) class Void(AnonymousDef): _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(Void, cls).__new__( cls, *args, **kwargs) return cls._instance def __init__(self): AnonymousDef.__init__(self, 0) def _pp(self, pp): return "void" class VarArgs(AnonymousDef): def _pp(self, pp): return "..." class PointerDef(AnonymousDef): def _pp(self, pp): t = pp.run(self.type) return "%s*" % (t,) class BaseTypeDef(Def): inttypes = ['DW_ATE_signed', 'DW_ATE_unsigned', 'DW_ATE_unsigned_char'] def _pp(self, pp): if self.encoding in self.inttypes: sign = '' if self.encoding == 'DW_ATE_signed' else 'u' bits = int(self.byte_size, 0) * 8 return '%sint%s_t' % (sign, bits) elif self.encoding == 'DW_ATE_signed_char' and int(self.byte_size, 0) == 1: return 'char'; elif self.encoding == 'DW_ATE_boolean' and int(self.byte_size, 0) == 1: return 'bool'; elif self.encoding == 'DW_ATE_float': return self._mapval(int(self.byte_size, 0), { 16: 'long double', 8: 'double', 4: 'float', }) raise NotImplementedError('Invalid encoding: %s' % self) class TypeAliasDef(Def): _is_alias = True def _pp(self, pp): alias = self._alias() # push typedef name if self.name and not alias.name: alias.name = 'T(%s)' % self.name # return type with modifiers return self.type._pp(pp) class EnumerationTypeDef(Def): def _pp(self, pp): return 'enum ' + self._name_opt('UNKNOWN') class ConstTypeDef(AnonymousDef): _is_alias = True def _pp(self, pp): return 'const ' + self.type._pp(pp) class VolatileTypeDef(AnonymousDef): _is_alias = True def _pp(self, pp): return 'volatile ' + self.type._pp(pp) class RestrictTypeDef(AnonymousDef): _is_alias = True def _pp(self, pp): return 'restrict ' + self.type._pp(pp) class ArrayDef(AnonymousDef): def _pp(self, pp): t = pp.run(self.type) assert len(self.subranges) == 1 try: sz = int(self.subranges[0].upper_bound) + 1 except ValueError: s = re.sub(r'\(.+\)', '', self.subranges[0].upper_bound) sz = int(s) + 1 return '%s[%s]' % (t, sz) class ArraySubrangeDef(AnonymousDef): pass class FunctionDef(Def): def _pp(self, pp): result = pp.run(self.result) if not self.params: params = "void" else: params = ', '.join([ pp.run(x) for x in self.params ]) return "%s %s(%s);" % (result, self.name, params) class FunctionTypeDef(Def): def _pp(self, pp): result = pp.run(self.result) if not self.params: params = "void" else: params = ', '.join([ pp.run(x) for x in self.params ]) return "F(%s, %s, (%s))" % (self._name_opt(), result, params) class ParameterDef(Def): def _pp(self, pp): t = pp.run(self.type) return "%s %s" % (t, self._name_opt()) class VariableDef(Def): def _pp(self, pp): t = pp.run(self.type) return "%s %s" % (t, self._name_opt()) # TODO class StructForwardDef(Def): pass class IncompleteDef(Def): def update(self, complete, cache=None): self.complete = complete complete.incomplete = self if cache != None: cached = cache.get(self.id) if cached != None and isinstance(cached, IncompleteDef): cache.replace(self.id, complete) class StructIncompleteDef(IncompleteDef): def _pp(self, pp): return "struct %s" % (self.name,) class UnionIncompleteDef(IncompleteDef): def _pp(self, pp): return "union %s" % (self.name,) class StructDef(Def): def _pp_ex(self, pp, suffix=';'): members = [ pp.run(x) for x in self.members ] return "struct %s { %s }%s" % \ (self._name_opt(), ' '.join(members), suffix) def _pp(self, pp): if self.name: pp.run_nested(self) return "struct %s" % (self.name,) else: return self._pp_ex(pp, suffix='') class UnionDef(Def): def _pp_ex(self, pp, suffix=';'): members = [ pp.run(x) for x in self.members ] return "union %s { %s }%s" % \ (self._name_opt(), ' '.join(members), suffix) def _pp(self, pp): if self.name: pp.run_nested(self) return "union %s" % (self.name,) else: return self._pp_ex(pp, suffix='') class MemberDef(Def): def _pp(self, pp): t = pp.run(self.type) if self.bit_size: bits = ":%s" % self.bit_size else: bits = "" return "%s %s%s;" % (t, self._name_opt(), bits) class Dwarf(object): cmpcache = Cache(enabled=Config.cmpcache_enabled) def __init__(self, dump): self.dump = dump def _build_optarg_type(self, praw): type = praw.optarg('type', Void()) if type != Void(): type = self.buildref(praw.unit, type) return type def build_subprogram(self, raw): if raw.optname == None: raw.setname('SUBPROGRAM_NONAME_' + raw.arg('low_pc')); params = [ self.build(x) for x in raw.nested ] result = self._build_optarg_type(raw) return FunctionDef(raw.id, raw.name, params=params, result=result) def build_variable(self, raw): type = self._build_optarg_type(raw) return VariableDef(raw.id, raw.optname, type=type) def build_subroutine_type(self, raw): params = [ self.build(x) for x in raw.nested ] result = self._build_optarg_type(raw) return FunctionTypeDef(raw.id, raw.optname, params=params, result=result) def build_formal_parameter(self, raw): type = self._build_optarg_type(raw) return ParameterDef(raw.id, raw.optname, type=type) def build_pointer_type(self, raw): type = self._build_optarg_type(raw) return PointerDef(raw.id, type=type) def build_member(self, raw): type = self.buildref(raw.unit, raw.arg('type')) return MemberDef(raw.id, raw.name, type=type, bit_size=raw.optarg('bit_size', None)) def build_structure_type(self, raw): incomplete = raw.unit.incomplete.get(raw.id) if incomplete == None: incomplete = StructIncompleteDef(raw.id, raw.optname) raw.unit.incomplete.put(raw.id, incomplete) else: return incomplete members = [ self.build(x) for x in raw.nested ] byte_size = raw.optarg('byte_size', None) if byte_size == None: obj = StructForwardDef(raw.id, raw.name, members=members, forcename=raw.name) obj = StructDef(raw.id, raw.optname, members=members, byte_size=byte_size) incomplete.update(obj, cache=raw.unit.cache) return obj def build_union_type(self, raw): incomplete = raw.unit.incomplete.get(raw.id) if incomplete == None: incomplete = UnionIncompleteDef(raw.id, raw.optname) raw.unit.incomplete.put(raw.id, incomplete) else: return incomplete members = [ self.build(x) for x in raw.nested ] byte_size = raw.optarg('byte_size', None) obj = UnionDef(raw.id, raw.optname, members=members, byte_size=byte_size) obj.incomplete = incomplete incomplete.complete = obj return obj def build_typedef(self, raw): type = self._build_optarg_type(raw) return TypeAliasDef(raw.id, raw.name, type=type) def build_const_type(self, raw): type = self._build_optarg_type(raw) return ConstTypeDef(raw.id, type=type) def build_volatile_type(self, raw): type = self._build_optarg_type(raw) return VolatileTypeDef(raw.id, type=type) def build_restrict_type(self, raw): type = self._build_optarg_type(raw) return RestrictTypeDef(raw.id, type=type) def build_enumeration_type(self, raw): # TODO handle DW_TAG_enumerator ??? return EnumerationTypeDef(raw.id, name=raw.optname, byte_size=raw.arg('byte_size')) def build_base_type(self, raw): return BaseTypeDef(raw.id, raw.optname, byte_size=raw.arg('byte_size'), encoding=raw.arg('encoding')) def build_array_type(self, raw): type = self.buildref(raw.unit, raw.arg('type')) subranges = [ self.build(x) for x in raw.nested ] return ArrayDef(raw.id, type=type, subranges=subranges) def build_subrange_type(self, raw): type = self.buildref(raw.unit, raw.arg('type')) return ArraySubrangeDef(raw.id, type=type, upper_bound=raw.optarg('upper_bound', 0)) def build_unspecified_parameters(self, raw): return VarArgs(raw.id) def _get_id(self, id): try: return int(id) except ValueError: if (id.startswith('<') and id.endswith('>')): return int(id[1:-1], 0) else: raise ValueError("Invalid dwarf id: %s" % id) def build(self, raw): obj = raw.unit.cache.get(raw.id) if obj != None: return obj builder_name = raw.tag.replace('DW_TAG_', 'build_') try: builder = getattr(self, builder_name) except AttributeError: raise AttributeError("Unknown dwarf tag: %s" % raw) obj = builder(raw) raw.unit.cache.put(obj.id, obj) return obj def buildref(self, unit, id): id = self._get_id(id) raw = unit.tags[id] obj = self.build(raw) return obj # }}} class Shlib(object): def __init__(self, libfile): self.libfile = libfile self.versions = {} self.alias_syms = {} def parse_objdump(self): objdump = ObjdumpParser(self.libfile) objdump.run() for p in objdump.dynamic_symbols: vername = p['ver'] if vername.startswith('(') and vername.endswith(')'): vername = vername[1:-1] if not Config.version_filter.match(vername): continue if not Config.symbol_filter.match(p['symbol']): continue sym = Symbol(p['symbol'], p['offset'], vername, self) if vername not in self.versions: self.versions[vername] = VersionMap(vername) self.versions[vername].append(sym) if Config.alias_prefixes: self.local_offsetmap = objdump.local_offsetmap for p in objdump.local_symbols: for prefix in Config.alias_prefixes: if not p['symbol'].startswith(prefix): continue alias = SymbolAlias(p['symbol'], prefix, p['offset']) if alias.name in self.alias_syms: prevalias = self.alias_syms[alias.name] if alias.name != prevalias.name or \ alias.offset != prevalias.offset: warn(Config.w_alias, "Symbol alias is " \ "already defined: %s: %s at %08x -- %s at %08x" % \ (alias.alias, alias.name, alias.offset, prevalias.name, prevalias.offset)) self.alias_syms[alias.name] = alias def parse_dwarfdump(self): dwarfdump = DwarfdumpParser(self.libfile) def lookup(sym): raw = None try: raw = dwarfdump.offsetmap[sym.offset] except: try: localnames = self.local_offsetmap[sym.offset] localnames.sort(key=lambda x: -len(x)) for localname in localnames: if localname not in self.alias_syms: continue alias = self.alias_syms[localname] raw = dwarfdump.offsetmap[alias.offset] break except: pass return raw dwarfdump.run() dwarf = Dwarf(dwarfdump) for ver in self.versions.values(): for sym in ver.symbols.values(): raw = lookup(sym); if not raw: warn(Config.w_symbol, "Symbol %s (%s) not found at offset 0x%x" % \ (sym.name_ver, self.libfile, sym.offset)) continue if Config.verbose >= 3: - print "Parsing symbol %s (%s)" % (sym.name_ver, self.libfile) + print("Parsing symbol %s (%s)" % (sym.name_ver, self.libfile)) sym.definition = dwarf.build(raw) def parse(self): if not os.path.isfile(self.libfile): - print >> sys.stderr, ("No such file: %s" % self.libfile) + print("No such file: %s" % self.libfile, file=sys.stderr) sys.exit(1) self.parse_objdump() self.parse_dwarfdump() # {{{ parsers class Parser(object): def __init__(self, proc): self.proc = proc self.parser = self.parse_begin def run(self): fd = os.popen(self.proc, 'r') while True: line = fd.readline() if (not line): break line = line.strip() if (line): self.parser(line) err = fd.close() if err: - print >> sys.stderr, ("Execution failed: %s" % self.proc) + print("Execution failed: %s" % self.proc, file=sys.stderr) sys.exit(2) def parse_begin(self, line): print(line) class ObjdumpParser(Parser): re_header = re.compile('(?P\w*)\s*SYMBOL TABLE:') re_local_symbol = re.compile('(?P[0-9a-fA-F]+)\s+(?P\w+)\s+(?P\w+)\s+(?P
[^\s]+)\s+(?P[0-9a-fA-F]+)\s*(?P[^\s]*)') re_lame_symbol = re.compile('(?P[0-9a-fA-F]+)\s+(?P\w+)\s+\*[A-Z]+\*') re_dynamic_symbol = re.compile('(?P[0-9a-fA-F]+)\s+(?P\w+)\s+(?P\w+)\s+(?P
[^\s]+)\s+(?P[0-9a-fA-F]+)\s*(?P[^\s]*)\s*(?P[^\s]*)') def __init__(self, libfile): Parser.__init__(self, "%s -wtT %s" % (Config.objdump, libfile)) self.dynamic_symbols = [] self.local_symbols = [] self.local_offsetmap = {} def parse_begin(self, line): self.parse_header(line) def add_symbol(self, table, symbol, offsetmap = None): offset = int(symbol['offset'], 16); symbol['offset'] = offset if (offset == 0): return table.append(symbol) if offsetmap != None: if offset not in offsetmap: offsetmap[offset] = [symbol['symbol']] else: offsetmap[offset].append(symbol['symbol']) def parse_header(self, line): m = self.re_header.match(line) if (m): table = m.group('table') if (table == "DYNAMIC"): self.parser = self.parse_dynamic elif table == '': self.parser = self.parse_local else: raise ValueError("Invalid symbol table: %s" % table) return True return False def parse_local(self, line): if (self.parse_header(line)): return if (self.re_lame_symbol.match(line)): return m = self.re_local_symbol.match(line) if (not m): return #raise ValueError("Invalid symbol definition: %s" % line) p = m.groupdict() if (p['symbol'] and p['symbol'].find('@') == -1): self.add_symbol(self.local_symbols, p, self.local_offsetmap); def parse_dynamic(self, line): if (self.parse_header(line)): return if (self.re_lame_symbol.match(line)): return m = self.re_dynamic_symbol.match(line) if (not m): raise ValueError("Invalid symbol definition: %s" % line) p = m.groupdict() if (p['symbol'] and p['ver']): self.add_symbol(self.dynamic_symbols, p); class DwarfdumpParser(Parser): tagcache_stats = Cache.CacheStats() class Unit(object): def __init__(self): self.cache = Cache(enabled=Config.dwarfcache_enabled, stats=DwarfdumpParser.tagcache_stats) self.incomplete = Cache() self.tags = {} class Tag(object): def __init__(self, unit, data): self.unit = unit self.id = int(data['id'], 0) self.level = int(data['level']) self.tag = data['tag'] self.args = {} self.nested = [] @property def name(self): return self.arg('name') @property def optname(self): return self.optarg('name', None) def setname(self, name): self.args['DW_AT_name'] = name def arg(self, a): name = 'DW_AT_' + a try: return self.args[name] except KeyError: raise KeyError("Argument '%s' not found in %s: %s" % (name, self, self.args)) def optarg(self, a, default): try: return self.arg(a) except KeyError: return default def __repr__(self): return "Tag(%d, %d, %s)" % (self.level, self.id, self.tag) re_header = re.compile('<(?P\d+)><(?P[0xX0-9a-fA-F]+(?:\+(0[xX])?[0-9a-fA-F]+)?)><(?P\w+)>') re_argname = re.compile('(?P\w+)<') re_argunknown = re.compile('<[^<>]+>') skip_tags = set([ 'DW_TAG_lexical_block', 'DW_TAG_inlined_subroutine', 'DW_TAG_label', 'DW_TAG_variable', ]) external_tags = set([ 'DW_TAG_variable', ]) def __init__(self, libfile): Parser.__init__(self, "%s -di %s" % (Config.dwarfdump, libfile)) self.current_unit = None self.offsetmap = {} self.stack = [] def parse_begin(self, line): if line == '.debug_info': self.parser = self.parse_debuginfo else: raise ValueError("Invalid dwarfdump header: %s" % line) def parse_argvalue(self, args): assert args.startswith('<') i = 1 cnt = 1 while i < len(args) and args[i]: if args[i] == '<': cnt += 1 elif args[i] == '>': cnt -= 1 if cnt == 0: break i = i + 1 value = args[1:i] args = args[i+1:] return (args, value) def parse_arg(self, tag, args): m = self.re_argname.match(args) if not m: m = self.re_argunknown.match(args) if not m: raise ValueError("Invalid dwarfdump: couldn't parse arguments: %s" % args) args = args[len(m.group(0)):].lstrip() return args argname = m.group('arg') args = args[len(argname):] value = [] while len(args) > 0 and args.startswith('<'): (args, v) = self.parse_argvalue(args) value.append(v) args = args.lstrip() if len(value) == 1: value = value[0] tag.args[argname] = value return args def parse_debuginfo(self, line): m = self.re_header.match(line) if not m: raise ValueError("Invalid dwarfdump: %s" % line) if m.group('level') == '0': self.current_unit = DwarfdumpParser.Unit() return tag = DwarfdumpParser.Tag(self.current_unit, m.groupdict()) args = line[len(m.group(0)):].lstrip() while args: args = self.parse_arg(tag, args) tag.unit.tags[tag.id] = tag def parse_offset(tag): if 'DW_AT_low_pc' in tag.args: return int(tag.args['DW_AT_low_pc'], 16) elif 'DW_AT_location' in tag.args: location = tag.args['DW_AT_location'] if location.startswith('DW_OP_addr'): return int(location.replace('DW_OP_addr', ''), 16) return None offset = parse_offset(tag) if offset is not None and \ (tag.tag not in DwarfdumpParser.skip_tags or \ ('DW_AT_external' in tag.args and \ tag.tag in DwarfdumpParser.external_tags)): if offset in self.offsetmap: raise ValueError("Dwarf dump parse error: " + "symbol is aleady defined at offset 0x%x" % offset) self.offsetmap[offset] = tag if len(self.stack) > 0: prev = self.stack.pop() while prev.level >= tag.level and len(self.stack) > 0: prev = self.stack.pop() if prev.level < tag.level: assert prev.level == tag.level - 1 # TODO check DW_AT_sibling ??? if tag.tag not in DwarfdumpParser.skip_tags: prev.nested.append(tag) self.stack.append(prev) self.stack.append(tag) assert len(self.stack) == tag.level # }}} def list_str(l): l = [ str(x) for x in l ] l.sort() return ', '.join(l) def names_ver_str(vername, names): return list_str([ x + "@" + vername for x in names ]) def common_symbols(origlib, newlib): result = [] verdiff = ListDiff(origlib.versions.keys(), newlib.versions.keys()) if Config.verbose >= 1: - print 'Original versions: ', list_str(verdiff.orig) - print 'New versions: ', list_str(verdiff.new) + print('Original versions: ', list_str(verdiff.orig)) + print('New versions: ', list_str(verdiff.new)) for vername in verdiff.added: - print 'Added version: ', vername - print ' Added symbols: ', \ - names_ver_str(vername, newlib.versions[vername].names()) + print('Added version: ', vername) + print(' Added symbols: ', \ + names_ver_str(vername, newlib.versions[vername].names())) for vername in verdiff.removed: - print 'Removed version: ', vername - print ' Removed symbols: ', \ - names_ver_str(vername, origlib.versions[vername].names()) + print('Removed version: ', vername) + print(' Removed symbols: ', \ + names_ver_str(vername, origlib.versions[vername].names())) added = [] removed = [] for vername in verdiff.common: origver = origlib.versions[vername] newver = newlib.versions[vername] namediff = ListDiff(origver.names(), newver.names()) if namediff.added: added.append(names_ver_str(vername, namediff.added)) if namediff.removed: removed.append(names_ver_str(vername, namediff.removed)) commonver = VersionMap(vername) result.append(commonver) for n in namediff.common: sym = CommonSymbol(origver.symbols[n], newver.symbols[n]) commonver.append(sym) if added: - print 'Added symbols:' + print('Added symbols:') for i in added: - print ' ', i + print(' ', i) if removed: - print 'Removed symbols:' + print('Removed symbols:') for i in removed: - print ' ', i + print(' ', i) return result def cmp_symbols(commonver): for ver in commonver: names = ver.names(); names.sort() for symname in names: sym = ver.symbols[symname] missing = sym.origsym.definition is None or sym.newsym.definition is None match = not missing and sym.origsym.definition == sym.newsym.definition if not match: App.result_code = 1 if Config.verbose >= 1 or not match: if missing: - print '%s: missing definition' % \ - (sym.origsym.name_ver,) + print('%s: missing definition' % \ + (sym.origsym.name_ver,)) continue - print '%s: definitions %smatch' % \ - (sym.origsym.name_ver, "" if match else "mis") + print('%s: definitions %smatch' % \ + (sym.origsym.name_ver, "" if match else "mis")) if Config.dump or (not match and not Config.no_dump): for x in [(sym.origsym, Config.origfile), (sym.newsym, Config.newfile)]: xsym = x[0] xout = x[1].out if not xsym.definition: - print >> xout, '\n// Definition not found: %s %s' % \ - (xsym.name_ver, xsym.lib.libfile) + print('\n// Definition not found: %s %s' % \ + (xsym.name_ver, xsym.lib.libfile), file=xout) continue - print >> xout, '\n// Definitions mismatch: %s %s' % \ - (xsym.name_ver, xsym.lib.libfile) + print('\n// Definitions mismatch: %s %s' % \ + (xsym.name_ver, xsym.lib.libfile), file=xout) pp = PrettyPrinter() pp.run(xsym.definition) for i in pp.nested(): - print >> xout, i - print >> xout, pp.result() + print(i, file=xout) + print(pp.result(), file=xout) def dump_symbols(commonver): class SymbolDump(object): def __init__(self, io_conf): self.io_conf = io_conf self.pp = PrettyPrinter() self.res = [] def run(self, sym): r = self.pp.run(sym.definition) self.res.append('/* %s@%s */ %s' % (sym.name, sym.version, r)) def finish(self): - print >> self.io_conf.out, '\n// Symbol dump: version %s, library %s' % \ - (ver.name, self.io_conf.filename) + print('\n// Symbol dump: version %s, library %s' % \ + (ver.name, self.io_conf.filename), file=self.io_conf.out) for i in self.pp.nested(): - print >> self.io_conf.out, i - print >> self.io_conf.out, '' + print(i, file=self.io_conf.out) + print('', file=self.io_conf.out) for i in self.res: - print >> self.io_conf.out, i + print(i, file=self.io_conf.out) for ver in commonver: names = sorted(ver.names()); d_orig = SymbolDump(Config.origfile) d_new = SymbolDump(Config.newfile) for symname in names: sym = ver.symbols[symname] if not sym.origsym.definition or not sym.newsym.definition: # XXX warn(Config.w_symbol, 'Missing symbol definition: %s@%s' % \ (symname, ver.name)) continue d_orig.run(sym.origsym) d_new.run(sym.newsym) d_orig.finish() d_new.finish() if __name__ == '__main__': Config.init() parser = optparse.OptionParser(usage="usage: %prog origlib newlib", version="%prog " + Config.version) parser.add_option('-v', '--verbose', action='count', help="verbose mode, may be specified several times") parser.add_option('--alias-prefix', action='append', help="name prefix to try for symbol alias lookup", metavar="STR") parser.add_option('--dump', action='store_true', help="dump symbol definitions") parser.add_option('--no-dump', action='store_true', help="disable dump for mismatched symbols") parser.add_option('--out-orig', action='store', help="result output file for original library", metavar="ORIGFILE") parser.add_option('--out-new', action='store', help="result output file for new library", metavar="NEWFILE") parser.add_option('--dwarfdump', action='store', help="path to dwarfdump executable", metavar="DWARFDUMP") parser.add_option('--objdump', action='store', help="path to objdump executable", metavar="OBJDUMP") parser.add_option('--exclude-ver', action='append', metavar="RE") parser.add_option('--include-ver', action='append', metavar="RE") parser.add_option('--exclude-sym', action='append', metavar="RE") parser.add_option('--include-sym', action='append', metavar="RE") parser.add_option('--no-exclude-sym-default', action='store_true', help="don't exclude special symbols like _init, _end, __bss_start") for opt in ['alias', 'cached', 'symbol']: parser.add_option("--w-" + opt, action="store_true", dest="w_" + opt) parser.add_option("--w-no-" + opt, action="store_false", dest="w_" + opt) (opts, args) = parser.parse_args() if len(args) != 2: parser.print_help() sys.exit(-1) if opts.dwarfdump: Config.dwarfdump = opts.dwarfdump if opts.objdump: Config.objdump = opts.objdump if opts.out_orig: Config.origfile.init(opts.out_orig) if opts.out_new: Config.newfile.init(opts.out_new) if opts.no_dump: Config.dump = False Config.no_dump = True if opts.dump: Config.dump = True Config.no_dump = False Config.verbose = 1 if opts.verbose: Config.verbose = opts.verbose if opts.alias_prefix: Config.alias_prefixes = opts.alias_prefix Config.alias_prefixes.sort(key=lambda x: -len(x)) for (k, v) in ({ '_sym': Config.symbol_filter, '_ver': Config.version_filter }).items(): for a in [ 'exclude', 'include' ]: opt = getattr(opts, a + k) if opt: getattr(v, a).extend(opt) if not opts.no_exclude_sym_default: Config.symbol_filter.exclude.extend(Config.exclude_sym_default) Config.version_filter.compile() Config.symbol_filter.compile() for w in ['w_alias', 'w_cached', 'w_symbol']: if hasattr(opts, w): v = getattr(opts, w) if v != None: setattr(Config, w, v) (Config.origfile.filename, Config.newfile.filename) = (args[0], args[1]) origlib = Shlib(Config.origfile.filename) origlib.parse() newlib = Shlib(Config.newfile.filename) newlib.parse() commonver = common_symbols(origlib, newlib) if Config.dump: dump_symbols(commonver) cmp_symbols(commonver) if Config.verbose >= 4: - print Dwarf.cmpcache.stats.show('Cmp') - print DwarfdumpParser.tagcache_stats.show('Dwarf tag') + print(Dwarf.cmpcache.stats.show('Cmp')) + print(DwarfdumpParser.tagcache_stats.show('Dwarf tag')) sys.exit(App.result_code) Index: head/usr.bin/fortune/tools/do_uniq.py =================================================================== --- head/usr.bin/fortune/tools/do_uniq.py (revision 291040) +++ head/usr.bin/fortune/tools/do_uniq.py (revision 291041) @@ -1,68 +1,68 @@ #!/usr/local/bin/python # # $FreeBSD$ # # an aggressive little script for trimming duplicate cookies - +from __future__ import print_function import argparse import re wordlist = [ 'hadnot', 'donot', 'hadnt', 'dont', 'have', 'more', 'will', 'your', 'and', 'are', 'had', 'the', 'you', 'am', 'an', 'is', 'll', 've', 'we', 'a', 'd', 'i', 'm', 's', ] def hash(fortune): f = fortune f = f.lower() f = re.sub('[\W_]', '', f) for word in wordlist: f = re.sub(word, '', f) # f = re.sub('[aeiouy]', '', f) # f = re.sub('[^aeiouy]', '', f) f = f[:30] # f = f[-30:] return f def edit(datfile): dups = {} fortunes = [] fortune = "" with open(datfile, "r") as datfiledf: for line in datfiledf: if line == "%\n": key = hash(fortune) if key not in dups: dups[key] = [] dups[key].append(fortune) fortunes.append(fortune) fortune = "" else: fortune += line for key in list(dups.keys()): if len(dups[key]) == 1: del dups[key] with open(datfile + "~", "w") as o: for fortune in fortunes: key = hash(fortune) if key in dups: print('\n' * 50) for f in dups[key]: if f != fortune: print(f, '%') print(fortune, '%') if input("Remove last fortune? ") == 'y': del dups[key] continue o.write(fortune + "%\n") parser = argparse.ArgumentParser(description="trimming duplicate cookies") parser.add_argument("filename", type=str, nargs=1) args = parser.parse_args() edit(args.filename[0])