Changeset View
Changeset View
Standalone View
Standalone View
head/share/mk/meta2deps.py
Show All 19 Lines | |||||
'R' files read are what we really care about. | 'R' files read are what we really care about. | ||||
directories read, provide a clue to resolving | directories read, provide a clue to resolving | ||||
subsequent relative paths. That is if we cannot find | subsequent relative paths. That is if we cannot find | ||||
them relative to 'cwd', we check relative to the last | them relative to 'cwd', we check relative to the last | ||||
dir read. | dir read. | ||||
'W' files opened for write or read-write, | 'W' files opened for write or read-write, | ||||
for filemon V3 and earlier. | for filemon V3 and earlier. | ||||
'E' files executed. | 'E' files executed. | ||||
'L' files linked | 'L' files linked | ||||
'V' the filemon version, this record is used as a clue | 'V' the filemon version, this record is used as a clue | ||||
that we have reached the interesting bit. | that we have reached the interesting bit. | ||||
""" | """ | ||||
""" | """ | ||||
RCSid: | RCSid: | ||||
$FreeBSD$ | $FreeBSD$ | ||||
$Id: meta2deps.py,v 1.27 2017/05/24 00:04:04 sjg Exp $ | $Id: meta2deps.py,v 1.34 2020/10/02 03:11:17 sjg Exp $ | ||||
Copyright (c) 2011-2013, Juniper Networks, Inc. | Copyright (c) 2011-2020, Simon J. Gerraty | ||||
Copyright (c) 2011-2017, Juniper Networks, Inc. | |||||
All rights reserved. | All rights reserved. | ||||
Redistribution and use in source and binary forms, with or without | Redistribution and use in source and binary forms, with or without | ||||
modification, are permitted provided that the following conditions | modification, are permitted provided that the following conditions | ||||
are met: | are met: | ||||
1. Redistributions of source code must retain the above copyright | 1. Redistributions of source code must retain the above copyright | ||||
notice, this list of conditions and the following disclaimer. | notice, this list of conditions and the following disclaimer. | ||||
2. Redistributions in binary form must reproduce the above copyright | 2. Redistributions in binary form must reproduce the above copyright | ||||
notice, this list of conditions and the following disclaimer in the | notice, this list of conditions and the following disclaimer in the | ||||
documentation and/or other materials provided with the distribution. | documentation and/or other materials provided with the distribution. | ||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | ||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | ||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | ||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | ||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | ||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | ||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | ||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | ||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | ||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||||
""" | """ | ||||
import os, re, sys | import os, re, sys | ||||
def getv(dict, key, d=None): | def getv(dict, key, d=None): | ||||
"""Lookup key in dict and return value or the supplied default.""" | """Lookup key in dict and return value or the supplied default.""" | ||||
if key in dict: | if key in dict: | ||||
return dict[key] | return dict[key] | ||||
return d | return d | ||||
def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr): | def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr): | ||||
""" | """ | ||||
Return an absolute path, resolving via cwd or last_dir if needed. | Return an absolute path, resolving via cwd or last_dir if needed. | ||||
""" | """ | ||||
if path.endswith('/.'): | if path.endswith('/.'): | ||||
path = path[0:-2] | path = path[0:-2] | ||||
if len(path) > 0 and path[0] == '/': | if len(path) > 0 and path[0] == '/': | ||||
if os.path.exists(path): | |||||
return path | return path | ||||
if debug > 2: | |||||
print("skipping non-existent:", path, file=debug_out) | |||||
return None | |||||
if path == '.': | if path == '.': | ||||
return cwd | return cwd | ||||
if path.startswith('./'): | if path.startswith('./'): | ||||
return cwd + path[1:] | return cwd + path[1:] | ||||
if last_dir == cwd: | if last_dir == cwd: | ||||
last_dir = None | last_dir = None | ||||
for d in [last_dir, cwd]: | for d in [last_dir, cwd]: | ||||
if not d: | if not d: | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
def abspath(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr): | def abspath(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr): | ||||
""" | """ | ||||
Return an absolute path, resolving via cwd or last_dir if needed. | Return an absolute path, resolving via cwd or last_dir if needed. | ||||
this gets called a lot, so we try to avoid calling realpath. | this gets called a lot, so we try to avoid calling realpath. | ||||
""" | """ | ||||
rpath = resolve(path, cwd, last_dir, debug, debug_out) | rpath = resolve(path, cwd, last_dir, debug, debug_out) | ||||
if rpath: | if rpath: | ||||
path = rpath | path = rpath | ||||
elif len(path) > 0 and path[0] == '/': | |||||
return None | |||||
if (path.find('/') < 0 or | if (path.find('/') < 0 or | ||||
path.find('./') > 0 or | path.find('./') > 0 or | ||||
path.endswith('/..')): | path.endswith('/..')): | ||||
path = cleanpath(path) | path = cleanpath(path) | ||||
return path | return path | ||||
def sort_unique(list, cmp=None, key=None, reverse=False): | def sort_unique(list, cmp=None, key=None, reverse=False): | ||||
list.sort(cmp, key, reverse) | list.sort(cmp, key, reverse) | ||||
nl = [] | nl = [] | ||||
le = None | le = None | ||||
for e in list: | for e in list: | ||||
if e == le: | if e == le: | ||||
continue | continue | ||||
le = e | le = e | ||||
nl.append(e) | nl.append(e) | ||||
return nl | return nl | ||||
def add_trims(x): | def add_trims(x): | ||||
return ['/' + x + '/', | return ['/' + x + '/', | ||||
'/' + x, | '/' + x, | ||||
x + '/', | x + '/', | ||||
x] | x] | ||||
class MetaFile: | class MetaFile: | ||||
"""class to parse meta files generated by bmake.""" | """class to parse meta files generated by bmake.""" | ||||
conf = None | conf = None | ||||
dirdep_re = None | dirdep_re = None | ||||
host_target = None | host_target = None | ||||
srctops = [] | srctops = [] | ||||
objroots = [] | objroots = [] | ||||
excludes = [] | excludes = [] | ||||
seen = {} | seen = {} | ||||
obj_deps = [] | obj_deps = [] | ||||
src_deps = [] | src_deps = [] | ||||
file_deps = [] | file_deps = [] | ||||
def __init__(self, name, conf={}): | def __init__(self, name, conf={}): | ||||
"""if name is set we will parse it now. | """if name is set we will parse it now. | ||||
conf can have the follwing keys: | conf can have the follwing keys: | ||||
SRCTOPS list of tops of the src tree(s). | SRCTOPS list of tops of the src tree(s). | ||||
CURDIR the src directory 'bmake' was run from. | CURDIR the src directory 'bmake' was run from. | ||||
RELDIR the relative path from SRCTOP to CURDIR | RELDIR the relative path from SRCTOP to CURDIR | ||||
MACHINE the machine we built for. | MACHINE the machine we built for. | ||||
set to 'none' if we are not cross-building. | set to 'none' if we are not cross-building. | ||||
More specifically if machine cannot be deduced from objdirs. | More specifically if machine cannot be deduced from objdirs. | ||||
TARGET_SPEC | TARGET_SPEC | ||||
Sometimes MACHINE isn't enough. | Sometimes MACHINE isn't enough. | ||||
HOST_TARGET | HOST_TARGET | ||||
when we build for the pseudo machine 'host' | when we build for the pseudo machine 'host' | ||||
the object tree uses HOST_TARGET rather than MACHINE. | the object tree uses HOST_TARGET rather than MACHINE. | ||||
OBJROOTS a list of the common prefix for all obj dirs it might | OBJROOTS a list of the common prefix for all obj dirs it might | ||||
end in '/' or '-'. | end in '/' or '-'. | ||||
DPDEPS names an optional file to which per file dependencies | DPDEPS names an optional file to which per file dependencies | ||||
will be appended. | will be appended. | ||||
For example if 'some/path/foo.h' is read from SRCTOP | For example if 'some/path/foo.h' is read from SRCTOP | ||||
then 'DPDEPS_some/path/foo.h +=' "RELDIR" is output. | then 'DPDEPS_some/path/foo.h +=' "RELDIR" is output. | ||||
This can allow 'bmake' to learn all the dirs within | This can allow 'bmake' to learn all the dirs within | ||||
the tree that depend on 'foo.h' | the tree that depend on 'foo.h' | ||||
EXCLUDES | EXCLUDES | ||||
A list of paths to ignore. | A list of paths to ignore. | ||||
ccache(1) can otherwise be trouble. | ccache(1) can otherwise be trouble. | ||||
debug desired debug level | debug desired debug level | ||||
debug_out open file to send debug output to (sys.stderr) | debug_out open file to send debug output to (sys.stderr) | ||||
""" | """ | ||||
self.name = name | self.name = name | ||||
self.debug = getv(conf, 'debug', 0) | self.debug = getv(conf, 'debug', 0) | ||||
self.debug_out = getv(conf, 'debug_out', sys.stderr) | self.debug_out = getv(conf, 'debug_out', sys.stderr) | ||||
self.machine = getv(conf, 'MACHINE', '') | self.machine = getv(conf, 'MACHINE', '') | ||||
self.machine_arch = getv(conf, 'MACHINE_ARCH', '') | self.machine_arch = getv(conf, 'MACHINE_ARCH', '') | ||||
self.target_spec = getv(conf, 'TARGET_SPEC', '') | self.target_spec = getv(conf, 'TARGET_SPEC', '') | ||||
self.curdir = getv(conf, 'CURDIR') | self.curdir = getv(conf, 'CURDIR') | ||||
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | def __init__(self, name, conf={}): | ||||
self.try_parse() | self.try_parse() | ||||
def reset(self): | def reset(self): | ||||
"""reset state if we are being passed meta files from multiple directories.""" | """reset state if we are being passed meta files from multiple directories.""" | ||||
self.seen = {} | self.seen = {} | ||||
self.obj_deps = [] | self.obj_deps = [] | ||||
self.src_deps = [] | self.src_deps = [] | ||||
self.file_deps = [] | self.file_deps = [] | ||||
def dirdeps(self, sep='\n'): | def dirdeps(self, sep='\n'): | ||||
"""return DIRDEPS""" | """return DIRDEPS""" | ||||
return sep.strip() + sep.join(self.obj_deps) | return sep.strip() + sep.join(self.obj_deps) | ||||
def src_dirdeps(self, sep='\n'): | def src_dirdeps(self, sep='\n'): | ||||
"""return SRC_DIRDEPS""" | """return SRC_DIRDEPS""" | ||||
return sep.strip() + sep.join(self.src_deps) | return sep.strip() + sep.join(self.src_deps) | ||||
def file_depends(self, out=None): | def file_depends(self, out=None): | ||||
"""Append DPDEPS_${file} += ${RELDIR} | """Append DPDEPS_${file} += ${RELDIR} | ||||
for each file we saw, to the output file.""" | for each file we saw, to the output file.""" | ||||
if not self.reldir: | if not self.reldir: | ||||
return None | return None | ||||
for f in sort_unique(self.file_deps): | for f in sort_unique(self.file_deps): | ||||
print('DPDEPS_%s += %s' % (f, self.reldir), file=out) | print('DPDEPS_%s += %s' % (f, self.reldir), file=out) | ||||
# these entries provide for reverse DIRDEPS lookup | # these entries provide for reverse DIRDEPS lookup | ||||
for f in self.obj_deps: | for f in self.obj_deps: | ||||
print('DEPDIRS_%s += %s' % (f, self.reldir), file=out) | print('DEPDIRS_%s += %s' % (f, self.reldir), file=out) | ||||
def seenit(self, dir): | def seenit(self, dir): | ||||
"""rememer that we have seen dir.""" | """rememer that we have seen dir.""" | ||||
self.seen[dir] = 1 | self.seen[dir] = 1 | ||||
def add(self, list, data, clue=''): | def add(self, list, data, clue=''): | ||||
"""add data to list if it isn't already there.""" | """add data to list if it isn't already there.""" | ||||
if data not in list: | if data not in list: | ||||
list.append(data) | list.append(data) | ||||
if self.debug: | if self.debug: | ||||
print("%s: %sAdd: %s" % (self.name, clue, data), file=self.debug_out) | print("%s: %sAdd: %s" % (self.name, clue, data), file=self.debug_out) | ||||
def find_top(self, path, list): | def find_top(self, path, list): | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | class MetaFile: | ||||
def try_parse(self, name=None, file=None): | def try_parse(self, name=None, file=None): | ||||
"""give file and line number causing exception""" | """give file and line number causing exception""" | ||||
try: | try: | ||||
self.parse(name, file) | self.parse(name, file) | ||||
except: | except: | ||||
# give a useful clue | # give a useful clue | ||||
print('{}:{}: '.format(self.name, self.line), end=' ', file=sys.stderr) | print('{}:{}: '.format(self.name, self.line), end=' ', file=sys.stderr) | ||||
raise | raise | ||||
def parse(self, name=None, file=None): | def parse(self, name=None, file=None): | ||||
"""A meta file looks like: | """A meta file looks like: | ||||
# Meta data file "path" | # Meta data file "path" | ||||
CMD "command-line" | CMD "command-line" | ||||
CWD "cwd" | CWD "cwd" | ||||
TARGET "target" | TARGET "target" | ||||
-- command output -- | -- command output -- | ||||
-- filemon acquired metadata -- | -- filemon acquired metadata -- | ||||
# buildmon version 3 | # buildmon version 3 | ||||
V 3 | V 3 | ||||
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines | def parse(self, name=None, file=None): | ||||
if w[0] == 'F': | if w[0] == 'F': | ||||
npid = int(w[2]) | npid = int(w[2]) | ||||
pid_cwd[npid] = cwd | pid_cwd[npid] = cwd | ||||
pid_last_dir[npid] = cwd | pid_last_dir[npid] = cwd | ||||
last_pid = npid | last_pid = npid | ||||
continue | continue | ||||
elif w[0] == 'C': | elif w[0] == 'C': | ||||
cwd = abspath(w[2], cwd, None, self.debug, self.debug_out) | cwd = abspath(w[2], cwd, None, self.debug, self.debug_out) | ||||
if not cwd: | |||||
cwd = w[2] | |||||
if self.debug > 1: | |||||
print("missing cwd=", cwd, file=self.debug_out) | |||||
if cwd.endswith('/.'): | if cwd.endswith('/.'): | ||||
cwd = cwd[0:-2] | cwd = cwd[0:-2] | ||||
self.last_dir = pid_last_dir[pid] = cwd | self.last_dir = pid_last_dir[pid] = cwd | ||||
pid_cwd[pid] = cwd | pid_cwd[pid] = cwd | ||||
if self.debug > 1: | if self.debug > 1: | ||||
print("cwd=", cwd, file=self.debug_out) | print("cwd=", cwd, file=self.debug_out) | ||||
continue | continue | ||||
if w[2] in self.seen: | if w[2] in self.seen: | ||||
if self.debug > 2: | if self.debug > 2: | ||||
print("seen:", w[2], file=self.debug_out) | print("seen:", w[2], file=self.debug_out) | ||||
continue | continue | ||||
# file operations | # file operations | ||||
if w[0] in 'ML': | if w[0] in 'ML': | ||||
# these are special, tread src as read and | # these are special, tread src as read and | ||||
# target as write | # target as write | ||||
self.parse_path(w[1].strip("'"), cwd, 'R', w) | self.parse_path(w[2].strip("'"), cwd, 'R', w) | ||||
self.parse_path(w[2].strip("'"), cwd, 'W', w) | self.parse_path(w[3].strip("'"), cwd, 'W', w) | ||||
continue | continue | ||||
elif w[0] in 'ERWS': | elif w[0] in 'ERWS': | ||||
path = w[2] | path = w[2] | ||||
if path == '.': | |||||
continue | |||||
self.parse_path(path, cwd, w[0], w) | self.parse_path(path, cwd, w[0], w) | ||||
assert(version > 0) | |||||
if not file: | if not file: | ||||
f.close() | f.close() | ||||
def is_src(self, base, dir, rdir): | def is_src(self, base, dir, rdir): | ||||
"""is base in srctop""" | """is base in srctop""" | ||||
for dir in [dir,rdir]: | for dir in [dir,rdir]: | ||||
if not dir: | if not dir: | ||||
continue | continue | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | def parse_path(self, path, cwd, op=None, w=[]): | ||||
return | return | ||||
if os.path.isdir(path): | if os.path.isdir(path): | ||||
if op in 'RW': | if op in 'RW': | ||||
self.last_dir = path; | self.last_dir = path; | ||||
if self.debug > 1: | if self.debug > 1: | ||||
print("ldir=", self.last_dir, file=self.debug_out) | print("ldir=", self.last_dir, file=self.debug_out) | ||||
return | return | ||||
if op in 'ERW': | if op in 'ER': | ||||
# finally, we get down to it | # finally, we get down to it | ||||
if dir == self.cwd or dir == self.curdir: | if dir == self.cwd or dir == self.curdir: | ||||
return | return | ||||
if self.is_src(base, dir, rdir): | if self.is_src(base, dir, rdir): | ||||
self.seenit(w[2]) | self.seenit(w[2]) | ||||
if not rdir: | if not rdir: | ||||
return | return | ||||
Show All 11 Lines | def parse_path(self, path, cwd, op=None, w=[]): | ||||
if self.dpdeps and objroot.endswith('/stage/'): | if self.dpdeps and objroot.endswith('/stage/'): | ||||
sp = '/'.join(path.replace(objroot,'').split('/')[1:]) | sp = '/'.join(path.replace(objroot,'').split('/')[1:]) | ||||
self.add(self.file_deps, sp, 'file') | self.add(self.file_deps, sp, 'file') | ||||
else: | else: | ||||
# don't waste time looking again | # don't waste time looking again | ||||
self.seenit(w[2]) | self.seenit(w[2]) | ||||
self.seenit(dir) | self.seenit(dir) | ||||
def main(argv, klass=MetaFile, xopts='', xoptf=None): | def main(argv, klass=MetaFile, xopts='', xoptf=None): | ||||
"""Simple driver for class MetaFile. | """Simple driver for class MetaFile. | ||||
Usage: | Usage: | ||||
script [options] [key=value ...] "meta" ... | script [options] [key=value ...] "meta" ... | ||||
Options and key=value pairs contribute to the | Options and key=value pairs contribute to the | ||||
dictionary passed to MetaFile. | dictionary passed to MetaFile. | ||||
-S "SRCTOP" | -S "SRCTOP" | ||||
add "SRCTOP" to the "SRCTOPS" list. | add "SRCTOP" to the "SRCTOPS" list. | ||||
-C "CURDIR" | -C "CURDIR" | ||||
-O "OBJROOT" | -O "OBJROOT" | ||||
add "OBJROOT" to the "OBJROOTS" list. | add "OBJROOT" to the "OBJROOTS" list. | ||||
-m "MACHINE" | -m "MACHINE" | ||||
-a "MACHINE_ARCH" | -a "MACHINE_ARCH" | ||||
-H "HOST_TARGET" | -H "HOST_TARGET" | ||||
-D "DPDEPS" | -D "DPDEPS" | ||||
-d bumps debug level | -d bumps debug level | ||||
""" | """ | ||||
import getopt | import getopt | ||||
# import Psyco if we can | # import Psyco if we can | ||||
# it can speed things up quite a bit | # it can speed things up quite a bit | ||||
have_psyco = 0 | have_psyco = 0 | ||||
Show All 23 Lines | try: | ||||
objroot = os.environ['SB_OBJROOT'] | objroot = os.environ['SB_OBJROOT'] | ||||
if objroot: | if objroot: | ||||
conf['OBJROOTS'].append(objroot) | conf['OBJROOTS'].append(objroot) | ||||
except: | except: | ||||
pass | pass | ||||
debug = 0 | debug = 0 | ||||
output = True | output = True | ||||
opts, args = getopt.getopt(argv[1:], 'a:dS:C:O:R:m:D:H:qT:X:' + xopts) | opts, args = getopt.getopt(argv[1:], 'a:dS:C:O:R:m:D:H:qT:X:' + xopts) | ||||
for o, a in opts: | for o, a in opts: | ||||
if o == '-a': | if o == '-a': | ||||
conf['MACHINE_ARCH'] = a | conf['MACHINE_ARCH'] = a | ||||
elif o == '-d': | elif o == '-d': | ||||
debug += 1 | debug += 1 | ||||
elif o == '-q': | elif o == '-q': | ||||
output = False | output = False | ||||
▲ Show 20 Lines • Show All 88 Lines • Show Last 20 Lines |