diff --git a/tools/pkgbase/metalog_reader.lua b/tools/pkgbase/metalog_reader.lua index 72cfcf46c95b..3e8cf8461914 100644 --- a/tools/pkgbase/metalog_reader.lua +++ b/tools/pkgbase/metalog_reader.lua @@ -1,523 +1,529 @@ #!/usr/libexec/flua -- SPDX-License-Identifier: BSD-2-Clause-FreeBSD -- -- Copyright(c) 2020 The FreeBSD Foundation. -- -- Redistribution and use in source and binary forms, with or without -- modification, are permitted provided that the following conditions -- are met: -- 1. Redistributions of source code must retain the above copyright -- notice, this list of conditions and the following disclaimer. -- 2. Redistributions in binary form must reproduce the above copyright -- notice, this list of conditions and the following disclaimer in the -- documentation and/or other materials provided with the distribution. -- -- THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -- ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -- SUCH DAMAGE. -- $FreeBSD$ function main(args) if #args == 0 then usage() end local filename local printall, checkonly, pkgonly = #args == 1, false, false local dcount, dsize, fuid, fgid, fid = false, false, false, false, false local verbose = false local w_notagdirs = false local i = 1 while i <= #args do if args[i] == '-h' then usage(true) elseif args[i] == '-a' then printall = true elseif args[i] == '-c' then printall = false checkonly = true elseif args[i] == '-p' then printall = false pkgonly = true while i < #args do i = i+1 if args[i] == '-count' then dcount = true elseif args[i] == '-size' then dsize = true elseif args[i] == '-fsetuid' then fuid = true elseif args[i] == '-fsetgid' then fgid = true elseif args[i] == '-fsetid' then fid = true else i = i-1 break end end elseif args[i] == '-v' then verbose = true elseif args[i] == '-Wcheck-notagdir' then w_notagdirs = true elseif args[i]:match('^%-') then io.stderr:write('Unknown argument '..args[i]..'.\n') usage() else filename = args[i] end i = i+1 end if filename == nil then io.stderr:write('Missing filename.\n') usage() end local sess = Analysis_session(filename, verbose, w_notagdirs) + local errors if printall then io.write('--- PACKAGE REPORTS ---\n') io.write(sess.pkg_report_full()) io.write('--- LINTING REPORTS ---\n') - print_lints(sess) + errors = print_lints(sess) elseif checkonly then - print_lints(sess) + errors = print_lints(sess) elseif pkgonly then io.write(sess.pkg_report_simple(dcount, dsize, { fuid and sess.pkg_issetuid or nil, fgid and sess.pkg_issetgid or nil, fid and sess.pkg_issetid or nil })) else io.stderr:write('This text should not be displayed.') usage() end + + if errors then + return 1 + end end --- @param man boolean function usage(man) local sn = 'Usage: '..arg[0].. ' [-h] [-a | -c | -p [-count] [-size] [-f...]] [-W...] metalog-path \n' if man then io.write('\n') io.write(sn) io.write( [[ The script reads METALOG file created by pkgbase (make packages) and generates reports about the installed system and issues. It accepts an mtree file in a format that's returned by `mtree -c | mtree -C` Options: -a prints all scan results. this is the default option if no option is provided. -c lints the file and gives warnings/errors, including duplication and conflicting metadata -Wcheck-notagdir entries with dir type and no tags will be also included the first time they appear -p list all package names found in the file as exactly specified by `tags=package=...` -count display the number of files of the package -size display the size of the package -fsetgid only include packages with setgid files -fsetuid only include packages with setuid files -fsetid only include packages with setgid or setuid files -v verbose mode -h help page ]]) os.exit() else io.stderr:write(sn) os.exit(1) end end --- @param sess Analysis_session function print_lints(sess) local dupwarn, duperr = sess.dup_report() io.write(dupwarn) io.write(duperr) local inodewarn, inodeerr = sess.inode_report() io.write(inodewarn) io.write(inodeerr) + return #duperr > 0 or #inodeerr > 0 end --- @param t table function sortedPairs(t) local sortedk = {} for k in next, t do sortedk[#sortedk+1] = k end table.sort(sortedk) local i = 0 return function() i = i + 1 return sortedk[i], t[sortedk[i]] end end --- @param t table --- @param f function U> function table_map(t, f) local res = {} for k, v in pairs(t) do res[k] = f(v) end return res end --- @class MetalogRow -- a table contaning file's info, from a line content from METALOG file -- all fields in the table are strings -- sample output: -- { -- filename = ./usr/share/man/man3/inet6_rthdr_segments.3.gz -- lineno = 5 -- attrs = { -- gname = 'wheel' -- uname = 'root' -- mode = '0444' -- size = '1166' -- time = nil -- type = 'file' -- tags = 'package=clibs,debug' -- } -- } --- @param line string function MetalogRow(line, lineno) local res, attrs = {}, {} local filename, rest = line:match('^(%S+) (.+)$') -- mtree file has space escaped as '\\040', not affecting splitting -- string by space for attrpair in rest:gmatch('[^ ]+') do local k, v = attrpair:match('^(.-)=(.+)') attrs[k] = v end res.filename = filename res.linenum = lineno res.attrs = attrs return res end -- check if an array of MetalogRows are equivalent. if not, the first field -- that's different is returned secondly --- @param rows MetalogRow[] --- @param ignore_name boolean --- @param ignore_tags boolean function metalogrows_all_equal(rows, ignore_name, ignore_tags) local __eq = function(l, o) if not ignore_name and l.filename ~= o.filename then return false, 'filename' end -- ignoring linenum in METALOG file as it's not relavant for k in pairs(l.attrs) do if ignore_tags and k == 'tags' then goto continue end if l.attrs[k] ~= o.attrs[k] and o.attrs[k] ~= nil then return false, k end ::continue:: end return true end for _, v in ipairs(rows) do local bol, offby = __eq(v, rows[1]) if not bol then return false, offby end end return true end --- @param tagstr string function pkgname_from_tag(tagstr) local ext, pkgname, pkgend = '', '', '' for seg in tagstr:gmatch('[^,]+') do if seg:match('package=') then pkgname = seg:sub(9) elseif seg == 'development' or seg == 'profile' or seg == 'debug' or seg == 'docs' then pkgend = seg else ext = ext == '' and seg or ext..'-'..seg end end pkgname = pkgname ..(ext == '' and '' or '-'..ext) ..(pkgend == '' and '' or '-'..pkgend) return pkgname end --- @class Analysis_session --- @param metalog string --- @param verbose boolean --- @param w_notagdirs boolean turn on to also check directories function Analysis_session(metalog, verbose, w_notagdirs) local stage_root = {} local files = {} -- map -- set is map. if bool is true then elem exists local pkgs = {} -- map> ----- used to keep track of files not belonging to a pkg. not used so ----- it is commented with ----- -----local nopkg = {} -- set --- @public local swarn = {} --- @public local serrs = {} -- returns number of files in package and size of package -- nil is returned upon errors --- @param pkgname string local function pkg_size(pkgname) local filecount, sz = 0, 0 for filename in pairs(pkgs[pkgname]) do local rows = files[filename] -- normally, there should be only one row per filename -- if these rows are equal, there should be warning, but it -- does not affect size counting. if not, it is an error if #rows > 1 and not metalogrows_all_equal(rows) then return nil end local row = rows[1] if row.attrs.type == 'file' then sz = sz + tonumber(row.attrs.size) end filecount = filecount + 1 end return filecount, sz end --- @param pkgname string --- @param mode number local function pkg_ismode(pkgname, mode) for filename in pairs(pkgs[pkgname]) do for _, row in ipairs(files[filename]) do if tonumber(row.attrs.mode, 8) & mode ~= 0 then return true end end end return false end --- @param pkgname string --- @public local function pkg_issetuid(pkgname) return pkg_ismode(pkgname, 2048) end --- @param pkgname string --- @public local function pkg_issetgid(pkgname) return pkg_ismode(pkgname, 1024) end --- @param pkgname string --- @public local function pkg_issetid(pkgname) return pkg_issetuid(pkgname) or pkg_issetgid(pkgname) end -- sample return: -- { [*string]: { count=1, size=2, issetuid=true, issetgid=true } } local function pkg_report_helper_table() local res = {} for pkgname in pairs(pkgs) do res[pkgname] = {} res[pkgname].count, res[pkgname].size = pkg_size(pkgname) res[pkgname].issetuid = pkg_issetuid(pkgname) res[pkgname].issetgid = pkg_issetgid(pkgname) end return res end -- returns a string describing package scan report --- @public local function pkg_report_full() local sb = {} for pkgname, v in sortedPairs(pkg_report_helper_table()) do sb[#sb+1] = 'Package '..pkgname..':' if v.issetuid or v.issetgid then sb[#sb+1] = ''..table.concat({ v.issetuid and ' setuid' or '', v.issetgid and ' setgid' or '' }, '') end sb[#sb+1] = '\n number of files: '..(v.count or '?') ..'\n total size: '..(v.size or '?') sb[#sb+1] = '\n' end return table.concat(sb, '') end --- @param have_count boolean --- @param have_size boolean --- @param filters function[] --- @public -- returns a string describing package size report. -- sample: "mypackage 2 2048"* if both booleans are true local function pkg_report_simple(have_count, have_size, filters) filters = filters or {} local sb = {} for pkgname, v in sortedPairs(pkg_report_helper_table()) do local pred = true -- doing a foldl to all the function results with (and) for _, f in pairs(filters) do pred = pred and f(pkgname) end if pred then sb[#sb+1] = pkgname..table.concat({ have_count and (' '..(v.count or '?')) or '', have_size and (' '..(v.size or '?')) or ''}, '') ..'\n' end end return table.concat(sb, '') end -- returns a string describing duplicate file warnings, -- returns a string describing duplicate file errors --- @public local function dup_report() local warn, errs = {}, {} for filename, rows in sortedPairs(files) do if #rows == 1 then goto continue end local iseq, offby = metalogrows_all_equal(rows) if iseq then -- repeated line, just a warning warn[#warn+1] = 'warning: '..filename .. ' ' .. rows[1].attrs.type ..' repeated with same meta: line ' ..table.concat( table_map(rows, function(e) return e.linenum end), ',') warn[#warn+1] = '\n' elseif not metalogrows_all_equal(rows, false, true) then -- same filename (possibly different tags), different metadata, an error errs[#errs+1] = 'error: '..filename ..' exists in multiple locations and with different meta: line ' ..table.concat( table_map(rows, function(e) return e.linenum end), ',') ..'. off by "'..offby..'"' errs[#errs+1] = '\n' end ::continue:: end return table.concat(warn, ''), table.concat(errs, '') end -- returns a string describing warnings of found hard links -- returns a string describing errors of found hard links --- @public local function inode_report() -- obtain inodes of filenames local attributes = require('lfs').attributes local inm = {} -- map local unstatables = {} -- string[] for filename in pairs(files) do -- i only took the first row of a filename, -- and skip links and folders if files[filename][1].attrs.type ~= 'file' then goto continue end local fs = attributes(stage_root .. filename) if fs == nil then unstatables[#unstatables+1] = filename goto continue end local inode = fs.ino inm[inode] = inm[inode] or {} table.insert(inm[inode], filename) ::continue:: end local warn, errs = {}, {} for _, filenames in pairs(inm) do if #filenames == 1 then goto continue end -- i only took the first row of a filename local rows = table_map(filenames, function(e) return files[e][1] end) local iseq, offby = metalogrows_all_equal(rows, true, true) if not iseq then errs[#errs+1] = 'error: ' ..'entries point to the same inode but have different meta: ' ..table.concat(filenames, ',')..' in line ' ..table.concat( table_map(rows, function(e) return e.linenum end), ',') ..'. off by "'..offby..'"' errs[#errs+1] = '\n' end ::continue:: end if #unstatables > 0 then warn[#warn+1] = verbose and 'note: skipped checking inodes: '..table.concat(unstatables, ',')..'\n' or 'note: skipped checking inodes for '..#unstatables..' entries\n' end return table.concat(warn, ''), table.concat(errs, '') end -- The METALOG file is assumed to be at the top of the stage directory. stage_root = string.gsub(metalog, '/[^/]*$', '/') do local fp, errmsg, errcode = io.open(metalog, 'r') if fp == nil then io.stderr:write('cannot open '..metalog..': '..errmsg..': '..errcode..'\n') os.exit(1) end -- scan all lines and put file data into the dictionaries local firsttimes = {} -- set local lineno = 0 for line in fp:lines() do -----local isinpkg = false lineno = lineno + 1 -- skip lines beginning with # if line:match('^%s*#') then goto continue end -- skip blank lines if line:match('^%s*$') then goto continue end local data = MetalogRow(line, lineno) -- entries with dir and no tags... ignore for the first time if not w_notagdirs and data.attrs.tags == nil and data.attrs.type == 'dir' and not firsttimes[data.filename] then firsttimes[data.filename] = true goto continue end files[data.filename] = files[data.filename] or {} table.insert(files[data.filename], data) if data.attrs.tags ~= nil then pkgname = pkgname_from_tag(data.attrs.tags) pkgs[pkgname] = pkgs[pkgname] or {} pkgs[pkgname][data.filename] = true ------isinpkg = true end -----if not isinpkg then nopkg[data.filename] = true end ::continue:: end fp:close() end return { warn = swarn, errs = serrs, pkg_issetuid = pkg_issetuid, pkg_issetgid = pkg_issetgid, pkg_issetid = pkg_issetid, pkg_report_full = pkg_report_full, pkg_report_simple = pkg_report_simple, dup_report = dup_report, inode_report = inode_report } end -main(arg) +os.exit(main(arg))