diff --git a/Tools/scripts/port_conflicts_check.lua b/Tools/scripts/port_conflicts_check.lua index ed81ad008298..c5265af19a5a 100755 --- a/Tools/scripts/port_conflicts_check.lua +++ b/Tools/scripts/port_conflicts_check.lua @@ -1,492 +1,507 @@ #!/usr/libexec/flua --[[ SPDX-License-Identifier: BSD-2-Clause-FreeBSD Copyright (c) 2022 Stefan Esser Generate a list of existing and required CONFLICTS_INSTALL lines for all ports (limited to ports for which official packages are provided). This script depends on the ports-mgmt/pkg-provides port for the list of files installed by all pre-built packages for the architecture the script is run on. The script generates a list of ports by running "pkg provides ." and a mapping from package base name to origin via "pkg rquery '%n %o'". The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is only representative for the options configured for each port (i.e. if non-default options have been selected and registered, these may lead to a non-default list of conflicts). The script detects files used by more than one port, than lists by origin the existing definition and the list of package base names that have been detected to cause install conflicts followed by the list of duplicate files separated by a hash character "#". This script uses the "hidden" LUA interpreter in the FreeBSD base systems and does not need any port except "pkg-provides" to be run. The run-time on my system checking the ~32000 packages available for -CURRENT on amd64 is less than 250 seconds. Example output: # Port: games/sol # Files: bin/sol # < aisleriot gnome-games # > aisleriot portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol The output is per port (for all flavors of the port, if applicable), gives examples of conflicting files (mostly to understand whether different versions of a port could co-exist), the current CONFLICTS and CONFLICTS_INSTALL entries merged, and a suggested new entry. This information is followed by a portedit command line that should do the right thing for simple cases, but the result should always be checked before the resulting Makefile is committed. --]] require "lfs" ------------------------------------------------------------------- local file_pattern = "." local database = "/var/db/pkg/provides/provides.db" +local max_age = 1 * 24 * 3600 -- maximum age of database file in seconds ------------------------------------------------------------------- local function table_sorted_keys(t) local result = {} for k, _ in pairs(t) do result[#result + 1] = k end table.sort(result) return result end ------------------------------------------------------------------- local function table_sort_uniq(t) local result = {} if t then local last table.sort(t) for _, entry in ipairs(t) do if entry ~= last then last = entry result[#result + 1] = entry end end end return result end ------------------------------------------------------------------- local function fnmatch(name, pattern) local function fnsubst(s) s = string.gsub(s, "%%", "%%%%") s = string.gsub(s, "%+", "%%+") s = string.gsub(s, "%-", "%%-") s = string.gsub(s, "%.", "%%.") s = string.gsub(s, "%?", ".") s = string.gsub(s, "%*", ".*") return s end local rexpr = "" local left, middle, right while true do left, middle, right = string.match(pattern, "([^[]*)(%[[^]]+%])(.*)") if not left then break end rexpr = rexpr .. fnsubst(left) .. middle pattern = right end rexpr = "^" .. rexpr .. fnsubst(pattern) .. "$" return string.find(name, rexpr) end ------------------------------------------------------------------- local function fetch_pkgs_origins() local pkgs = {} local pipe = io.popen("pkg rquery '%n %o'") for line in pipe:lines() do local pkgbase, origin = string.match(line, "(%S+) (%S+)") pkgs[origin] = pkgbase end pipe:close() pipe = io.popen("pkg rquery '%n %o %At %Av'") for line in pipe:lines() do local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)") if tag == "flavor" then pkgs[origin] = nil pkgs[origin .. "@" .. value] = pkgbase end end pipe:close() return pkgs end ------------------------------------------------------------------- local BAD_FILE_PATTERN = { "^[^/]+$", "^lib/python[%d%.]+/site%-packages/examples/[^/]+$", "^lib/python[%d%.]+/site%-packages/samples/[^/]+$", "^lib/python[%d%.]+/site%-packages/test/[^/]+$", "^lib/python[%d%.]+/site%-packages/test_app/[^/]+$", "^lib/python[%d%.]+/site%-packages/tests/[^/]+$", "^lib/python[%d%.]+/site%-packages/tests/unit/[^/]+$", } local BAD_FILE_PKGS = {} local function check_bad_file(pkgbase, file) for _, pattern in ipairs(BAD_FILE_PATTERN) do if string.match(file, pattern) then BAD_FILE_PKGS[pkgbase] = BAD_FILE_PKGS[pkgbase] or {} table.insert(BAD_FILE_PKGS[pkgbase], file) break end end end ------------------------------------------------------------------- local function read_files(pattern) local files_table = {} + local now = os.time() + local modification_time = lfs.attributes(database, "modification") + if not modification_time then + print("# Aborting: package file database " .. database .. " does not exist.") + print("# Install the 'pkg-provides' package and add it as a module to 'pkg.conf'.") + print("# Then fetch the database with 'pkg update' or 'pkg provides -u'.") + os.exit(1) + end + if now - modification_time > max_age then + print("# Aborting: package file database " .. database) + print("# is older than " .. max_age .. " seconds.") + print("# Use 'pkg provides -u' to update the database.") + os.exit(2) + end local pipe = io.popen("locate -d " .. database .. " " .. pattern) if pipe then for line in pipe:lines() do local pkgbase, file = string.match(line, "([^*]+)%*([^*]+)") if file:sub(1, 11) == "/usr/local/" then file = file:sub(12) end check_bad_file(pkgbase, file) local t = files_table[file] or {} t[#t + 1] = pkgbase files_table[file] = t end pipe:close() end return files_table end ------------------------------------------------------------------- local DUPLICATE_FILE = {} local function fetch_pkg_pairs(pattern) local pkg_pairs = {} for file, pkgbases in pairs(read_files(pattern)) do if #pkgbases >= 2 then DUPLICATE_FILE[file] = true for i = 1, #pkgbases -1 do local pkg_i = pkgbases[i] for j = i + 1, #pkgbases do local pkg_j = pkgbases[j] if pkg_i ~= pkg_j then local p1 = pkg_pairs[pkg_i] or {} local p2 = p1[pkg_j] or {} p2[#p2 + 1] = file p1[pkg_j] = p2 pkg_pairs[pkg_i] = p1 end end end end end return pkg_pairs end ------------------------------------------------------------------- local function conflicts_delta(old, new) local old_seen = {} local changed for i = 1, #new do local matched for j = 1, #old do if new[i] == old[j] or fnmatch(new[i], old[j]) then new[i] = old[j] old_seen[j] = true matched = true break end end changed = changed or not matched end if not changed then for j = 1, #old do if not old_seen[j] then changed = true break end end end if changed then return table_sort_uniq(new) end end ------------------------------------------------------------------- local function fetch_port_conflicts(origin) local dir, flavor = origin:match("([^@]+)@?(.*)") if flavor ~= "" then flavor = " FLAVOR=" .. flavor end local seen = {} local portdir = "/usr/ports/" .. dir local pipe = io.popen("make -C " .. portdir .. flavor .. " -V CONFLICTS -V CONFLICTS_INSTALL 2>/dev/null") for line in pipe:lines() do for word in line:gmatch("(%S+)%s?") do seen[word] = true end end pipe:close() return table_sorted_keys(seen) end ------------------------------------------------------------------- local function conflicting_pkgs(conflicting) local pkgs = {} for origin, pkgbase in pairs(fetch_pkgs_origins()) do if conflicting[pkgbase] then pkgs[origin] = pkgbase end end return pkgs end ------------------------------------------------------------------- local function collect_conflicts(pkg_pairs) local pkgs = {} for pkg_i, p1 in pairs(pkg_pairs) do for pkg_j, _ in pairs(p1) do pkgs[pkg_i] = pkgs[pkg_i] or {} table.insert(pkgs[pkg_i], pkg_j) pkgs[pkg_j] = pkgs[pkg_j] or {} table.insert(pkgs[pkg_j], pkg_i) end end return pkgs end ------------------------------------------------------------------- local function split_origins(origin_list) local port_list = {} local flavors = {} local last_port for _, origin in ipairs(origin_list) do local port, flavor = string.match(origin, "([^@]+)@?(.*)") if port ~= last_port then port_list[#port_list + 1] = port if flavor ~= "" then flavors[port] = {flavor} end else table.insert(flavors[port], flavor) end last_port = port end return port_list, flavors end ------------------------------------------------------------------- local PKG_PAIR_FILES = fetch_pkg_pairs(file_pattern) local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES) local PKGBASE = conflicting_pkgs(CONFLICT_PKGS) local ORIGIN_LIST = table_sorted_keys(PKGBASE) local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST) local function conflicting_files(pkg_i, pkgs) local files = {} local all_files = {} local f local p1 = PKG_PAIR_FILES[pkg_i] if p1 then for _, pkg_j in ipairs(pkgs) do f = p1[pkg_j] if f then table.sort(f) files[#files + 1] = f[1] table.move(f, 1, #f, #all_files + 1, all_files) end end end for _, pkg_j in ipairs(pkgs) do p1 = PKG_PAIR_FILES[pkg_j] f = p1 and p1[pkg_i] if f then table.sort(f) files[#files + 1] = f[1] table.move(f, 1, #f, #all_files + 1, all_files) end end return table_sort_uniq(files), table_sort_uniq(all_files) end --------------------------------------------------------------------- local version_pattern = { "^lib/python%d%.%d/", "^share/py3%d%d%?-", "^share/%a+/py3%d%d%?-", "^lib/lua/%d%.%d/", "^share/lua/%d%.%d/", "^lib/perl5/[%d%.]+/", "^lib/perl5/site_perl/mach/[%d%.]+/", "^lib/ruby/gems/%d%.%d/", "^lib/ruby/site_ruby/%d%.%d/", } local function generalize_patterns(pkgs, files) local function match_any(str, pattern_array) for _, pattern in ipairs(pattern_array) do if string.match(str, pattern) then return true end end return false end local function unversioned_files() for i = 1, #files do if not match_any(files[i], version_pattern) then return true end end return false end local function pkg_wildcards(from, ...) local to_list = {...} local result = {} for i = 1, #pkgs do local orig_pkg = pkgs[i] for _, to in ipairs(to_list) do result[string.gsub(orig_pkg, from, to)] = true end end pkgs = table_sorted_keys(result) end local pkg_pfx_php = "php[0-9][0-9]-" local pkg_sfx_php = "-php[0-9][0-9]" local pkg_pfx_python2 local pkg_sfx_python2 local pkg_pfx_python3 local pkg_sfx_python3 local pkg_pfx_lua local pkg_pfx_ruby pkgs = table_sort_uniq(pkgs) if unversioned_files() then pkg_pfx_python2 = "py3[0-9]-" -- e.g. py39- pkg_sfx_python2 = "-py3[0-9]" pkg_pfx_python3 = "py3[0-9][0-9]-" -- e.g. py311- pkg_sfx_python3 = "-py3[0-9][0-9]" pkg_pfx_lua = "lua[0-9][0-9]-" pkg_pfx_ruby = "ruby[0-9][0-9]-" else pkg_pfx_python2 = "${PYTHON_PKGNAMEPREFIX}" pkg_sfx_python2 = "${PYTHON_PKGNAMESUFFIX}" pkg_pfx_python3 = nil pkg_sfx_python3 = nil pkg_pfx_lua = "${LUA_PKGNAMEPREFIX}" pkg_pfx_ruby = "${RUBY_PKGNAMEPREFIX}" end pkg_wildcards("^php%d%d%-", pkg_pfx_php) pkg_wildcards("-php%d%d$", pkg_sfx_php) pkg_wildcards("^phpunit%d%-", "phpunit[0-9]-") pkg_wildcards("^py3%d%-", pkg_pfx_python2, pkg_pfx_python3) pkg_wildcards("-py3%d%$", pkg_sfx_python2, pkg_sfx_python3) pkg_wildcards("^py3%d%d%-", pkg_pfx_python2, pkg_pfx_python3) pkg_wildcards("-py3%d%d%$", pkg_sfx_python2, pkg_sfx_python3) pkg_wildcards("^lua%d%d%-", pkg_pfx_lua) pkg_wildcards("-emacs_[%a_]*", "-emacs_*") pkg_wildcards("^ghostscript%d%-", "ghostscript[0-9]-") pkg_wildcards("^bacula%d%-", "bacula[0-9]-") pkg_wildcards("^bacula%d%d%-", "bacula[0-9][0-9]-") pkg_wildcards("^bareos%d%-", "bareos[0-9]-") pkg_wildcards("^bareos%d%d%-", "bareos[0-9][0-9]-") pkg_wildcards("^moosefs%d%-", "moosefs[0-9]-") pkg_wildcards("^ruby%d+-", pkg_pfx_ruby) return table_sort_uniq(pkgs) end ------------------------------------------------------------------- for _, port in ipairs(PORT_LIST) do local function merge_table(t1, t2) table.move(t2, 1, #t2, #t1 + 1, t1) end local port_conflicts = {} local files = {} local msg_files = {} local conflict_pkgs = {} local function merge_data(origin) local pkgbase = PKGBASE[origin] if not BAD_FILE_PKGS[pkgbase] then local pkg_confl_file1, pkg_confl_files = conflicting_files(pkgbase, CONFLICT_PKGS[pkgbase]) merge_table(msg_files, pkg_confl_file1) -- 1 file per flavor merge_table(files, pkg_confl_files) -- all conflicting files merge_table(conflict_pkgs, CONFLICT_PKGS[pkgbase]) merge_table(port_conflicts, fetch_port_conflicts(origin)) end end local flavors = FLAVORS[port] if flavors then for _, flavor in ipairs(flavors) do merge_data(port .. "@" .. flavor) end else merge_data(port) end files = table_sort_uniq(files) msg_files = table_sort_uniq(msg_files) conflict_pkgs = generalize_patterns(conflict_pkgs, files) if #port_conflicts then port_conflicts = table_sort_uniq(port_conflicts) conflict_pkgs = conflicts_delta(port_conflicts, conflict_pkgs) end if conflict_pkgs then local conflicts_string_cur = table.concat(port_conflicts, " ") local conflicts_string_new = table.concat(conflict_pkgs, " ") local file_list = table.concat(msg_files, " ") print("# Port: " .. port) print("# Files: " .. file_list) if conflicts_string_cur ~= "" then print("# < " .. conflicts_string_cur) end print("# > " .. conflicts_string_new) print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new .. " # " .. file_list .. "' /usr/ports/" .. port) print() end end ------------------------------------------------------------------- local BAD_FILES_ORIGINS = {} for _, origin in ipairs(ORIGIN_LIST) do local pkgbase = PKGBASE[origin] local files = BAD_FILE_PKGS[pkgbase] if files then for _, file in ipairs(files) do if DUPLICATE_FILE[file] then local port = string.match(origin, "([^@]+)@?") BAD_FILES_ORIGINS[port] = BAD_FILES_ORIGINS[origin] or {} table.insert(BAD_FILES_ORIGINS[port], file) end end end end ------------------------------------------------------------------- local bad_origins = table_sorted_keys(BAD_FILES_ORIGINS) if #bad_origins > 0 then print ("# Ports with badly named files:") print () for _, port in ipairs(bad_origins) do print ("# " .. port) local files = BAD_FILES_ORIGINS[port] table.sort(files) for _, file in ipairs(files) do print ("#", file) end print() end end