X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcruft_report.py;h=4541bf6eb6477c8c09cc91022b8c41636d031ebc;hb=6021a91078574c97968d076c0320182a20c33649;hp=8640ea6163e02581f86885717ce1742d61520763;hpb=430daaea3b7a870b4828cccb22a15b8ead3ba965;p=dak.git diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 8640ea61..4541bf6e 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Check for obsolete binary packages +""" Check for obsolete binary packages """ # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup # This program is free software; you can redistribute it and/or modify @@ -23,21 +23,22 @@ # you might as well write some letters to God about how unfair entropy # is while you're at it.'' -- 20020802143104.GA5628@azure.humbug.org.au -## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of duplicate source package stuff, improve experimental ?, add overrides, avoid ANAIS for duplicated packages +## TODO: fix NBS looping for version, implement Dubious NBS, fix up output of +## duplicate source package stuff, improve experimental ?, add overrides, +## avoid ANAIS for duplicated packages ################################################################################ -import commands, pg, os, sys, time, re +import commands, os, sys, re import apt_pkg -from daklib import database + +from daklib.config import Config +from daklib.dbconn import * from daklib import utils +from daklib.regexes import re_extract_src_version ################################################################################ -Cnf = None -projectB = None -suite = "unstable" # Default -suite_id = None no_longer_in_suite = {}; # Really should be static to add_nbs, but I'm lazy source_binaries = {} @@ -57,13 +58,16 @@ Check for obsolete or duplicated packages. ################################################################################ -def add_nbs(nbs_d, source, version, package): +def add_nbs(nbs_d, source, version, package, suite_id, session): # Ensure the package is still in the suite (someone may have already removed it) if no_longer_in_suite.has_key(package): return else: - q = projectB.query("SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = %s AND b.package = '%s' LIMIT 1" % (suite_id, package)) - if not q.getresult(): + q = session.execute("""SELECT b.id FROM binaries b, bin_associations ba + WHERE ba.bin = b.id AND ba.suite = :suite_id + AND b.package = :package LIMIT 1""", {'suite_id': suite_id, + 'package': package}) + if not q.fetchall(): no_longer_in_suite[package] = "" return @@ -74,7 +78,7 @@ def add_nbs(nbs_d, source, version, package): ################################################################################ # Check for packages built on architectures they shouldn't be. -def do_anais(architecture, binaries_list, source): +def do_anais(architecture, binaries_list, source, session): if architecture == "any" or architecture == "all": return "" @@ -83,8 +87,12 @@ def do_anais(architecture, binaries_list, source): for arch in architecture.split(): architectures[arch.strip()] = "" for binary in binaries_list: - q = projectB.query("SELECT a.arch_string, b.version FROM binaries b, bin_associations ba, architecture a WHERE ba.suite = %s AND ba.bin = b.id AND b.architecture = a.id AND b.package = '%s'" % (suite_id, binary)) - ql = q.getresult() + q = session.execute("""SELECT a.arch_string, b.version + FROM binaries b, bin_associations ba, architecture a + WHERE ba.suite = :suiteid AND ba.bin = b.id + AND b.architecture = a.id AND b.package = :package""", + {'suiteid': suite_id, 'package': binary}) + ql = q.fetchall() versions = [] for i in ql: arch = i[0] @@ -146,12 +154,13 @@ def do_nfu(nfu_packages): print def parse_nfu(architecture): + cnf = Config() # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date] r = re.compile("^\w+/([^_]+)_.*: Not-For-Us") ret = set() - filename = "%s/%s-all.txt" % (Cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture) + filename = "%s/%s-all.txt" % (cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture) # Not all architectures may have a wanna-build dump, so we want to ignore missin # files @@ -167,36 +176,42 @@ def parse_nfu(architecture): f.close() else: - utils.warn("No wanna-build dump file for architecture %s", architecture) + utils.warn("No wanna-build dump file for architecture %s" % architecture) return ret ################################################################################ -def do_nviu(): - experimental_id = database.get_suite_id("experimental") - if experimental_id == -1: +def do_newer_version(lowersuite_name, highersuite_name, code, session): + lowersuite = get_suite(lowersuite_name, session) + if not lowersuite: + return + + highersuite = get_suite(highersuite_name, session) + if not highersuite: return - # Check for packages in experimental obsoleted by versions in unstable - q = projectB.query(""" -SELECT s.source, s.version AS experimental, s2.version AS unstable + + # Check for packages in $highersuite obsoleted by versions in $lowersuite + q = session.execute(""" +SELECT s.source, s.version AS lower, s2.version AS higher FROM src_associations sa, source s, source s2, src_associations sa2 - WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id + WHERE sa.suite = :highersuite_id AND sa2.suite = :lowersuite_id AND sa.source = s.id AND sa2.source = s2.id AND s.source = s2.source - AND versioncmp(s.version, s2.version) < 0""" % (experimental_id, - database.get_suite_id("unstable"))) - ql = q.getresult() + AND s.version < s2.version""", {'lowersuite_id': lowersuite.suite_id, + 'highersuite_id': highersuite.suite_id}) + ql = q.fetchall() if ql: - nviu_to_remove = [] - print "Newer version in unstable" - print "-------------------------" + nv_to_remove = [] + print "Newer version in %s" % lowersuite.suite_name + print "-----------------" + "-" * len(lowersuite.suite_name) print for i in ql: - (source, experimental_version, unstable_version) = i - print " o %s (%s, %s)" % (source, experimental_version, unstable_version) - nviu_to_remove.append(source) + (source, higher_version, lower_version) = i + print " o %s (%s, %s)" % (source, higher_version, lower_version) + nv_to_remove.append(source) print print "Suggested command:" - print " dak rm -m \"[auto-cruft] NVIU\" -s experimental %s" % (" ".join(nviu_to_remove)) + print " dak rm -m \"[auto-cruft] %s\" -s %s %s" % (code, highersuite.suite_name, + " ".join(nv_to_remove)) print ################################################################################ @@ -205,7 +220,7 @@ def do_nbs(real_nbs): output = "Not Built from Source\n" output += "---------------------\n\n" - nbs_to_remove = [] + cmd_output = "" nbs_keys = real_nbs.keys() nbs_keys.sort() for source in nbs_keys: @@ -215,21 +230,22 @@ def do_nbs(real_nbs): output += " but no longer builds:\n" versions = real_nbs[source].keys() versions.sort(apt_pkg.VersionCompare) + all_packages = [] for version in versions: packages = real_nbs[source][version].keys() packages.sort() - for pkg in packages: - nbs_to_remove.append(pkg) + all_packages.extend(packages) output += " o %s: %s\n" % (version, ", ".join(packages)) + if all_packages: + all_packages.sort() + cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s\n\n" % (source, suite.suite_name, " ".join(all_packages)) output += "\n" - if nbs_to_remove: + if len(cmd_output): print output - - print "Suggested command:" - print " dak rm -m \"[auto-cruft] NBS\" -s %s -b %s" % (suite, " ".join(nbs_to_remove)) - print + print "Suggested commands:\n" + print cmd_output ################################################################################ @@ -298,16 +314,16 @@ def do_obsolete_source(duplicate_bins, bin2source): print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove)) print -def get_suite_binaries(): +def get_suite_binaries(suite, session): # Initalize a large hash table of all binary packages binaries = {} - before = time.time() - sys.stderr.write("[Getting a list of binary packages in %s..." % (suite)) - q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id)) - ql = q.getresult() - sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) - for i in ql: + print "Getting a list of binary packages in %s..." % suite.suite_name + q = session.execute("""SELECT distinct b.package + FROM binaries b, bin_associations ba + WHERE ba.suite = :suiteid AND ba.bin = b.id""", + {'suiteid': suite.suite_id}) + for i in q.fetchall(): binaries[i[0]] = "" return binaries @@ -315,42 +331,41 @@ def get_suite_binaries(): ################################################################################ def main (): - global Cnf, projectB, suite, suite_id, source_binaries, source_versions + global suite, suite_id, source_binaries, source_versions - Cnf = utils.get_conf() + cnf = Config() Arguments = [('h',"help","Cruft-Report::Options::Help"), ('m',"mode","Cruft-Report::Options::Mode", "HasArg"), ('s',"suite","Cruft-Report::Options::Suite","HasArg"), ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")] for i in [ "help" ]: - if not Cnf.has_key("Cruft-Report::Options::%s" % (i)): - Cnf["Cruft-Report::Options::%s" % (i)] = "" - Cnf["Cruft-Report::Options::Suite"] = Cnf["Dinstall::DefaultSuite"] + if not cnf.has_key("Cruft-Report::Options::%s" % (i)): + cnf["Cruft-Report::Options::%s" % (i)] = "" + cnf["Cruft-Report::Options::Suite"] = cnf["Dinstall::DefaultSuite"] - if not Cnf.has_key("Cruft-Report::Options::Mode"): - Cnf["Cruft-Report::Options::Mode"] = "daily" + if not cnf.has_key("Cruft-Report::Options::Mode"): + cnf["Cruft-Report::Options::Mode"] = "daily" - if not Cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"): - Cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu" + if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"): + cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu" - apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) + apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) - Options = Cnf.SubTree("Cruft-Report::Options") + Options = cnf.SubTree("Cruft-Report::Options") if Options["Help"]: usage() # Set up checks based on mode if Options["Mode"] == "daily": - checks = [ "nbs", "nviu", "obsolete source" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source" ] elif Options["Mode"] == "full": - checks = [ "nbs", "nviu", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] else: utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"])) usage(1) - projectB = pg.connect(Cnf["DB::Name"], Cnf["DB::Host"], int(Cnf["DB::Port"])) - database.init(Cnf, projectB) + session = DBConn().session() bin_pkgs = {} src_pkgs = {} @@ -364,20 +379,21 @@ def main (): nfu_packages = {} - suite = Options["Suite"] - suite_id = database.get_suite_id(suite) + suite = get_suite(Options["Suite"].lower(), session) + suite_id = suite.suite_id + suite_name = suite.suite_name.lower() bin_not_built = {} if "bnb" in checks: - bins_in_suite = get_suite_binaries() + bins_in_suite = get_suite_binaries(suite_name, session) # Checks based on the Sources files - components = Cnf.ValueList("Suite::%s::Components" % (suite)) + components = cnf.ValueList("Suite::%s::Components" % (suite_name)) for component in components: - filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component) + filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite_name, component) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - temp_filename = utils.temp_filename() + (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) @@ -399,7 +415,7 @@ def main (): bin_not_built[source][binary] = "" if "anais" in checks: - anais_output += do_anais(architecture, binaries_list, source) + anais_output += do_anais(architecture, binaries_list, source, session) # Check for duplicated packages and build indices for checking "no source" later source_index = component + '/' + source @@ -422,14 +438,19 @@ def main (): # Checks based on the Packages files check_components = components[:] - if suite != "experimental": + if suite_name != "experimental": check_components.append('main/debian-installer'); + for component in check_components: - architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))) + architectures = [ a.arch_string for a in get_suite_architectures(suite_name, + skipsrc=True, skipall=True, + session=session) ] for architecture in architectures: - filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture) + if component == 'main/debian-installer' and re.match("kfreebsd", architecture): + continue + filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suite_name, component, architecture) # apt_pkg.ParseTagFile needs a real file handle - temp_filename = utils.temp_filename() + (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) @@ -456,7 +477,7 @@ def main (): bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: - m = utils.re_extract_src_version.match(source) + m = re_extract_src_version.match(source) source = m.group(1) version = m.group(2) if not bin_pkgs.has_key(package): @@ -472,9 +493,10 @@ def main (): duplicate_bins.setdefault(key, []) if package not in duplicate_bins[key]: duplicate_bins[key].append(package) - if package in nfu_entries and \ - version != source_versions[source]: # only suggest to remove out-of-date packages - nfu_packages[architecture].append((package,version,source_versions[source])) + if "nfu" in checks: + if package in nfu_entries and \ + version != source_versions[source]: # only suggest to remove out-of-date packages + nfu_packages[architecture].append((package,version,source_versions[source])) packages.close() os.unlink(temp_filename) @@ -492,12 +514,15 @@ def main (): latest_version = versions.pop() source_version = source_versions.get(source,"0") if apt_pkg.VersionCompare(latest_version, source_version) == 0: - add_nbs(dubious_nbs, source, latest_version, package) + add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) else: - add_nbs(real_nbs, source, latest_version, package) + add_nbs(real_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: - do_nviu() + do_newer_version('unstable', 'experimental', 'NVIU', session) + + if "nvit" in checks: + do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) if "nbs" in checks: do_nbs(real_nbs)