X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcruft_report.py;h=30650d4139ec7860ea35158329bf87126f06f875;hb=b612f3da207fa0d75a5d3b204ac8f02bb244231a;hp=238cdd497463dbea6824437b4faa994940619eb5;hpb=5819288346c6ea5d9b23125586969cbec3bc6d44;p=dak.git diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 238cdd49..30650d41 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Check for obsolete binary packages +""" Check for obsolete binary packages """ # Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup # This program is free software; you can redistribute it and/or modify @@ -27,10 +27,11 @@ ################################################################################ -import commands, pg, os, sys, time +import commands, pg, os, sys, time, re import apt_pkg -import daklib.database as database -import daklib.utils as utils +from daklib import database +from daklib import utils +from daklib.regexes import re_extract_src_version ################################################################################ @@ -51,7 +52,8 @@ Check for obsolete or duplicated packages. -h, --help show this help and exit. -m, --mode=MODE chose the MODE to run in (full or daily). - -s, --suite=SUITE check suite SUITE.""" + -s, --suite=SUITE check suite SUITE. + -w, --wanna-build-dump where to find the copies of http://buildd.debian.org/stats/*.txt""" sys.exit(exit_code) ################################################################################ @@ -114,6 +116,61 @@ def do_anais(architecture, binaries_list, source): anais_output += " o %s: %s\n" % (version, ", ".join(arches)) return anais_output + +################################################################################ + +# Check for out-of-date binaries on architectures that do not want to build that +# package any more, and have them listed as Not-For-Us +def do_nfu(nfu_packages): + output = "" + + a2p = {} + + for architecture in nfu_packages: + a2p[architecture] = [] + for (package,bver,sver) in nfu_packages[architecture]: + output += " * [%s] does not want %s (binary %s, source %s)\n" % (architecture, package, bver, sver) + a2p[architecture].append(package) + + + if output: + print "Obsolete by Not-For-Us" + print "----------------------" + print + print output + + print "Suggested commands:" + for architecture in a2p: + if a2p[architecture]: + print (" dak rm -m \"[auto-cruft] NFU\" -s %s -a %s -b %s" % + (suite, architecture, " ".join(a2p[architecture]))) + print + +def parse_nfu(architecture): + # utils/hpodder_1.1.5.0: Not-For-Us [optional:out-of-date] + r = re.compile("^\w+/([^_]+)_.*: Not-For-Us") + + ret = set() + + filename = "%s/%s-all.txt" % (Cnf["Cruft-Report::Options::Wanna-Build-Dump"], architecture) + + # Not all architectures may have a wanna-build dump, so we want to ignore missin + # files + if os.path.exists(filename): + f = utils.open_file(filename) + for line in f: + if line[0] == ' ': + continue + + m = r.match(line) + if m: + ret.add(m.group(1)) + + f.close() + else: + utils.warn("No wanna-build dump file for architecture %s" % architecture) + return ret + ################################################################################ def do_nviu(): @@ -126,8 +183,8 @@ SELECT s.source, s.version AS experimental, s2.version AS unstable FROM src_associations sa, source s, source s2, src_associations sa2 WHERE sa.suite = %s AND sa2.suite = %d AND sa.source = s.id AND sa2.source = s2.id AND s.source = s2.source - AND versioncmp(s.version, s2.version) < 0""" % (experimental_id, - database.get_suite_id("unstable"))) + AND s.version < s2.version""" % (experimental_id, + database.get_suite_id("unstable"))) ql = q.getresult() if ql: nviu_to_remove = [] @@ -149,7 +206,7 @@ def do_nbs(real_nbs): output = "Not Built from Source\n" output += "---------------------\n\n" - nbs_to_remove = [] + cmd_output = "" nbs_keys = real_nbs.keys() nbs_keys.sort() for source in nbs_keys: @@ -159,21 +216,22 @@ def do_nbs(real_nbs): output += " but no longer builds:\n" versions = real_nbs[source].keys() versions.sort(apt_pkg.VersionCompare) + all_packages = [] for version in versions: packages = real_nbs[source][version].keys() packages.sort() - for pkg in packages: - nbs_to_remove.append(pkg) + all_packages.extend(packages) output += " o %s: %s\n" % (version, ", ".join(packages)) + if all_packages: + all_packages.sort() + cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s\n\n" % (source, suite, " ".join(all_packages)) output += "\n" - if nbs_to_remove: + if len(cmd_output): print output - - print "Suggested command:" - print " dak rm -m \"[auto-cruft] NBS\" -s %s -b %s" % (suite, " ".join(nbs_to_remove)) - print + print "Suggested commands:\n" + print cmd_output ################################################################################ @@ -242,6 +300,20 @@ def do_obsolete_source(duplicate_bins, bin2source): print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove)) print +def get_suite_binaries(): + # Initalize a large hash table of all binary packages + binaries = {} + before = time.time() + + sys.stderr.write("[Getting a list of binary packages in %s..." % (suite)) + q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id)) + ql = q.getresult() + sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) + for i in ql: + binaries[i[0]] = "" + + return binaries + ################################################################################ def main (): @@ -251,7 +323,8 @@ def main (): Arguments = [('h',"help","Cruft-Report::Options::Help"), ('m',"mode","Cruft-Report::Options::Mode", "HasArg"), - ('s',"suite","Cruft-Report::Options::Suite","HasArg")] + ('s',"suite","Cruft-Report::Options::Suite","HasArg"), + ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")] for i in [ "help" ]: if not Cnf.has_key("Cruft-Report::Options::%s" % (i)): Cnf["Cruft-Report::Options::%s" % (i)] = "" @@ -260,6 +333,9 @@ def main (): if not Cnf.has_key("Cruft-Report::Options::Mode"): Cnf["Cruft-Report::Options::Mode"] = "daily" + if not Cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"): + Cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu" + apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv) Options = Cnf.SubTree("Cruft-Report::Options") @@ -270,7 +346,7 @@ def main (): if Options["Mode"] == "daily": checks = [ "nbs", "nviu", "obsolete source" ] elif Options["Mode"] == "full": - checks = [ "nbs", "nviu", "obsolete source", "dubious nbs", "bnb", "bms", "anais" ] + checks = [ "nbs", "nviu", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] else: utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"])) usage(1) @@ -288,27 +364,22 @@ def main (): anais_output = "" duplicate_bins = {} + nfu_packages = {} + suite = Options["Suite"] suite_id = database.get_suite_id(suite) bin_not_built = {} if "bnb" in checks: - # Initalize a large hash table of all binary packages - before = time.time() - sys.stderr.write("[Getting a list of binary packages in %s..." % (suite)) - q = projectB.query("SELECT distinct b.package FROM binaries b, bin_associations ba WHERE ba.suite = %s AND ba.bin = b.id" % (suite_id)) - ql = q.getresult() - sys.stderr.write("done. (%d seconds)]\n" % (int(time.time()-before))) - for i in ql: - bins_in_suite[i[0]] = "" + bins_in_suite = get_suite_binaries() # Checks based on the Sources files components = Cnf.ValueList("Suite::%s::Components" % (suite)) for component in components: filename = "%s/dists/%s/%s/source/Sources.gz" % (Cnf["Dir::Root"], suite, component) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... - temp_filename = utils.temp_filename() + (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) @@ -356,15 +427,22 @@ def main (): if suite != "experimental": check_components.append('main/debian-installer'); for component in check_components: - architectures = filter(utils.real_arch, Cnf.ValueList("Suite::%s::Architectures" % (suite))) + architectures = filter(utils.real_arch, database.get_suite_architectures(suite)) for architecture in architectures: + if component == 'main/debian-installer' and re.match("kfreebsd", architecture): + continue filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (Cnf["Dir::Root"], suite, component, architecture) # apt_pkg.ParseTagFile needs a real file handle - temp_filename = utils.temp_filename() + (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) + + if "nfu" in checks: + nfu_packages.setdefault(architecture,[]) + nfu_entries = parse_nfu(architecture) + packages = utils.open_file(temp_filename) Packages = apt_pkg.ParseTagFile(packages) while Packages.Step(): @@ -382,7 +460,7 @@ def main (): bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: - m = utils.re_extract_src_version.match(source) + m = re_extract_src_version.match(source) source = m.group(1) version = m.group(2) if not bin_pkgs.has_key(package): @@ -398,6 +476,11 @@ def main (): duplicate_bins.setdefault(key, []) if package not in duplicate_bins[key]: duplicate_bins[key].append(package) + if "nfu" in checks: + if package in nfu_entries and \ + version != source_versions[source]: # only suggest to remove out-of-date packages + nfu_packages[architecture].append((package,version,source_versions[source])) + packages.close() os.unlink(temp_filename) @@ -430,6 +513,9 @@ def main (): print "="*75 print + if "nfu" in checks: + do_nfu(nfu_packages) + if "bnb" in checks: print "Unbuilt binary packages" print "-----------------------"