X-Git-Url: https://git.decadent.org.uk/gitweb/?p=dak.git;a=blobdiff_plain;f=dak%2Fcruft_report.py;h=a960a686717cbedf881d07317cad2078e08aa4ab;hp=6c3f16e83dace7a0d1a8e1118b575b8a2f72ecca;hb=2a10967af5974e4fa513d029fb995e122d90501a;hpb=c94d8bb3dc23bf45f88fa9e53e6dd0ca767d8725 diff --git a/dak/cruft_report.py b/dak/cruft_report.py index 6c3f16e8..a960a686 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -58,9 +58,10 @@ def usage(exit_code=0): Check for obsolete or duplicated packages. -h, --help show this help and exit. - -m, --mode=MODE chose the MODE to run in (full or daily). + -m, --mode=MODE chose the MODE to run in (full, daily, bdo). -s, --suite=SUITE check suite SUITE. - -w, --wanna-build-dump where to find the copies of http://buildd.debian.org/stats/*.txt""" + -R, --rdep-check check reverse dependencies + -w, --wanna-build-dump where to find the copies of https://buildd.debian.org/stats/*.txt""" sys.exit(exit_code) ################################################################################ @@ -106,7 +107,7 @@ def do_anais(architecture, binaries_list, source, session): version = i[1] if architectures.has_key(arch): versions.append(version) - versions.sort(apt_pkg.VersionCompare) + versions.sort(apt_pkg.version_compare) if versions: latest_version = versions.pop() else: @@ -123,7 +124,7 @@ def do_anais(architecture, binaries_list, source, session): if versions_d != {}: anais_output += "\n (*) %s_%s [%s]: %s\n" % (binary, latest_version, source, architecture) versions = versions_d.keys() - versions.sort(apt_pkg.VersionCompare) + versions.sort(apt_pkg.version_compare) for version in versions: arches = versions_d[version] arches.sort() @@ -157,7 +158,7 @@ def do_nfu(nfu_packages): for architecture in a2p: if a2p[architecture]: print (" dak rm -m \"[auto-cruft] NFU\" -s %s -a %s -b %s" % - (suite, architecture, " ".join(a2p[architecture]))) + (suite.suite_name, architecture, " ".join(a2p[architecture]))) print def parse_nfu(architecture): @@ -229,7 +230,7 @@ def queryWithoutSource(suite_id, session): order by ub.package""" return session.execute(query, { 'suite_id': suite_id }) -def reportWithoutSource(suite_name, suite_id, session): +def reportWithoutSource(suite_name, suite_id, session, rdeps=False): rows = queryWithoutSource(suite_id, session) title = 'packages without source in suite %s' % suite_name if rows.rowcount > 0: @@ -240,8 +241,15 @@ def reportWithoutSource(suite_name, suite_id, session): print "* package %s in version %s is no longer built from source" % \ (package, version) print " - suggested command:" - print " dak rm -m %s -s %s -a all -p -R -b %s\n" % \ + print " dak rm -m %s -s %s -a all -p -R -b %s" % \ (message, suite_name, package) + if rdeps: + if utils.check_reverse_depends([package], suite_name, [], session, True): + print + else: + print " - No dependency problem found\n" + else: + print def queryNewerAll(suite_name, session): """searches for arch != all packages that have an arch == all @@ -361,7 +369,7 @@ with uptodate_arch as select * from outdated_packages order by source""" return session.execute(query, { 'suite_id': suite_id }) -def reportNBS(suite_name, suite_id): +def reportNBS(suite_name, suite_id, rdeps=False): session = DBConn().session() nbsRows = queryNBS(suite_id, session) title = 'NBS packages in suite %s' % suite_name @@ -377,14 +385,21 @@ def reportNBS(suite_name, suite_id): print " on %s" % arch_string print " - suggested command:" message = '"[auto-cruft] NBS (no longer built by %s)"' % source - print " dak rm -m %s -s %s -a %s -p -R -b %s\n" % \ + print " dak rm -m %s -s %s -a %s -p -R -b %s" % \ (message, suite_name, arch_string, pkg_string) + if rdeps: + if utils.check_reverse_depends(pkg_list, suite_name, arch_list, session, True): + print + else: + print " - No dependency problem found\n" + else: + print session.close() -def reportAllNBS(suite_name, suite_id, session): - reportWithoutSource(suite_name, suite_id, session) +def reportAllNBS(suite_name, suite_id, session, rdeps=False): + reportWithoutSource(suite_name, suite_id, session, rdeps) reportNewerAll(suite_name, session) - reportNBS(suite_name, suite_id) + reportNBS(suite_name, suite_id, rdeps) ################################################################################ @@ -401,7 +416,7 @@ def do_dubious_nbs(dubious_nbs): source_binaries.get(source, "(source does not exist)")) print " won't admit to building:" versions = dubious_nbs[source].keys() - versions.sort(apt_pkg.VersionCompare) + versions.sort(apt_pkg.version_compare) for version in versions: packages = dubious_nbs[source][version].keys() packages.sort() @@ -506,6 +521,84 @@ def get_suite_binaries(suite, session): ################################################################################ +def report_outdated_nonfree(suite, session, rdeps=False): + + packages = {} + query = """WITH outdated_sources AS ( + SELECT s.source, s.version, s.id + FROM source s + JOIN src_associations sa ON sa.source = s.id + WHERE sa.suite IN ( + SELECT id + FROM suite + WHERE suite_name = :suite ) + AND sa.created < (now() - interval :delay) + EXCEPT SELECT s.source, max(s.version) AS version, max(s.id) + FROM source s + JOIN src_associations sa ON sa.source = s.id + WHERE sa.suite IN ( + SELECT id + FROM suite + WHERE suite_name = :suite ) + AND sa.created < (now() - interval :delay) + GROUP BY s.source ), + binaries AS ( + SELECT b.package, s.source, ( + SELECT a.arch_string + FROM architecture a + WHERE a.id = b.architecture ) AS arch + FROM binaries b + JOIN outdated_sources s ON s.id = b.source + JOIN bin_associations ba ON ba.bin = b.id + JOIN override o ON o.package = b.package AND o.suite = ba.suite + WHERE ba.suite IN ( + SELECT id + FROM suite + WHERE suite_name = :suite ) + AND o.component IN ( + SELECT id + FROM component + WHERE name = 'non-free' ) ) + SELECT DISTINCT package, source, arch + FROM binaries + ORDER BY source, package, arch""" + + res = session.execute(query, {'suite': suite, 'delay': "'15 days'"}) + for package in res: + binary = package[0] + source = package[1] + arch = package[2] + if arch == 'all': + continue + if not source in packages: + packages[source] = {} + if not binary in packages[source]: + packages[source][binary] = set() + packages[source][binary].add(arch) + if packages: + title = 'Outdated non-free binaries in suite %s' % suite + message = '"[auto-cruft] outdated non-free binaries"' + print '%s\n%s\n' % (title, '-' * len(title)) + for source in sorted(packages): + archs = set() + binaries = set() + print '* package %s has outdated non-free binaries' % source + print ' - suggested command:' + for binary in sorted(packages[source]): + binaries.add(binary) + archs = archs.union(packages[source][binary]) + print ' dak rm -m %s -s %s -a %s -p -R -b %s' % \ + (message, suite, ','.join(archs), ' '.join(binaries)) + if rdeps: + if utils.check_reverse_depends(list(binaries), suite, archs, session, True): + print + else: + print " - No dependency problem found\n" + else: + print + +################################################################################ + def main (): global suite, suite_id, source_binaries, source_versions @@ -513,32 +606,41 @@ def main (): Arguments = [('h',"help","Cruft-Report::Options::Help"), ('m',"mode","Cruft-Report::Options::Mode", "HasArg"), + ('R',"rdep-check", "Cruft-Report::Options::Rdep-Check"), ('s',"suite","Cruft-Report::Options::Suite","HasArg"), ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")] - for i in [ "help" ]: + for i in [ "help", "Rdep-Check" ]: if not cnf.has_key("Cruft-Report::Options::%s" % (i)): cnf["Cruft-Report::Options::%s" % (i)] = "" - cnf["Cruft-Report::Options::Suite"] = cnf["Dinstall::DefaultSuite"] + + cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") if not cnf.has_key("Cruft-Report::Options::Mode"): cnf["Cruft-Report::Options::Mode"] = "daily" if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"): - cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp.debian.org/scripts/nfu" + cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu" - apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv) + apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) - Options = cnf.SubTree("Cruft-Report::Options") + Options = cnf.subtree("Cruft-Report::Options") if Options["Help"]: usage() + if Options["Rdep-Check"]: + rdeps = True + else: + rdeps = False + # Set up checks based on mode if Options["Mode"] == "daily": - checks = [ "nbs", "nviu", "nvit", "obsolete source" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu" ] elif Options["Mode"] == "full": - checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais" ] + elif Options["Mode"] == "bdo": + checks = [ "nbs", "obsolete source" ] else: - utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"])) + utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"])) usage(1) session = DBConn().session() @@ -551,11 +653,13 @@ def main (): source_versions = {} anais_output = "" - duplicate_bins = {} nfu_packages = {} suite = get_suite(Options["Suite"].lower(), session) + if not suite: + utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) + suite_id = suite.suite_id suite_name = suite.suite_name.lower() @@ -563,7 +667,10 @@ def main (): report_obsolete_source(suite_name, session) if "nbs" in checks: - reportAllNBS(suite_name, suite_id, session) + reportAllNBS(suite_name, suite_id, session, rdeps) + + if "outdated non-free" in checks: + report_outdated_nonfree(suite_name, session, rdeps) bin_not_built = {} @@ -571,22 +678,22 @@ def main (): bins_in_suite = get_suite_binaries(suite, session) # Checks based on the Sources files - components = cnf.ValueList("Suite::%s::Components" % (suite_name)) + components = get_component_names(session) for component in components: - filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite_name, component) - # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... + filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path, suite_name, component) + # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance... (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) sources = utils.open_file(temp_filename) - Sources = apt_pkg.ParseTagFile(sources) - while Sources.Step(): - source = Sources.Section.Find('Package') - source_version = Sources.Section.Find('Version') - architecture = Sources.Section.Find('Architecture') - binaries = Sources.Section.Find('Binary') + Sources = apt_pkg.TagFile(sources) + while Sources.step(): + source = Sources.section.find('Package') + source_version = Sources.section.find('Version') + architecture = Sources.section.find('Architecture') + binaries = Sources.section.find('Binary') binaries_list = [ i.strip() for i in binaries.split(',') ] if "bnb" in checks: @@ -599,18 +706,10 @@ def main (): if "anais" in checks: anais_output += do_anais(architecture, binaries_list, source, session) - # Check for duplicated packages and build indices for checking "no source" later + # build indices for checking "no source" later source_index = component + '/' + source - #if src_pkgs.has_key(source): - # print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]) src_pkgs[source] = source_index for binary in binaries_list: - if bin_pkgs.has_key(binary): - key_list = [ source, bin_pkgs[binary] ] - key_list.sort() - key = '_'.join(key_list) - duplicate_bins.setdefault(key, []) - duplicate_bins[key].append(binary) bin_pkgs[binary] = source source_binaries[source] = binaries source_versions[source] = source_version @@ -630,8 +729,8 @@ def main (): for architecture in architectures: if component == 'main/debian-installer' and re.match("kfreebsd", architecture): continue - filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suite_name, component, architecture) - # apt_pkg.ParseTagFile needs a real file handle + filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (suite.archive.path, suite_name, component, architecture) + # apt_pkg.TagFile needs a real file handle (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): @@ -643,15 +742,15 @@ def main (): nfu_entries = parse_nfu(architecture) packages = utils.open_file(temp_filename) - Packages = apt_pkg.ParseTagFile(packages) - while Packages.Step(): - package = Packages.Section.Find('Package') - source = Packages.Section.Find('Source', "") - version = Packages.Section.Find('Version') + Packages = apt_pkg.TagFile(packages) + while Packages.step(): + package = Packages.section.find('Package') + source = Packages.section.find('Source', "") + version = Packages.section.find('Version') if source == "": source = package if bin2source.has_key(package) and \ - apt_pkg.VersionCompare(version, bin2source[package]["version"]) > 0: + apt_pkg.version_compare(version, bin2source[package]["version"]) > 0: bin2source[package]["version"] = version bin2source[package]["source"] = source else: @@ -667,14 +766,6 @@ def main (): nbs[source].setdefault(package, {}) nbs[source][package][version] = "" else: - previous_source = bin_pkgs[package] - if previous_source != source: - key_list = [ source, previous_source ] - key_list.sort() - key = '_'.join(key_list) - duplicate_bins.setdefault(key, []) - if package not in duplicate_bins[key]: - duplicate_bins[key].append(package) if "nfu" in checks: if package in nfu_entries and \ version != source_versions[source]: # only suggest to remove out-of-date packages @@ -688,10 +779,10 @@ def main (): for source in nbs.keys(): for package in nbs[source].keys(): versions = nbs[source][package].keys() - versions.sort(apt_pkg.VersionCompare) + versions.sort(apt_pkg.version_compare) latest_version = versions.pop() source_version = source_versions.get(source,"0") - if apt_pkg.VersionCompare(latest_version, source_version) == 0: + if apt_pkg.version_compare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: @@ -722,15 +813,7 @@ def main (): print if "bms" in checks: - print "Built from multiple source packages" - print "-----------------------------------" - print - keys = duplicate_bins.keys() - keys.sort() - for key in keys: - (source_a, source_b) = key.split("_") - print " o %s & %s => %s" % (source_a, source_b, ", ".join(duplicate_bins[key])) - print + report_multiple_source(suite) if "anais" in checks: print "Architecture Not Allowed In Source"