X-Git-Url: https://git.decadent.org.uk/gitweb/?a=blobdiff_plain;f=dak%2Fcruft_report.py;h=384ad0e7d3ddbc71abf71fa38bc280d32ad00d85;hb=2e37e696ba3b6eb5401727a2dad2288687bb2f5b;hp=b62b2f386f708010ce922faa5878dc7335b37cf5;hpb=0c984dc91cfb5a567190e6eac044f3ab3d15963a;p=dak.git diff --git a/dak/cruft_report.py b/dak/cruft_report.py index b62b2f38..384ad0e7 100755 --- a/dak/cruft_report.py +++ b/dak/cruft_report.py @@ -29,7 +29,7 @@ ################################################################################ -import commands, os, sys, time, re +import commands, os, sys, re import apt_pkg from daklib.config import Config @@ -65,7 +65,7 @@ def add_nbs(nbs_d, source, version, package, suite_id, session): else: q = session.execute("""SELECT b.id FROM binaries b, bin_associations ba WHERE ba.bin = b.id AND ba.suite = :suite_id - AND b.package = :package LIMIT 1""" % {'suite_id': suite_id, + AND b.package = :package LIMIT 1""", {'suite_id': suite_id, 'package': package}) if not q.fetchall(): no_longer_in_suite[package] = "" @@ -92,8 +92,9 @@ def do_anais(architecture, binaries_list, source, session): WHERE ba.suite = :suiteid AND ba.bin = b.id AND b.architecture = a.id AND b.package = :package""", {'suiteid': suite_id, 'package': binary}) + ql = q.fetchall() versions = [] - for i in q.fetchall(): + for i in ql: arch = i[0] version = i[1] if architectures.has_key(arch): @@ -195,7 +196,7 @@ SELECT s.source, s.version AS lower, s2.version AS higher FROM src_associations sa, source s, source s2, src_associations sa2 WHERE sa.suite = :highersuite_id AND sa2.suite = :lowersuite_id AND sa.source = s.id AND sa2.source = s2.id AND s.source = s2.source - AND s.version < s2.version""" % {'lowersuite_id': lowersuite.suite_id, + AND s.version < s2.version""", {'lowersuite_id': lowersuite.suite_id, 'highersuite_id': highersuite.suite_id}) ql = q.fetchall() if ql: @@ -237,7 +238,7 @@ def do_nbs(real_nbs): output += " o %s: %s\n" % (version, ", ".join(packages)) if all_packages: all_packages.sort() - cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s\n\n" % (source, suite, " ".join(all_packages)) + cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s -R\n\n" % (source, suite.suite_name, " ".join(all_packages)) output += "\n" @@ -271,47 +272,64 @@ def do_dubious_nbs(dubious_nbs): ################################################################################ -def do_obsolete_source(duplicate_bins, bin2source): - obsolete = {} - for key in duplicate_bins.keys(): - (source_a, source_b) = key.split('_') - for source in [ source_a, source_b ]: - if not obsolete.has_key(source): - if not source_binaries.has_key(source): - # Source has already been removed - continue - else: - obsolete[source] = [ i.strip() for i in source_binaries[source].split(',') ] - for binary in duplicate_bins[key]: - if bin2source.has_key(binary) and bin2source[binary]["source"] == source: - continue - if binary in obsolete[source]: - obsolete[source].remove(binary) - - to_remove = [] - output = "Obsolete source package\n" - output += "-----------------------\n\n" - obsolete_keys = obsolete.keys() - obsolete_keys.sort() - for source in obsolete_keys: - if not obsolete[source]: - to_remove.append(source) - output += " * %s (%s)\n" % (source, source_versions[source]) - for binary in [ i.strip() for i in source_binaries[source].split(',') ]: - if bin2source.has_key(binary): - output += " o %s (%s) is built by %s.\n" \ - % (binary, bin2source[binary]["version"], - bin2source[binary]["source"]) - else: - output += " o %s is not built.\n" % binary - output += "\n" - - if to_remove: - print output - - print "Suggested command:" - print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove)) - print +def obsolete_source(suite_name, session): + """returns obsolete source packages for suite_name sorted by + install_date""" + + query = """ +SELECT os.src, os.source, os.version, os.install_date + FROM obsolete_source os + JOIN suite s on s.id = os.suite + WHERE s.suite_name = :suite_name + ORDER BY install_date""" + args = { 'suite_name': suite_name } + return session.execute(query, args) + +def source_bin(source, session): + """returns binaries built by source for all or no suite""" + + query = """ +SELECT package + FROM source_bin + WHERE source = :source + ORDER BY package""" + args = { 'source': source } + return session.execute(query, args) + +def newest_source_bab(suite_name, package, session): + """returns newest source that builds binary package in suite""" + + query = """ +SELECT source, srcver + FROM newest_source_bab nsb + JOIN suite s on s.id = nsb.suite + WHERE s.suite_name = :suite_name AND nsb.package = :package + ORDER BY source""" + args = { 'suite_name': suite_name, 'package': package } + return session.execute(query, args) + +def report_obsolete_source(suite_name, session): + rows = obsolete_source(suite_name, session) + if rows.rowcount == 0: + return + print \ +"""Obsolete source packages in suite %s +----------------------------------%s\n""" % \ + (suite_name, '-' * len(suite_name)) + for os_row in rows.fetchall(): + (src, old_source, version, install_date) = os_row + print " * obsolete source %s version %s installed at %s" % \ + (old_source, version, install_date) + for sb_row in source_bin(old_source, session): + (package, ) = sb_row + print " - has built binary %s" % package + for nsb_row in newest_source_bab(suite_name, package, session): + (new_source, srcver) = nsb_row + print " currently built by source %s version %s" % \ + (new_source, srcver) + print " - suggested command:" + rm_opts = "-S -p -m \"[auto-cruft] obsolete source package\"" + print " dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source) def get_suite_binaries(suite, session): # Initalize a large hash table of all binary packages @@ -357,9 +375,9 @@ def main (): # Set up checks based on mode if Options["Mode"] == "daily": - checks = [ "nbs", "nviu", "obsolete source" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source" ] elif Options["Mode"] == "full": - checks = [ "nbs", "nviu", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] + checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ] else: utils.warn("%s is not a recognised mode - only 'full' or 'daily' are understood." % (Options["Mode"])) usage(1) @@ -380,16 +398,20 @@ def main (): suite = get_suite(Options["Suite"].lower(), session) suite_id = suite.suite_id + suite_name = suite.suite_name.lower() + + if "obsolete source" in checks: + report_obsolete_source(suite_name, session) bin_not_built = {} if "bnb" in checks: - bins_in_suite = get_suite_binaries(suite.suite_name, session) + bins_in_suite = get_suite_binaries(suite, session) # Checks based on the Sources files - components = cnf.ValueList("Suite::%s::Components" % (suite)) + components = cnf.ValueList("Suite::%s::Components" % (suite_name)) for component in components: - filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite.suite_name, component) + filename = "%s/dists/%s/%s/source/Sources.gz" % (cnf["Dir::Root"], suite_name, component) # apt_pkg.ParseTagFile needs a real file handle and can't handle a GzipFile instance... (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) @@ -417,8 +439,8 @@ def main (): # Check for duplicated packages and build indices for checking "no source" later source_index = component + '/' + source - if src_pkgs.has_key(source): - print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]) + #if src_pkgs.has_key(source): + # print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source]) src_pkgs[source] = source_index for binary in binaries_list: if bin_pkgs.has_key(binary): @@ -436,17 +458,17 @@ def main (): # Checks based on the Packages files check_components = components[:] - if suite.suite_name != "experimental": + if suite_name != "experimental": check_components.append('main/debian-installer'); for component in check_components: - architectures = [ a.arch_string for a in get_suite_architectures(suite.suite_name, + architectures = [ a.arch_string for a in get_suite_architectures(suite_name, skipsrc=True, skipall=True, session=session) ] for architecture in architectures: if component == 'main/debian-installer' and re.match("kfreebsd", architecture): continue - filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suite.suite_name, component, architecture) + filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (cnf["Dir::Root"], suite_name, component, architecture) # apt_pkg.ParseTagFile needs a real file handle (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) @@ -499,9 +521,6 @@ def main (): packages.close() os.unlink(temp_filename) - if "obsolete source" in checks: - do_obsolete_source(duplicate_bins, bin2source) - # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} real_nbs = {} @@ -519,6 +538,9 @@ def main (): if "nviu" in checks: do_newer_version('unstable', 'experimental', 'NVIU', session) + if "nvit" in checks: + do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) + if "nbs" in checks: do_nbs(real_nbs)